piotr/pt-scrapper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

remove daily course sweep

Browse source
This commit is contained in:
Piotr Oleszczyk 2025-05-10 19:09:04 +02:00
parent 746b569ad8
commit 36f3f8e50a
1 changed files with 0 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

40
src/ptscrapper/scheduler.py
View file

@ -186,45 +186,6 @@ async def daily_snapshot_job():
await client.aclose()
async def daily_course_sweep_job():
"""Once-a-day refetch of all known courses to detect changes."""
ts = datetime.now(timezone.utc)
# read all course_ids
conn = sqlite3.connect(STATE_DB)
c = conn.cursor()
c.execute("SELECT course_id FROM courses")
course_ids = [row[0] for row in c.fetchall()]
conn.close()
client = get_async_client()
try:
# fetch in batches of 10
for batch in chunked(course_ids, 10):
try:
recs = await fetch_course_posts(client, batch)
for rec in recs:
# flatten & store, same as in fetch_and_store_course
rows = [
{
"fetch_date": ts,
"course_id": rec["c"],
"stop_id": stop["s"],
"dep_time": stop["t"],
}
for stop in rec.get("r", [])
]
table = pa.Table.from_pylist(rows)
write_course_posts(table, rec["c"], ts)
record_course_fetched(rec["c"], ts)
write_course_geometry(rec["p"], rec["c"], ts)
logger.info("wrote course %s during daily sweep", rec["c"])
except Exception:
logger.exception("job=daily_course_sweep_job")
finally:
await client.aclose()
async def daily_compact_and_push():
# compact yesterday’s positions once/day
yesterday = date.today() - timedelta(days=1)
@ -298,7 +259,6 @@ async def start_scheduler():
# daily data jobs (UTC times here)
scheduler.add_job(daily_snapshot_job, CronTrigger(hour=2, minute=0))
scheduler.add_job(daily_course_sweep_job, CronTrigger(hour=3, minute=0))
scheduler.add_job(daily_compact_and_push, CronTrigger(hour=4, minute=0))
scheduler.add_job(daily_compact_and_push_courses, CronTrigger(hour=5, minute=0))
scheduler.add_job(