piotr/pt-scrapper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

process times in courses properly

Browse source
This commit is contained in:
Piotr Oleszczyk 2025-05-12 14:36:37 +02:00
parent f37fd68e07
commit 6891e3b206
1 changed files with 15 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

16
src/ptscrapper/scheduler.py
View file

@ -158,15 +158,21 @@ async def fetch_and_store_course(course_id: int, ts: datetime, client=None):
recs = await fetch_course_posts(client, [course_id]) recs = await fetch_course_posts(client, [course_id])
# now do your I/O; catch+log only errors here # now do your I/O; catch+log only errors here
for rec in recs: for rec in recs:
rows = [ rows = []
for seq, stop in enumerate(rec.get("r", []), start=1):
# stop["t"] is like "1900-01-01 05:53:00"
# parse out the time portion
t = datetime.strptime(stop["t"], "%Y-%m-%d %H:%M:%S").time()
rows.append(
{ {
"fetch_date": ts, "fetch_date": ts, # UTC fetch timestamp
"course_id": rec["c"], "course_id": rec["c"],
"stop_seq": seq, # preserve order!
"stop_id": stop["s"], "stop_id": stop["s"],
"dep_time": stop["t"], "dep_time": t, # real datetime.time
"dep_zone": "Europe/Warsaw",
} }
for stop in rec.get("r", []) )
]
table = pa.Table.from_pylist(rows) table = pa.Table.from_pylist(rows)
write_course_posts(table, rec["c"], ts) write_course_posts(table, rec["c"], ts)
record_course_fetched(rec["c"], ts) record_course_fetched(rec["c"], ts)