process times in courses properly
This commit is contained in:
parent
f37fd68e07
commit
6891e3b206
1 changed files with 15 additions and 9 deletions
|
|
@ -158,15 +158,21 @@ async def fetch_and_store_course(course_id: int, ts: datetime, client=None):
|
|||
recs = await fetch_course_posts(client, [course_id])
|
||||
# now do your I/O; catch+log only errors here
|
||||
for rec in recs:
|
||||
rows = [
|
||||
rows = []
|
||||
for seq, stop in enumerate(rec.get("r", []), start=1):
|
||||
# stop["t"] is like "1900-01-01 05:53:00"
|
||||
# parse out the time portion
|
||||
t = datetime.strptime(stop["t"], "%Y-%m-%d %H:%M:%S").time()
|
||||
rows.append(
|
||||
{
|
||||
"fetch_date": ts,
|
||||
"fetch_date": ts, # UTC fetch timestamp
|
||||
"course_id": rec["c"],
|
||||
"stop_seq": seq, # preserve order!
|
||||
"stop_id": stop["s"],
|
||||
"dep_time": stop["t"],
|
||||
"dep_time": t, # real datetime.time
|
||||
"dep_zone": "Europe/Warsaw",
|
||||
}
|
||||
for stop in rec.get("r", [])
|
||||
]
|
||||
)
|
||||
table = pa.Table.from_pylist(rows)
|
||||
write_course_posts(table, rec["c"], ts)
|
||||
record_course_fetched(rec["c"], ts)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue