diff --git a/src/ptscrapper/scheduler.py b/src/ptscrapper/scheduler.py index bf1c9fe..feb73fc 100644 --- a/src/ptscrapper/scheduler.py +++ b/src/ptscrapper/scheduler.py @@ -108,13 +108,6 @@ async def positions_job(): table = pa.Table.from_pylist(positions) # write locally only write_positions(table, ts) - - # Schedule immediate fetch for any new course IDs - course_ids = {p["c"] for p in positions} - for cid in course_ids: - if should_fetch_course(cid): - await fetch_and_store_course(cid, ts, client) - logger.info("wrote positions shard for %s", ts) except Exception: # record error details @@ -122,6 +115,12 @@ async def positions_job(): finally: await client.aclose() + # schedule course fetches without awaiting + course_ids = {p["c"] for p in positions} + for cid in course_ids: + if should_fetch_course(cid): + asyncio.create_task(fetch_and_store_course(cid, ts)) + async def fetch_and_store_course(course_id: int, ts: datetime, client=None): """Fetch one course’s details and store them.""" diff --git a/src/ptscrapper/storage.py b/src/ptscrapper/storage.py index e3fb4bf..bfce5d7 100644 --- a/src/ptscrapper/storage.py +++ b/src/ptscrapper/storage.py @@ -190,17 +190,11 @@ def write_course_geometry(polyline_str: str, course_id: int, ts: datetime) -> st wkb_arr = ga.as_wkb([wkt]) # 4) Cast into a WkbType with EPSG:4326 built in - wkb_with_crs = wkb_arr.cast( - ga.wkb().with_crs("EPSG:4326") - ) + wkb_with_crs = wkb_arr.cast(ga.wkb().with_crs("EPSG:4326")) # 5) Build a 1-row table table = pa.Table.from_arrays( - [ - pa.array([ts]), - pa.array([course_id]), - wkb_with_crs - ], + [pa.array([ts]), pa.array([course_id]), wkb_with_crs], names=["fetch_date", "course_id", "geometry"], )