fix slow position grabbing

This commit is contained in:
Piotr Oleszczyk 2025-05-10 18:43:51 +02:00
parent c6d1d51b00
commit 8975c158a5
2 changed files with 8 additions and 15 deletions

View file

@ -108,13 +108,6 @@ async def positions_job():
table = pa.Table.from_pylist(positions) table = pa.Table.from_pylist(positions)
# write locally only # write locally only
write_positions(table, ts) write_positions(table, ts)
# Schedule immediate fetch for any new course IDs
course_ids = {p["c"] for p in positions}
for cid in course_ids:
if should_fetch_course(cid):
await fetch_and_store_course(cid, ts, client)
logger.info("wrote positions shard for %s", ts) logger.info("wrote positions shard for %s", ts)
except Exception: except Exception:
# record error details # record error details
@ -122,6 +115,12 @@ async def positions_job():
finally: finally:
await client.aclose() await client.aclose()
# schedule course fetches without awaiting
course_ids = {p["c"] for p in positions}
for cid in course_ids:
if should_fetch_course(cid):
asyncio.create_task(fetch_and_store_course(cid, ts))
async def fetch_and_store_course(course_id: int, ts: datetime, client=None): async def fetch_and_store_course(course_id: int, ts: datetime, client=None):
"""Fetch one courses details and store them.""" """Fetch one courses details and store them."""

View file

@ -190,17 +190,11 @@ def write_course_geometry(polyline_str: str, course_id: int, ts: datetime) -> st
wkb_arr = ga.as_wkb([wkt]) wkb_arr = ga.as_wkb([wkt])
# 4) Cast into a WkbType with EPSG:4326 built in # 4) Cast into a WkbType with EPSG:4326 built in
wkb_with_crs = wkb_arr.cast( wkb_with_crs = wkb_arr.cast(ga.wkb().with_crs("EPSG:4326"))
ga.wkb().with_crs("EPSG:4326")
)
# 5) Build a 1-row table # 5) Build a 1-row table
table = pa.Table.from_arrays( table = pa.Table.from_arrays(
[ [pa.array([ts]), pa.array([course_id]), wkb_with_crs],
pa.array([ts]),
pa.array([course_id]),
wkb_with_crs
],
names=["fetch_date", "course_id", "geometry"], names=["fetch_date", "course_id", "geometry"],
) )