fix slow position grabbing
This commit is contained in:
parent
c6d1d51b00
commit
8975c158a5
2 changed files with 8 additions and 15 deletions
|
|
@ -108,13 +108,6 @@ async def positions_job():
|
||||||
table = pa.Table.from_pylist(positions)
|
table = pa.Table.from_pylist(positions)
|
||||||
# write locally only
|
# write locally only
|
||||||
write_positions(table, ts)
|
write_positions(table, ts)
|
||||||
|
|
||||||
# Schedule immediate fetch for any new course IDs
|
|
||||||
course_ids = {p["c"] for p in positions}
|
|
||||||
for cid in course_ids:
|
|
||||||
if should_fetch_course(cid):
|
|
||||||
await fetch_and_store_course(cid, ts, client)
|
|
||||||
|
|
||||||
logger.info("wrote positions shard for %s", ts)
|
logger.info("wrote positions shard for %s", ts)
|
||||||
except Exception:
|
except Exception:
|
||||||
# record error details
|
# record error details
|
||||||
|
|
@ -122,6 +115,12 @@ async def positions_job():
|
||||||
finally:
|
finally:
|
||||||
await client.aclose()
|
await client.aclose()
|
||||||
|
|
||||||
|
# schedule course fetches without awaiting
|
||||||
|
course_ids = {p["c"] for p in positions}
|
||||||
|
for cid in course_ids:
|
||||||
|
if should_fetch_course(cid):
|
||||||
|
asyncio.create_task(fetch_and_store_course(cid, ts))
|
||||||
|
|
||||||
|
|
||||||
async def fetch_and_store_course(course_id: int, ts: datetime, client=None):
|
async def fetch_and_store_course(course_id: int, ts: datetime, client=None):
|
||||||
"""Fetch one course’s details and store them."""
|
"""Fetch one course’s details and store them."""
|
||||||
|
|
|
||||||
|
|
@ -190,17 +190,11 @@ def write_course_geometry(polyline_str: str, course_id: int, ts: datetime) -> st
|
||||||
wkb_arr = ga.as_wkb([wkt])
|
wkb_arr = ga.as_wkb([wkt])
|
||||||
|
|
||||||
# 4) Cast into a WkbType with EPSG:4326 built in
|
# 4) Cast into a WkbType with EPSG:4326 built in
|
||||||
wkb_with_crs = wkb_arr.cast(
|
wkb_with_crs = wkb_arr.cast(ga.wkb().with_crs("EPSG:4326"))
|
||||||
ga.wkb().with_crs("EPSG:4326")
|
|
||||||
)
|
|
||||||
|
|
||||||
# 5) Build a 1-row table
|
# 5) Build a 1-row table
|
||||||
table = pa.Table.from_arrays(
|
table = pa.Table.from_arrays(
|
||||||
[
|
[pa.array([ts]), pa.array([course_id]), wkb_with_crs],
|
||||||
pa.array([ts]),
|
|
||||||
pa.array([course_id]),
|
|
||||||
wkb_with_crs
|
|
||||||
],
|
|
||||||
names=["fetch_date", "course_id", "geometry"],
|
names=["fetch_date", "course_id", "geometry"],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue