piotr/pt-scrapper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix slow position grabbing

Browse source
This commit is contained in:
Piotr Oleszczyk 2025-05-10 18:43:51 +02:00
parent c6d1d51b00
commit 8975c158a5
2 changed files with 8 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

13
src/ptscrapper/scheduler.py
View file

@ -108,13 +108,6 @@ async def positions_job():
table = pa.Table.from_pylist(positions)
# write locally only
write_positions(table, ts)
# Schedule immediate fetch for any new course IDs
course_ids = {p["c"] for p in positions}
for cid in course_ids:
if should_fetch_course(cid):
await fetch_and_store_course(cid, ts, client)
logger.info("wrote positions shard for %s", ts)
except Exception:
# record error details
@ -122,6 +115,12 @@ async def positions_job():
finally:
await client.aclose()
# schedule course fetches without awaiting
course_ids = {p["c"] for p in positions}
for cid in course_ids:
if should_fetch_course(cid):
asyncio.create_task(fetch_and_store_course(cid, ts))
async def fetch_and_store_course(course_id: int, ts: datetime, client=None):
"""Fetch one course’s details and store them."""