piotr/pt-scrapper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

store timestamps for positions

Browse source
This commit is contained in:
Piotr Oleszczyk 2025-05-12 14:30:43 +02:00
parent a5349123ce
commit f37fd68e07
2 changed files with 36 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

14
src/ptscrapper/storage.py
View file

@ -24,10 +24,14 @@ def _ensure_dir(path: str):
os.makedirs(path, exist_ok=True)
def write_positions(table: pa.Table, ts: datetime) -> str:
def write_positions(
table: pa.Table,
server_ts: datetime,
) -> str:
"""
Convert x/y to a Point geometry, drop the originals,
and write out a GeoParquet.
and write out a GeoParquet, carrying through
the server_ts and fetched_ts columns.
"""
# 1) Extract x/y as numpy arrays (zero_copy_only=False to ensure a NumPy copy)
xs = table.column("x").to_numpy()
@ -44,11 +48,13 @@ def write_positions(table: pa.Table, ts: datetime) -> str:
# 4) drop old coords & append 'geometry'
table = table.drop(["x", "y"]).append_column("geometry", geom)
subdir = ts.strftime("positions/%Y/%m/%d/%H")
# (server_ts and fetched_ts are already columns in `table`)
subdir = server_ts.strftime("positions/%Y/%m/%d/%H")
local_dir = os.path.join(LOCAL_DATA_DIR, subdir)
_ensure_dir(local_dir)
filename = f"{ts.strftime('%M%S')}_{HOST_ID}.parquet"
filename = f"{server_ts.strftime('%M%S')}_{HOST_ID}.parquet"
local_path = os.path.join(local_dir, filename)
pq.write_table(table, local_path, compression="snappy")