fio-l1-nyc-csvwkt-addresses

Rerun

total 144
drwxr-xr-x 1 agent agent     34 Jun 16 17:56 .
drwxr-xr-x 1 root  root      14 Jun 16 17:56 ..
-rw-r--r-- 1 agent agent 143422 Jun 16 17:56 nyc_addresses.csv

Traceback (most recent call last):
  File "/work/solve.py", line 14, in <module>
    gdf["recorded_at"] = pd.to_datetime(gdf["recorded_at"]).astype("datetime64[us]")
                         ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.14/site-packages/pandas/core/generic.py", line 6541, in astype
    new_data = self._mgr.astype(dtype=dtype, errors=errors)
  File "/usr/local/lib/python3.14/site-packages/pandas/core/internals/managers.py", line 614, in astype
    return self.apply("astype", dtype=dtype, errors=errors)
           ~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.14/site-packages/pandas/core/internals/managers.py", line 445, in apply
    applied = getattr(b, f)(**kwargs)
  File "/usr/local/lib/python3.14/site-packages/pandas/core/internals/blocks.py", line 607, in astype
    new_values = astype_array_safe(values, dtype, errors=errors)
  File "/usr/local/lib/python3.14/site-packages/pandas/core/dtypes/astype.py", line 240, in astype_array_safe
    new_values = astype_array(values, dtype, copy=copy)
  File "/usr/local/lib/python3.14/site-packages/pandas/core/dtypes/astype.py", line 182, in astype_array
    values = values.astype(dtype, copy=copy)
  File "/usr/local/lib/python3.14/site-packages/pandas/core/arrays/datetimes.py", line 745, in astype
    raise TypeError(
    ...<3 lines>...
    )
TypeError: Cannot use .astype to convert from timezone-aware dtype to timezone-naive dtype. Use obj.tz_localize(None) or obj.tz_convert('UTC').tz_localize(None) instead.

=== Schema ===
id                        str
country                   str
postcode                  str
street                    str
number                    str
unit                      str
postal_city               str
recorded_at    datetime64[us]
unit_count              int32
geometry             geometry
dtype: object

=== Geometry CRS ===
EPSG:4326

=== Record count ===
1056

=== Sample data ===
                                     id  ...                    geometry
0  0005eda0-2540-4e36-bc81-dfb52e323d99  ...  POINT (-73.99223 40.71616)
1  00587e13-6253-485d-9258-e47d43af7559  ...  POINT (-73.99833 40.71204)
2  0070f126-d36a-43b3-b767-a8fbb7156602  ...   POINT (-74.0015 40.70733)
3  00a8fc38-5d9a-42da-94a9-7d2f4101f82c  ...  POINT (-73.99286 40.70268)
4  00f815af-51f6-44c2-b0d6-ee23092d14b2  ...   POINT (-74.0176 40.71024)

[5 rows x 10 columns]

cd /work && duckdb -c "
INSTALL spatial; LOAD spatial;
SELECT COUNT(*) AS cnt, SUM(unit_count) AS total_units
FROM 'addresses.geoparquet'
WHERE recorded_at > '2024-01-01';
"

cd /work && python -c "
import pyarrow.parquet as pq
import pyarrow.dataset as ds

# Read and show schema
schema = pq.read_schema('addresses.geoparquet')
print('=== Parquet Schema ===')
print(schema)

# Verify queries work
t = ds.dataset('addresses.geoparquet')
import pyarrow.compute as pc

# Filter: recorded_at > '2024-01-01'
filt = pc.field('recorded_at') > pc.scalar('2024-01-01', type='timestamp[us]')
result = t.to_table(filter=filt)
print(f'\n=== Rows with recorded_at > 2024-01-01: {len(result)} ===')

# SUM(unit_count)
total = pc.sum(result.column('unit_count'))
print(f'SUM(unit_count) = {total}')
"

=== Parquet Schema ===
id: large_string
country: large_string
postcode: large_string
street: large_string
number: large_string
unit: large_string
postal_city: large_string
recorded_at: timestamp[us]
unit_count: int32
geometry: binary
  -- field metadata --
  ARROW:extension:name: 'geoarrow.wkb'
  ARROW:extension:metadata: '{"crs": {"$schema": "https://proj.org/schema' + 1498
-- schema metadata --
pandas: '{"index_columns": [], "column_indexes": [], "columns": [{"name":' + 1215
geo: '{"primary_column": "geometry", "columns": {"geometry": {"encoding":' + 1373

Traceback (most recent call last):
  File "<string>", line 15, in <module>
    filt = pc.field('recorded_at') > pc.scalar('2024-01-01', type='timestamp[us]')
                                     ~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
TypeError: scalar() got an unexpected keyword argument 'type'

cd /work && python -c "
import pandas as pd
import geopandas as gpd

gdf = gpd.read_parquet('addresses.geoparquet')

# Filter: recorded_at > '2024-01-01'
mask = gdf['recorded_at'] > pd.Timestamp('2024-01-01')
filtered = gdf[mask]
print(f'Rows with recorded_at > 2024-01-01: {len(filtered)}')

# SUM(unit_count)
print(f'SUM(unit_count) = {filtered[\"unit_count\"].sum()}')

# Verify datatypes
print(f'\nrecorded_at dtype: {gdf[\"recorded_at\"].dtype}')
print(f'unit_count dtype: {gdf[\"unit_count\"].dtype}')
print(f'geometry dtype: {gdf[\"geometry\"].dtype}')
"

Rows with recorded_at > 2024-01-01: 1055
SUM(unit_count) = 1566

recorded_at dtype: datetime64[us]
unit_count dtype: int32
geometry dtype: geometry