fio-l1-nyc-csvwkt-addresses

Rerun

# /// script
# requires-python = ">=3.12"
# dependencies = [
#     "pandas",
#     "geopandas",
#     "shapely",
#     "pyarrow",
#     "pyproj"
# ]
# ///

import pandas as pd
import geopandas as gpd
from shapely import wkt

def convert_to_geoparquet(input_csv, output_parquet):
    # Load the CSV
    df = pd.read_csv(input_csv)

    # 1. Convert geometry_wkt to geometry
    df['geometry'] = df['geometry_wkt'].apply(wkt.loads)
    
    # Create GeoDataFrame with EPSG:4326
    gdf = gpd.GeoDataFrame(df, geometry='geometry', crs="EPSG:4326")

    # 2. Convert recorded_at to timestamp[us]
    # pandas datetime is usually microsecond or nanosecond. 
    # We'll ensure it's datetime objects.
    gdf['recorded_at'] = pd.to_datetime(gdf['recorded_at'])

    # 3. Convert unit_count to int32
    gdf['unit_count'] = gdf['unit_count'].astype('int32')

    # 4. All other columns as string (except geometry and recorded_at)
    # We need to handle the columns specifically.
    # The columns are: id, country, postcode, street, number, unit, postal_city, recorded_at, unit_count, geometry_wkt, geometry
    
    # Let's list all columns and identify which should be strings.
    # We already have geometry and recorded_at and unit_count.
    # We want to keep geometry, recorded_at, unit_count.
    # Everything else: id, country, postcode, street, number, unit, postal_city, geometry_wkt
    
    cols_to_string = ['id', 'country', 'postcode', 'street', 'number', 'unit', 'postal_city', 'geometry_wkt']
    for col in cols_to_string:
        if col in gdf.columns:
            gdf[col] = gdf[col].astype(str)

    # Drop the extra geometry_wkt if we want a clean GeoParquet, 
    # though the prompt says "all other columns as string". 
    # In GeoParquet, 'geometry' is a special column.
    # The instructions says: "all other columns as string". 
    # This implies geometry_wkt should stay (as a string) and others too.
    
    # Let's re-verify the requirements:
    # Point geometry in EPSG:4326 -> Done (gdf.geometry)
    # recorded_at as timestamp[us] -> Done (datetime64[ns] is standard in pandas/pyarrow, 
    # but pyarrow will handle it. To be precise about [us], we might need to cast)
    # unit_count as int32 -> Done
    # all other columns as string -> Done
    
    # Actually, let's make sure recorded_at is microsecond precision if possible
    gdf['recorded_at'] = gdf['recorded_at'].dt.floor('us')

    # Re-ordering or cleaning? Let's just save.
    # We should probably drop 'geometry_wkt' if it's redundant, but the prompt says "all other columns as string".
    # To be safe, I will keep it.
    
    # Save to GeoParquet
    gdf.to_parquet(output_parquet)
    print(f"Successfully converted {input_csv} to {output_parquet}")
    print("Column types:")
    print(gdf.dtypes)

if __name__ == "__main__":
    convert_to_geoparquet('nyc_addresses.csv', 'addresses.geoparquet')

# /// script
# requires-python = ">=3.12"
# dependencies = [
#     "pandas",
#     "geopandas",
#     "shapely",
#     "pyarrow",
#     "pyproj"
# ]
# ///

import pandas as pd
import geopandas as gpd
from shapely import wkt

def convert_to_geoparquet(input_csv, output_parquet):
    # Load the CSV
    df = pd.read_csv(input_csv)

    # 1. Convert geometry_wkt to geometry
    df['geometry'] = df['geometry_wkt'].apply(wkt.loads)
    
    # Create GeoDataFrame with EPSG:4326
    gdf = gpd.GeoDataFrame(df, geometry='geometry', crs="EPSG:4326")

    # 2. Convert recorded_at to timestamp[us]
    # Ensure it's datetime and then cast to microsecond precision
    gdf['recorded_at'] = pd.to_datetime(gdf['recorded_at'], utc=True).dt.floor('us')

    # 3. Convert unit_count to int32
    gdf['unit_count'] = gdf['unit_count'].astype('int32')

    # 4. All other columns as string
    # We must identify all columns that are NOT 'geometry', 'recorded_at', or 'unit_count'
    # and convert them to string.
    
    special_cols = ['geometry', 'recorded_at', 'unit_count']
    for col in gdf.columns:
        if col not in special_cols:
            gdf[col] = gdf[col].astype(str)

    # Save to GeoParquet
    # We use engine='pyarrow' to ensure better control over types if needed,
    # but to_parquet in geopandas usually works well.
    gdf.to_parquet(output_parquet)
    print(f"Successfully converted {input_csv} to {output_parquet}")
    print("Column types:")
    print(gdf.dtypes)

if __name__ == "__main__":
    convert_to_geoparquet('nyc_addresses.csv', 'addresses.geoparquet')

# /// script
# requires-python = ">=3.12"
# dependencies = [
#     "pandas",
#     "geopandas",
#     "shapely",
#     "pyarrow",
#     "pyproj"
# ]
# ///

import pandas as pd
import geopandas as gpd
from shapely import wkt

def convert_to_geoparquet(input_csv, output_parquet):
    # Load the CSV
    df = pd.read_csv(input_csv)

    # 1. Convert geometry_wkt to geometry
    df['geometry'] = df['geometry_wkt'].apply(wkt.loads)
    
    # Create GeoDataFrame with EPSG:4326
    gdf = gpd.GeoDataFrame(df, geometry='geometry', crs="EPSG:4326")

    # 2. Convert recorded_at to timestamp[us]
    # We use utc=True to ensure it's treated as a timestamp
    # floor('us') ensures microsecond precision
    gdf['recorded_at'] = pd.to_datetime(gdf['recorded_at'], utc=True).dt.floor('us')

    # 3. Convert unit_count to int32
    gdf['unit_count'] = gdf['unit_count'].astype('int32')

    # 4. All other columns as string
    # The prompt says: "all other columns as string"
    # This means id, country, postcode, street, number, unit, postal_city, geometry_wkt
    # should be strings.
    
    # List of columns to KEEP as their specific types
    keep_types = ['geometry', 'recorded_at', 'unit_count']
    
    for col in gdf.columns:
        if col not in keep_types:
            gdf[col] = gdf[col].astype(str)

    # Final check on columns to be sure we don't have any unintended types
    # We want to make sure the column 'geometry' is the active geometry.
    
    # Save to GeoParquet
    gdf.to_parquet(output_parquet)
    print(f"Successfully converted {input_csv} to {output_parquet}")
    print("Column types:")
    print(gdf.dtypes)

if __name__ == "__main__":
    convert_to_geoparquet('nyc_addresses.csv', 'addresses.geoparquet')