fio-l1-nyc-csvwkt-addresses

Rerun

import pandas as pd
import geopandas as gpd
from shapely import wkt

def convert():
    # Load CSV
    # Note: The CSV seems to have quotes. pandas handles this well.
    df = pd.read_csv('nyc_addresses.csv')
    
    # 1. Handle Geometry
    # geometry_wkt is in the CSV
    df['geometry'] = df['geometry_wkt'].apply(wkt.loads)
    gdf = gpd.GeoDataFrame(df, geometry='geometry', crs="EPSG:4326")
    
    # 2. Prepare columns
    # We need to drop geometry_wkt as we are using the geometry column
    gdf = gdf.drop(columns=['geometry_wkt'])
    
    # 3. Type Conversions
    # recorded_at as timestamp[us]
    gdf['recorded_at'] = pd.to_datetime(gdf['recorded_at']).dt.floor('us')
    
    # unit_count as int32
    gdf['unit_count'] = gdf['unit_count'].astype('int32')
    
    # All other columns as string
    # Identify other columns (excluding geometry and the ones we just handled specifically)
    # But the requirement says "all other columns as string". 
    # This implies everything except geometry, recorded_at, and unit_count.
    
    # Let's define the target types
    # geometry is special (GeoParquet)
    # recorded_at: datetime64[us]
    # unit_count: int32
    # everything else: str
    
    cols_to_convert = [col for col in gdf.columns if col not in ['geometry', 'recorded_at', 'unit_count']]
    for col in cols_to_convert:
        gdf[col] = gdf[col].astype(str)
        # Handle cases where 'nan' strings might be undesirable, 
        # but the prompt specifically says "all other columns as string".
        # Often in parquet, we'd want to preserve nulls, but if it's a string column, 
        # 'None' or empty string is common. Let's just use .astype(str) 
        # which might turn NaN into "nan".
        # Actually, it's better to use .astype(object) and then fillna or handle carefully.
        # But let's stick to the literal instruction: "all other columns as string".
        # In pandas/pyarrow, string columns are usually object or string.
    
    # Re-order columns to ensure geometry is handled correctly if needed, 
    # though GeoParquet usually expects a geometry column.
    
    # 4. Export to GeoParquet
    # Using pyarrow engine via geopandas
    gdf.to_parquet('addresses.geoparquet', index=False)
    print("Conversion successful: addresses.geoparquet created.")

if __name__ == "__main__":
    convert()

import pandas as pd
import geopandas as gpd
from shapely import wkt

def convert():
    # Load CSV
    df = pd.read_csv('nyc_addresses.csv')
    
    # 1. Handle Geometry
    df['geometry'] = df['geometry_wkt'].apply(wkt.loads)
    gdf = gpd.GeoDataFrame(df, geometry='geometry', crs="EPSG:4326")
    
    # Drop the old WKT column
    gdf = gdf.drop(columns=['geometry_wkt'])
    
    # 2. Type Conversions
    # recorded_at as timestamp[us]
    gdf['recorded_at'] = pd.to_datetime(gdf['recorded_at']).dt.floor('us')
    
    # unit_count as int32
    gdf['unit_count'] = gdf['unit_count'].astype('int32')
    
    # All other columns as string
    # In pandas/pyarrow, 'string' type is preferred for parquet
    cols_to_convert = [col for col in gdf.columns if col not in ['geometry', 'recorded_at', 'unit_count']]
    for col in cols_to_convert:
        gdf[col] = gdf[col].astype("string")
    
    # 3. Export to GeoParquet
    gdf.to_parquet('addresses.geoparquet', index=False)
    print("Conversion successful: addresses.geoparquet created.")

if __name__ == "__main__":
    convert()