dc-l2-cairo-invalid-dedup

Rerun

import geopandas as gpd

# Load the data
gdf = gpd.read_file('/work/cairo_parcels_legacy.geojson')

# Inspect columns and first few rows
print("Columns:", gdf.columns.tolist())
print("\nFirst few rows:")
print(gdf.head())
print("\nInfo:")
print(gdf.info())
print("\nGeometry types:")
print(gdf.geometry.type.value_counts())

import geopandas as gpd
from shapely.geometry import MultiPolygon, Polygon

def solve():
    # 1. Load the data
    gdf = gpd.read_file('/work/cairo_parcels_legacy.geojson')

    # 2. Clean geometries: remove null/empty and fix invalid geometries
    # Drop null/empty geometries first
    gdf = gdf[gdf.geometry.notnull() & ~gdf.geometry.is_empty].copy()
    
    # Fix invalid geometries (e.g. self-intersections) using buffer(0)
    gdf['geometry'] = gdf['geometry'].buffer(0)
    # Re-filter in case buffer(0) produced empty geometries
    gdf = gdf[gdf.geometry.notnull() & ~gdf.geometry.is_empty].copy()

    # 3. Handle Duplicates: keep record with earliest record_seq per parcel_id
    # Sort by parcel_id and record_seq (ascending)
    gdf = gdf.sort_values(by=['parcel_id', 'record_seq'], ascending=[True, True])
    # Drop duplicates based on parcel_id, keeping the first one (which has lowest record_seq)
    gdf = gdf.drop_duplicates(subset=['parcel_id'], keep='first').copy()

    # 4. Discard polygon fragments smaller than 1 m2
    # Note: The prompt asks to discard fragments smaller than 1m2. 
    # Since we are working with the parsed geometries, we check area of the geometry.
    gdf['area_calc'] = gdf.geometry.area
    gdf = gdf[gdf['area_calc'] >= 1.0].copy()
    # Drop the helper column
    gdf = gdf.drop(columns=['area_calc'])

    # 5. Ensure every feature is a MultiPolygon
    def ensure_multipolygon(geom):
        if isinstance(geom, Polygon):
            return MultiPolygon([geom])
        return geom

    gdf['geometry'] = gdf['geometry'].apply(ensure_multipolygon)

    # 6. Remove duplicate geometries
    # If multiple parcel_ids now have the exact same geometry, we should probably 
    # keep only one. The prompt says "no duplicate geometries".
    # To be safe, we drop duplicate geometries. 
    # Note: In a real registry, different parcels might share a geometry (e.g. zero-area or identical boundaries),
    # but the instruction says "no duplicate geometries".
    gdf = gdf.drop_duplicates(subset=['geometry']).copy()

    # 7. Final cleaning: Ensure no nulls in required columns and columns exist
    # Required columns: parcel_id, parcel_class, district, area_m2
    # area_m2 was in the original, but let's ensure it's accurate based on the geometry
    gdf['area_m2'] = gdf.geometry.area
    
    # Check for nulls in required columns
    cols_to_keep = ['parcel_id', 'parcel_class', 'district', 'area_m2', 'geometry']
    gdf = gdf[cols_to_keep]
    
    # Drop any remaining rows with nulls in these essential columns
    gdf = gdf.dropna(subset=['parcel_id', 'parcel_class', 'district', 'area_m2'])

    # Final verification of geometry types
    # All should be MultiPolygon
    unique_types = gdf.geometry.type.unique()
    print(f"Unique geometry types remaining: {unique_types}")
    
    if any(t != 'MultiPolygon' for t in unique_types):
         # This shouldn't happen given our logic, but as a safeguard:
         gdf['geometry'] = gdf['geometry'].apply(ensure_multipolygon)

    # 8. Write to GeoParquet
    gdf.to_parquet('/work/parcels_canonical.geoparquet')
    print("Successfully wrote parcels_canonical.geoparquet")
    print(f"Final count: {len(gdf)}")

if __name__ == "__main__":
    solve()

import geopandas as gpd

def verify():
    gdf = gpd.read_parquet('/work/parcels_canonical.geoparquet')
    
    print("Shape:", gdf.shape)
    print("\nColumns:", gdf.columns.tolist())
    
    # Check geometry type
    geom_types = gdf.geometry.type.unique()
    print("Geometry types:", geom_types)
    
    # Check for null geometries
    print("Null geometries:", gdf.geometry.isnull().any())
    print("Empty geometries:", gdf.geometry.is_empty.any())
    
    # Check for invalid geometries
    print("Invalid geometries:", not gdf.geometry.is_valid.all())
    
    # Check for duplicate geometries
    # We use the WKB representation to compare geometries accurately
    duplicated_geoms = gdf.geometry.duplicated().any()
    print("Duplicate geometries:", duplicated_geoms)
    
    # Check column requirements
    required_cols = ['parcel_id', 'parcel_class', 'district', 'area_m2']
    missing_cols = [c for c in required_cols if c not in gdf.columns]
    print("Missing required columns:", missing_cols)
    
    # Check for nulls in required columns
    nulls_in_cols = gdf[required_cols].isnull().any().any()
    print("Nulls in required columns:", nulls_in_cols)
    
    # Check area requirement (> 1)
    small_areas = (gdf.geometry.area < 1.0).any()
    print("Areas < 1m2:", small_areas)

if __name__ == "__main__":
    verify()