dc-l2-cairo-invalid-dedup

Rerun

import geopandas as gpd

def check_file(filename):
    try:
        gdf = gpd.read_file(filename)
        print(f"File: {filename}")
        print(f"Shape: {gdf.shape}")
        print(f"Columns: {gdf.columns.tolist()}")
        print(f"CRS: {gdf.crs}")
        print("\nFirst 5 rows:")
        print(gdf.head())
        print("\nInfo:")
        print(gdf.info())
    except Exception as e:
        print(f"Error reading {filename}: {e}")

if __name__ == "__main__":
    check_file("cairo_parcels_legacy.geojson")

import geopandas as gpd
from shapely.geometry import MultiPolygon, Polygon

def clean_parcels(input_file, output_file):
    print(f"Reading {input_file}...")
    gdf = gpd.read_file(input_file)
    
    initial_count = len(gdf)
    print(f"Initial count: {initial_count}")

    # 1. Remove null or empty geometries
    print("Removing null or empty geometries...")
    gdf = gdf[gdf.geometry.notnull() & ~gdf.geometry.is_empty]
    print(f"Count after removing null/empty: {len(gdf)}")

    # 2. Fix invalid geometries
    print("Fixing invalid geometries...")
    gdf['geometry'] = gdf.geometry.make_valid()
    
    # 3. Convert all to MultiPolygon
    print("Converting all geometries to MultiPolygon...")
    def ensure_multipolygon(geom):
        if geom.geom_type == 'Polygon':
            return MultiPolygon([geom])
        return geom

    gdf['geometry'] = gdf.geometry.apply(ensure_multipolygon)

    # 4. Handle duplicates: keep record with earliest record_seq
    print("Handling duplicates (keeping earliest record_seq)...")
    # Sort by parcel_id and record_seq
    gdf = gdf.sort_values(by=['parcel_id', 'record_seq'], ascending=[True, True])
    # Drop duplicates on parcel_id, keeping the first (which is the lowest record_seq)
    gdf = gdf.drop_duplicates(subset='parcel_id', keep='first')
    print(f"Count after deduplication: {len(gdf)}")

    # 5. Discard polygon fragments smaller than 1 m2
    print("Removing fragments smaller than 1 m2...")
    # We need to check the area of the geometry
    gdf = gdf[gdf.geometry.area >= 1.0]
    print(f"Count after area filter: {len(gdf)}")

    # 6. Remove duplicate geometries (different IDs but same shape)
    # This is a specific requirement: "no duplicate geometries"
    print("Removing duplicate geometries...")
    # Convert to WKT for reliable comparison if necessary, but geopandas/shapely handles this
    # We'll use a trick: create a string representation to identify duplicates
    gdf['geom_wkt'] = gdf.geometry.apply(lambda x: x.wkt)
    gdf = gdf.drop_duplicates(subset='geom_wkt')
    gdf = gdf.drop(columns=['geom_wkt'])
    print(f"Count after removing duplicate geometries: {len(gdf)}")

    # 7. Final column selection
    print("Selecting final columns...")
    final_cols = ['parcel_id', 'parcel_class', 'district', 'area_m2', 'geometry']
    gdf = gdf[final_cols]

    # Ensure no nulls in required columns
    gdf = gdf.dropna(subset=['parcel_id', 'parcel_class', 'district', 'area_m2'])

    # 8. Write to GeoParquet
    print(f"Writing to {output_file}...")
    gdf.to_parquet(output_file)
    print("Done!")

if __name__ == "__main__":
    clean_parcels("cairo_parcels_legacy.geojson", "parcels_canonical.geoparquet")

import geopandas as gpd

def verify_output(filename):
    print(f"Verifying {filename}...")
    gdf = gpd.read_file(filename)
    
    print(f"Shape: {gdf.shape}")
    print(f"Columns: {gdf.columns.tolist()}")
    
    # Check MultiPolygon
    is_multipolygon = all(geom.geom_type == 'MultiPolygon' for geom in gdf.geometry)
    print(f"All MultiPolygon: {is_multipolygon}")
    
    # Check for null or empty geometries
    null_empty = gdf.geometry.isna().any() or gdf.geometry.is_empty.any()
    print(f"Any null or empty geometries: {null_empty}")
    
    # Check for invalid geometries
    invalid = not gdf.geometry.is_valid.all()
    print(f"Any invalid geometries: {invalid}")
    
    # Check for duplicate geometries
    # Use WKT to compare
    wkt_series = gdf.geometry.apply(lambda x: x.wkt)
    has_duplicate_geoms = wkt_series.duplicated().any()
    print(f"Any duplicate geometries: {has_duplicate_geoms}")
    
    # Check area
    min_area = gdf.geometry.area.min()
    print(f"Minimum area: {min_area}")
    
    # Check columns
    expected_cols = {'parcel_id', 'parcel_class', 'district', 'area_m2', 'geometry'}
    actual_cols = set(gdf.columns)
    print(f"All required columns present: {expected_cols.issubset(actual_cols)}")

if __name__ == "__main__":
    verify_output("parcels_canonical.geoparquet")

import pandas as pd

def check_file(filename):
    try:
        df = pd.read_parquet(filename)
        print(f"File: {filename}")
        print(f"Shape: {df.shape}")
        print(f"Columns: {df.columns.tolist()}")
        print("\nFirst 5 rows:")
        print(df.head())
    except Exception as e:
        print(f"Error reading {filename}: {e}")

if __name__ == "__main__":
    check_file("parcels_canonical.geoparquet")

import geopandas as gpd
import pandas as pd

def verify_output(filename):
    print(f"Verifying {filename}...")
    # Read with pandas first to ensure it's a valid parquet, then convert to GDF
    df = pd.read_parquet(filename)
    gdf = gpd.GeoDataFrame(df, geometry='geometry', crs="EPSG:22992")
    
    print(f"Shape: {gdf.shape}")
    print(f"Columns: {gdf.columns.tolist()}")
    
    # Check MultiPolygon
    is_multipolygon = all(geom.geom_type == 'MultiPolygon' for geom in gdf.geometry)
    print(f"All MultiPolygon: {is_multipolygon}")
    
    # Check for null or empty geometries
    null_empty = gdf.geometry.isna().any() or gdf.geometry.is_empty.any()
    print(f"Any null or empty geometries: {null_empty}")
    
    # Check for invalid geometries
    invalid = not gdf.geometry.is_valid.all()
    print(f"Any invalid geometries: {invalid}")
    
    # Check for duplicate geometries
    # Use WKT to compare
    wkt_series = gdf.geometry.apply(lambda x: x.wkt)
    has_duplicate_geoms = wkt_series.duplicated().any()
    print(f"Any duplicate geometries: {has_duplicate_geoms}")
    
    # Check area
    min_area = gdf.geometry.area.min()
    print(f"Minimum area: {min_area}")
    
    # Check columns
    expected_cols = {'parcel_id', 'parcel_class', 'district', 'area_m2', 'geometry'}
    actual_cols = set(gdf.columns)
    print(f"All required columns present: {expected_cols.issubset(actual_cols)}")

if __name__ == "__main__":
    verify_output("parcels_canonical.geoparquet")

import geopandas as gpd
import pandas as pd
import shapely

def clean_parcels(input_file, output_file):
    print(f"Reading {input_file}...")
    gdf = gpd.read_file(input_file)
    
    initial_count = len(gdf)
    print(f"Initial count: {initial_count}")

    # 1. Remove null or empty geometries
    print("Removing null or empty geometries...")
    gdf = gdf[gdf.geometry.notnull() & ~gdf.geometry.is_empty]
    print(f"Count after removing null/empty: {len(gdf)}")

    # 2. Fix invalid geometries
    print("Fixing invalid geometries...")
    gdf['geometry'] = gdf.geometry.make_valid()
    
    # 3. Convert all to MultiPolygon
    print("Converting all geometries to MultiPolygon...")
    def ensure_multipolygon(geom):
        if geom.is_empty:
            return geom
        if geom.geom_type == 'Polygon':
            return shapely.geometry.MultiPolygon([geom])
        return geom

    gdf['geometry'] = gdf.geometry.apply(ensure_multipolygon)

    # 4. Handle duplicates: keep record with earliest record_seq
    print("Handling duplicates (keeping earliest record_seq)...")
    gdf = gdf.sort_values(by=['parcel_id', 'record_seq'], ascending=[True, True])
    gdf = gdf.drop_duplicates(subset='parcel_id', keep='first')
    print(f"Count after deduplication: {len(gdf)}")

    # 5. Discard polygon fragments smaller than 1 m2
    print("Removing fragments smaller than 1 m2...")
    gdf = gdf[gdf.geometry.area >= 1.0]
    print(f"Count after area filter: {len(gdf)}")

    # 6. Remove duplicate geometries (different IDs but same shape)
    print("Removing duplicate geometries...")
    gdf['geom_wkt'] = gdf.geometry.apply(lambda x: x.wkt)
    gdf = gdf.drop_duplicates(subset='geom_wkt')
    gdf = gdf.drop(columns=['geom_wkt'])
    print(f"Count after removing duplicate geometries: {len(gdf)}")

    # 7. Final column selection
    print("Selecting final columns...")
    final_cols = ['parcel_id', 'parcel_class', 'district', 'area_m2', 'geometry']
    gdf = gdf[final_cols]

    # Ensure no nulls in required columns
    gdf = gdf.dropna(subset=['parcel_id', 'parcel_class', 'district', 'area_m2'])

    # 8. Write to GeoParquet
    # IMPORTANT: Use engine='pyogrio' or ensure geometry is handled correctly
    print(f"Writing to {output_file}...")
    gdf.to_parquet(output_file)
    print("Done!")

if __name__ == "__main__":
    clean_parcels("cairo_parcels_legacy.geojson", "parcels_canonical.geoparquet")

import geopandas as gpd
import pandas as pd
from shapely import wkb

def verify_output(filename):
    print(f"Verifying {filename}...")
    # Read with pandas first to ensure it's a valid parquet, then convert to GDF
    df = pd.read_parquet(filename)
    
    # The geometry column in parquet for GeoParquet is often stored as WKB
    # We need to convert it back to shapely objects if it's not automatically recognized
    if df['geometry'].dtype == 'object':
        # Check if it's bytes (WKB)
        if isinstance(df['geometry'].iloc[0], bytes):
            print("Detected WKB bytes, converting...")
            df['geometry'] = df['geometry'].apply(lambda x: wkb.loads(x))
    
    gdf = gpd.GeoDataFrame(df, geometry='geometry', crs="EPSG:22992")
    
    print(f"Shape: {gdf.shape}")
    print(f"Columns: {gdf.columns.tolist()}")
    
    # Check MultiPolygon
    is_multipolygon = all(geom.geom_type == 'MultiPolygon' for geom in gdf.geometry)
    print(f"All MultiPolygon: {is_multipolygon}")
    
    # Check for null or empty geometries
    null_empty = gdf.geometry.isna().any() or gdf.geometry.is_empty.any()
    print(f"Any null or empty geometries: {null_empty}")
    
    # Check for invalid geometries
    invalid = not gdf.geometry.is_valid.all()
    print(f"Any invalid geometries: {invalid}")
    
    # Check for duplicate geometries
    wkt_series = gdf.geometry.apply(lambda x: x.wkt)
    has_duplicate_geoms = wkt_series.duplicated().any()
    print(f"Any duplicate geometries: {has_duplicate_geoms}")
    
    # Check area
    min_area = gdf.geometry.area.min()
    print(f"Minimum area: {min_area}")
    
    # Check columns
    expected_cols = {'parcel_id', 'parcel_class', 'district', 'area_m2', 'geometry'}
    actual_cols = set(gdf.columns)
    print(f"All required columns present: {expected_cols.issubset(actual_cols)}")

if __name__ == "__main__":
    verify_output("parcels_canonical.geoparquet")