dc-l2-cairo-invalid-dedup

Rerun

import geopandas as gpd

def inspect_data(file_path):
    gdf = gpd.read_file(file_path)
    print(f"Initial shape: {gdf.shape}")
    print("\nColumns:")
    print(gdf.columns.tolist())
    print("\nFirst 5 rows:")
    print(gdf.head())
    print("\nGeometry type counts:")
    print(gdf.geometry.type.value_counts())
    print("\nNull counts:")
    print(gdf.isnull().sum())
    
    # Check for invalid geometries
    invalid_count = gdf.geometry.is_valid.value_counts()
    print("\nValidity check:")
    print(invalid_count)

if __name__ == "__main__":
    inspect_data("/work/cairo_parcels_legacy.geojson")

import geopandas as gpd

def inspect_data(file_path):
    gdf = gpd.read_file(file_path)
    print(f"Initial shape: {gdf.shape}")
    print("\nColumns:")
    print(gdf.columns.tolist())
    print("\nFirst 5 rows:")
    print(gdf.head())
    print("\nGeometry type counts:")
    print(gdf.geometry.type.value_counts())
    print("\nNull counts:")
    print(gdf.isnull().sum())
    
    # Check for invalid geometries
    invalid_count = gdf.geometry.is_valid.value_counts()
    print("\nValidity check:")
    print(invalid_count)

if __name__ == "__main__":
    inspect_data("/work/cairo_parcels_legacy.geojson")

import geopandas as gpd
from shapely.geometry import MultiPolygon, Polygon

def solve():
    # Load data
    gdf = gpd.read_file("/work/cairo_parcels_legacy.geojson")
    
    # 1. Handle invalid geometries
    # We fix them by using buffer(0)
    gdf['geometry'] = gdf.geometry.buffer(0)
    
    # 2. Discard any polygon fragments smaller than 1 m2
    # The user mentioned "discard any polygon fragments smaller than 1 m2". 
    # Usually, this means the geometry itself. We'll filter by actual calculated area.
    gdf['actual_area'] = gdf.geometry.area
    gdf = gdf[gdf['actual_area'] >= 1.0].copy()
    
    # 3. One canonical record per parcel. 
    # When duplicates exist, keep the record with the earliest record_seq.
    # First, sort by parcel_id and record_seq.
    gdf = gdf.sort_values(by=['parcel_id', 'record_seq']).drop_duplicates(subset=['parcel_id'], keep='first')
    
    # 4. Every feature must be a MultiPolygon.
    def ensure_multipolygon(geom):
        if geom.is_empty:
            return None
        if isinstance(geom, MultiPolygon):
            return geom
        elif isinstance(geom, Polygon):
            return MultiPolygon([geom])
        else:
            # For other geometry types that might have been created by buffer(0)
            # e.g. if it becomes a collection or something else. 
            # But mostly we want to handle Single Polygons.
            try:
                return MultiPolygon([geom])
            except:
                return None

    gdf['geometry'] = gdf['geometry'].apply(ensure_multipolygon)
    
    # 5. Remove null or empty geometries
    gdf = gdf[gdf.geometry.notnull()]
    gdf = gdf[~gdf.geometry.is_empty]
    
    # 6. Remove duplicate geometries
    # "no duplicate geometries" - this usually means if two different parcel_ids 
    # have the exact same footprint, we should check if that's intended.
    # However, the prompt says "no duplicate geometries", implying we shouldn't have
    # two rows with the same shape. 
    # Usually in a registry, different IDs can have the same shape, but the prompt
    # specifically asks for no duplicate geometries.
    # Let's interpret this as: if multiple parcels have the same geometry, 
    # we should keep only one.
    gdf = gdf.drop_duplicates(subset=['geometry'])

    # 7. No invalid geometries (check again after cleaning)
    gdf = gdf[gdf.geometry.is_valid]
    
    # 8. "no artifact slivers"
    # This is a bit subjective, but the 1m2 rule covers small fragments.
    # Let's ensure no extremely thin geometries if possible, but 1m2 is the hard rule.

    # 9. Include specific columns: parcel_id, parcel_class, district, area_m2
    # and ensure area_m2 is the area of the geometry we kept.
    # Wait, the prompt says "including these columns: parcel_id, parcel_class, district, and area_m2".
    # It doesn't say "replace area_m2 with calculated area", but it's safer to 
    # ensure the area_m2 column is consistent with the geometry. 
    # Given the context of cleaning, I'll use the actual area.
    
    result = gdf[['parcel_id', 'parcel_class', 'district', 'area_m2', 'geometry']].copy()
    # Update area_m2 to be the actual area of the cleaned geometry to be safe?
    # The prompt says "including these columns", it doesn't explicitly say "calculate area_m2".
    # However, if we discarded fragments, the old area_m2 might be wrong.
    # Let's stick to the columns provided but recalculate area_m2 to be precise.
    result['area_m2'] = result.geometry.area

    # Final check for nulls in required columns
    result = result.dropna(subset=['parcel_id', 'parcel_class', 'district'])

    # Final output
    result.to_file("/work/parcels_canonical.geoparquet", driver="Parquet")
    print("Success! Saved to /work/parcels_canonical.geoparquet")
    print(f"Final count: {len(result)}")

if __name__ == "__main__":
    solve()

import geopandas as gpd
from shapely.geometry import MultiPolygon, Polygon

def solve():
    # Load data
    gdf = gpd.read_file("/work/cairo_parcels_legacy.geojson")
    
    # 1. Handle invalid geometries
    # We fix them by using buffer(0)
    gdf['geometry'] = gdf.geometry.buffer(0)
    
    # 2. Discard any polygon fragments smaller than 1 m2
    # The user mentioned "discard any polygon fragments smaller than 1 m2". 
    # Usually, this means the geometry itself. We'll filter by actual calculated area.
    gdf['actual_area'] = gdf.geometry.area
    gdf = gdf[gdf['actual_area'] >= 1.0].copy()
    
    # 3. One canonical record per parcel. 
    # When duplicates exist, keep the record with the earliest record_seq.
    # First, sort by parcel_id and record_seq.
    gdf = gdf.sort_values(by=['parcel_id', 'record_seq']).drop_duplicates(subset=['parcel_id'], keep='first')
    
    # 4. Every feature must be a MultiPolygon.
    def ensure_multipolygon(geom):
        if geom.is_empty:
            return None
        if isinstance(geom, MultiPolygon):
            return geom
        elif isinstance(geom, Polygon):
            return MultiPolygon([geom])
        else:
            # For other geometry types that might have been created by buffer(0)
            # e.g. if it becomes a collection or something else. 
            # But mostly we want to handle Single Polygons.
            try:
                return MultiPolygon([geom])
            except:
                return None

    gdf['geometry'] = gdf['geometry'].apply(ensure_multipolygon)
    
    # 5. Remove null or empty geometries
    gdf = gdf[gdf.geometry.notnull()]
    gdf = gdf[~gdf.geometry.is_empty]
    
    # 6. Remove duplicate geometries
    # "no duplicate geometries" - this usually means if two different parcel_ids 
    # have the exact same footprint, we should check if that's intended.
    # However, the prompt says "no duplicate geometries", implying we shouldn't have
    # two rows with the same shape. 
    # Usually in a registry, different IDs can have the same shape, but the prompt
    # specifically asks for no duplicate geometries.
    # Let's interpret this as: if multiple parcels have the same geometry, 
    # we should keep only one.
    gdf = gdf.drop_duplicates(subset=['geometry'])

    # 7. Remove invalid geometries (check again after cleaning)
    gdf = gdf[gdf.geometry.is_valid]
    
    # 8. "no artifact slivers"
    # This is a bit subjective, but the 1m2 rule covers small fragments.
    # Let's ensure no extremely thin geometries if possible, but 1m2 is the hard rule.

    # 9. Include specific columns: parcel_id, parcel_class, district, area_m2
    # and ensure area_m2 is the area of the geometry we kept.
    # Wait, the prompt says "including these columns: parcel_id, parcel_class, district, and area_m2".
    # It doesn't say "replace area_m2 with calculated area", but it's safer to 
    # ensure the area_m2 column is consistent with the geometry. 
    # Given the context of cleaning, I'll use the actual area.
    
    result = gdf[['parcel_id', 'parcel_class', 'district', 'area_m2', 'geometry']].copy()
    # Update area_m2 to be the actual area of the cleaned geometry to be safe.
    # The user's input file already has an area_m2 column. 
    # But after buffer(0) and dropping, it's better to recalculate it.
    # HOWEVER, the prompt asks to include "area_m2", implying it's one of the columns to keep.
    # I will use the area calculated from the geometry to ensure it's accurate.
    result['area_m2'] = result.geometry.area

    # Final check for nulls in required columns
    result = result.dropna(subset=['parcel_id', 'parcel_class', 'district'])

    # Final output
    # Since pyogrio failed with driver="Parquet", let's try standard geopandas to_parquet (which uses pyarrow/fastparquet)
    result.to_parquet("/work/parcels_canonical.geoparquet")
    print("Success! Saved to /work/parcels_canonical.geoparquet")
    print(f"Final count: {len(result)}")

if __name__ == "__main__":
    solve()

import geopandas as gpd

def verify():
    gdf = gpd.read_file("/work/parcels_canonical.geoparquet")
    print(f"Final Shape: {gdf.shape}")
    print("\nColumns:")
    print(gdf.columns.tolist())
    print("\nGeometry Types:")
    print(gdf.geometry.type.value_counts())
    print("\nIs Valid Check:")
    print(gdf.geometry.is_valid.value_counts())
    print("\nIs Empty Check:")
    print(gdf.geometry.is_empty.value_counts())
    print("\nMin Area:")
    print(gdf.geometry.area.min())
    print("\nNulls:")
    print(gdf.isnull().sum())
    
    # Check if all are MultiPolygons
    all_multipolygons = all(isinstance(g, gpd.MultiPolygon) or (hasattr(g, 'geom_type') and g.geom_type == 'MultiPolygon') for g in gdf.geometry)
    # Wait, in geopandas, the geom_type of a MultiPolygon is 'MultiPolygon'
    # Let's just check the value_counts of geom_type
    print("\nGeometry type counts in file:")
    print(gdf.geom_type.value_counts())

if __name__ == "__main__":
    verify()

import pandas as pd
import geopandas as gpd

def verify():
    # Since it's a geoparquet, let's try reading it with geopandas but use the right method if needed.
    # geoparquet is usually read with gpd.read_parquet
    gdf = gpd.read_parquet("/work/parcels_canonical.geoparquet")
    print(f"Final Shape: {gdf.shape}")
    print("\nColumns:")
    print(gdf.columns.tolist())
    print("\nGeometry type counts:")
    print(gdf.geom_type.value_counts())
    print("\nIs Valid Check:")
    print(gdf.geometry.is_valid.value_counts())
    print("\nIs Empty Check:")
    print(gdf.geometry.is_empty.value_counts())
    print("\nMin Area:")
    print(gdf.geometry.area.min())
    print("\nNulls:")
    print(gdf.isnull().sum())

if __name__ == "__main__":
    verify()