fio-l1-paris-kml-pois

Rerun

import geopandas as gpd

try:
    gdf = gpd.read_file('paris_late_night_pois.kml')
    print("KML loaded successfully.")
    print(gdf.head())
    print(gdf.columns)
except Exception as e:
    print(f"Error loading KML: {e}")

import geopandas as gpd
import pandas as pd
import re

def extract_date(text):
    if not text or not isinstance(text, str):
        return None
    # Look for patterns like "last verified: 2023-01-01" or "vérifié le 2023-01-01" 
    # or even just a date in ISO format.
    # Given the instruction says "pull each entry's 'last verified' date out of the HTML blurb"
    # I'll look for a date pattern.
    
    # Regex for YYYY-MM-DD or DD/MM/YYYY or similar.
    # Let's try a broad search for something that looks like a date.
    # Usually, HTML might have it inside a tag.
    
    # Example: <p>Last verified: 2022-05-12</p>
    match = re.search(r'(\d{4}-\d{2}-\d{2})', text)
    if match:
        return match.group(1)
    
    # Try DD/MM/YYYY
    match = re.search(r'(\d{2}/\d{2}/\d{4})', text)
    if match:
        try:
            return pd.to_datetime(match.group(1), dayfirst=True).strftime('%Y-%m-%d')
        except:
            return None
            
    return None

def process_kml(file_path):
    # KML can have multiple layers. The warning suggests 3 layers.
    # 'Cafés ouverts tard', 'Bibliothèques de nuit', 'Tours et infos touristiques'.
    # Let's use fiona to list layers first if we want to be sure, 
    # but we can also just read them all.
    
    import fiona
    layers = fiona.listlayers(file_path)
    print(f"Layers found: {layers}")
    
    all_gdf = []
    
    for layer in layers:
        gdf = gpd.read_file(file_path, layer=layer)
        # Add the category column from the layer name
        gdf['category'] = layer
        all_gdf.append(gdf)
        
    combined_gdf = pd.concat(all_gdf, ignore_index=True)
    
    # Rename 'Name' to 'name' if it exists (it's 'Name' in KML)
    if 'Name' in combined_gdf.columns:
        combined_gdf = combined_gdf.rename(columns={'Name': 'name'})
    
    # Handle 'description' which contains the HTML blurb
    # Extract verified_date
    combined_gdf['verified_date'] = combined_gdf['description'].apply(extract_date)
    
    # Convert verified_date to datetime objects for better handling, 
    # but the requirement says "as an ISO date". 
    # GeoJSON often likes strings for dates or just handles them.
    # Let's ensure they are strings in YYYY-MM-DD format.
    combined_gdf['verified_date'] = pd.to_datetime(combined_gdf['verified_date'], errors='coerce').dt.strftime('%Y-%m-%d')

    # Keep only requested columns: name, category, verified_date, geometry
    # Note: We must keep geometry for it to be a valid GeoDataFrame/GeoJSON
    cols_to_keep = ['name', 'category', 'verified_date', 'geometry']
    # Check which ones actually exist
    existing_cols = [c for c in cols_to_keep if c in combined_gdf.columns]
    final_gdf = combined_gdf[existing_cols]
    
    return final_gdf

if __name__ == "__main__":
    input_file = 'paris_late_night_pois.kml'
    output_file = 'paris_pois.geojson'
    
    try:
        final_gdf = process_kml(input_file)
        print("Processing complete.")
        print(final_gdf.head())
        final_gdf.to_file(output_file, driver='GeoJSON')
        print(f"Saved to {output_file}")
    except Exception as e:
        print(f"Error: {e}")

import geopandas as gpd
import pandas as pd
import re
import fiona

def extract_date(text):
    if not text or not isinstance(text, str):
        return None
    
    # The user mentioned "last verified" date is in the HTML blurb.
    # Let's be more robust. Let's look for "last verified" or "vérifié le"
    # and then extract the date.
    
    # Try to find date in YYYY-MM-DD format first
    iso_match = re.search(r'(\d{4}-\d{2}-\d{2})', text)
    if iso_match:
        return iso_match.group(1)
    
    # Try to find date in DD/MM/YYYY format
    dm_match = re.search(r'(\d{2}/\d{2}/\d{4})', text)
    if dm_match:
        try:
            return pd.to_datetime(dm_match.group(1), dayfirst=True).strftime('%Y-%m-%d')
        except:
            pass
            
    return None

def process_kml(file_path):
    layers = fiona.listlayers(file_path)
    print(f"Layers found: {layers}")
    
    all_gdf = []
    
    for layer in layers:
        gdf = gpd.read_file(file_path, layer=layer)
        # Add the category column from the layer name
        gdf['category'] = layer
        all_gdf.append(gdf)
        
    combined_gdf = pd.concat(all_gdf, ignore_index=True)
    
    # Rename 'Name' to 'name' if it exists
    if 'Name' in combined_gdf.columns:
        combined_gdf = combined_gdf.rename(columns={'Name': 'name'})
    
    # If 'name' is not present but 'Name' was, handle it.
    # (Done above)

    # The description might be in a column called 'description'
    if 'description' in combined_gdf.columns:
        combined_gdf['verified_date'] = combined_gdf['description'].apply(extract_date)
    else:
        combined_gdf['verified_date'] = None

    # Convert verified_date to string ISO format
    combined_gdf['verified_date'] = pd.to_datetime(combined_gdf['verified_date'], errors='coerce').dt.strftime('%Y-%m-%d')

    # Keep only requested columns: name, category, verified_date, geometry
    cols_to_keep = ['name', 'category', 'verified_date', 'geometry']
    existing_cols = [c for c in cols_to_keep if c in combined_gdf.columns]
    final_gdf = combined_gdf[existing_cols]
    
    return final_gdf

if __name__ == "__main__":
    input_file = 'paris_late_night_pois.kml'
    output_file = 'paris_pois.geojson'
    
    try:
        final_gdf = process_kml(input_file)
        print("Processing complete.")
        print(final_gdf.head())
        # Ensure geometry is 2D for GeoJSON compatibility if it's POINT Z
        final_gdf.geometry = final_gdf.geometry.map(lambda geom: geom if geom.has_z is False else 
                                                    from_wkt(f"POINT ({geom.x} {geom.y})"))
        # Wait, the above is complex. Let's just use standard way to drop Z.
        import shapely.geometry
        from shapely.wkt import loads as load_wkt
        
        def strip_z(geom):
            if geom.has_z:
                return shapely.geometry.Point(geom.x, geom.y)
            return geom
            
        final_gdf.geometry = final_gdf.geometry.map(strip_z)
        
        final_gdf.to_file(output_file, driver='GeoJSON')
        print(f"Saved to {output_file}")
    except Exception as e:
        import traceback
        traceback.print_exc()
        print(f"Error: {e}")

import geopandas as gpd
import pandas as pd
import re
import fiona
import shapely.geometry

def extract_date(text):
    if not text or not isinstance(text, str):
        return None
    
    # Look for "last verified" or similar in the HTML
    # Let's try to find a date in YYYY-MM-DD or DD/MM/YYYY
    
    # First attempt: YYYY-MM-DD
    iso_match = re.search(r'(\d{4}-\d{2}-\d{2})', text)
    if iso_match:
        return iso_match.group(1)
    
    # Second attempt: DD/MM/YYYY
    dm_match = re.search(r'(\d{2}/\d{2}/\d{4})', text)
    if dm_match:
        try:
            return pd.to_datetime(dm_match.group(1), dayfirst=True).strftime('%Y-%m-%d')
        except:
            pass
            
    return None

def strip_z(geom):
    if geom.has_z:
        return shapely.geometry.Point(geom.x, geom.y)
    return geom

def process_kml(file_path):
    layers = fiona.listlayers(file_path)
    print(f"Layers found: {layers}")
    
    all_gdf = []
    
    for layer in layers:
        gdf = gpd.read_file(file_path, layer=layer)
        # Add the category column from the layer name
        gdf['category'] = layer
        all_gdf.append(gdf)
        
    combined_gdf = pd.concat(all_gdf, ignore_index=True)
    
    # Rename 'Name' to 'name' if it exists
    if 'Name' in combined_gdf.columns:
        combined_gdf = combined_gdf.rename(columns={'Name': 'name'})
    
    # Extract verified_date from 'description'
    if 'description' in combined_gdf.columns:
        combined_gdf['verified_date'] = combined_gdf['description'].apply(extract_date)
    else:
        combined_gdf['verified_date'] = None

    # Ensure verified_date is ISO date string (YYYY-MM-DD)
    combined_gdf['verified_date'] = pd.to_datetime(combined_gdf['verified_date'], errors='coerce').dt.strftime('%Y-%m-%d')

    # Keep only requested columns: name, category, verified_date, geometry
    cols_to_keep = ['name', 'category', 'verified_date', 'geometry']
    existing_cols = [c for c in cols_to_keep if c in combined_gdf.columns]
    final_gdf = combined_gdf[existing_cols].copy()
    
    # Strip Z coordinates to ensure flat GeoJSON
    final_gdf.geometry = final_gdf.geometry.map(strip_z)
    
    return final_gdf

if __name__ == "__main__":
    input_file = 'paris_late_night_pois.kml'
    output_file = 'paris_pois.geojson'
    
    try:
        final_gdf = process_kml(input_file)
        print("Processing complete.")
        print(final_gdf.head())
        final_gdf.to_file(output_file, driver='GeoJSON')
        print(f"Saved to {output_file}")
    except Exception as e:
        import traceback
        traceback.print_exc()
        print(f"Error: {e}")

import geopandas as gpd
import pandas as pd
import re
import fiona
import shapely.geometry

def extract_date(text):
    if not text or not isinstance(text, str):
        return None
    
    # The user mentioned "last verified" date is in the HTML blurb.
    # Let's look for "last verified" or "vérifié le" and then extract the date.
    # We will look for dates in YYYY-MM-DD or DD/MM/YYYY format.
    
    # First attempt: YYYY-MM-DD
    iso_match = re.search(r'(\d{4}-\d{2}-\d{2})', text)
    if iso_match:
        return iso_match.group(1)
    
    # Second attempt: DD/MM/YYYY
    dm_match = re.search(r'(\d{2}/\d{2}/\d{4})', text)
    if dm_match:
        try:
            return pd.to_datetime(dm_match.group(1), dayfirst=True).strftime('%Y-%m-%d')
        except:
            pass
            
    return None

def strip_z(geom):
    if geom.has_z:
        return shapely.geometry.Point(geom.x, geom.y)
    return geom

def process_kml(file_path):
    layers = fiona.listlayers(file_path)
    print(f"Layers found: {layers}")
    
    all_gdf = []
    
    for layer in layers:
        gdf = gpd.read_file(file_path, layer=layer)
        # Add the category column from the layer name
        gdf['category'] = layer
        all_gdf.append(gdf)
        
    combined_gdf = pd.concat(all_gdf, ignore_index=True)
    
    # Rename 'Name' to 'name' if it exists
    if 'Name' in combined_gdf.columns:
        combined_gdf = combined_gdf.rename(columns={'Name': 'name'})
    
    # Extract verified_date from 'description'
    if 'description' in combined_gdf.columns:
        combined_gdf['verified_date'] = combined_gdf['description'].apply(extract_date)
    else:
        combined_gdf['verified_date'] = None

    # Ensure verified_date is ISO date string (YYYY-MM-DD)
    combined_gdf['verified_date'] = pd.to_datetime(combined_gdf['verified_date'], errors='coerce').dt.strftime('%Y-%m-%d')

    # Keep only requested columns: name, category, verified_date, geometry
    cols_to_keep = ['name', 'category', 'verified_date', 'geometry']
    existing_cols = [c for c in cols_to_keep if c in combined_gdf.columns]
    final_gdf = combined_gdf[existing_cols].copy()
    
    # Strip Z coordinates to ensure flat GeoJSON
    final_gdf.geometry = final_gdf.geometry.map(strip_z)
    
    return final_gdf

if __name__ == "__main__":
    input_file = 'paris_late_night_pois.kml'
    output_file = 'paris_pois.geojson'
    
    try:
        final_gdf = process_kml(input_file)
        print("Processing complete.")
        print(final_gdf.head())
        final_gdf.to_file(output_file, driver='GeoJSON')
        print(f"Saved to {output_file}")
    except Exception as e:
        import traceback
        traceback.print_exc()
        print(f"Error: {e}")