fio-l1-paris-kml-pois

Rerun

import geopandas as gpd
import pandas as pd
import re
from bs4 import BeautifulSoup
import json

def parse_description(desc_html):
    """
    Parses the HTML description to extract category and verified_date.
    Example HTML:
    <b>% Arabica</b><br/>Cat&eacute;gorie&nbsp;: Cafés ouverts tard<br/><a href="...">Voir la fiche</a><br/>Derni&egrave;re v&eacute;rification&nbsp;: 2026-01-01
    """
    if not desc_html:
        return None, None
    
    soup = BeautifulSoup(desc_html, 'html.parser')
    text = soup.get_text(separator='|')
    # The separator '|' might be helpful if there are complex structures, 
    # but get_text() usually handles entities well.
    # Let's use a more robust way to find the patterns.
    
    category = None
    verified_date = None
    
    # Look for category
    # Pattern: Catégorie : <value>
    # In the text, 'Cat&eacute;gorie' becomes 'Catégorie'
    cat_match = re.search(r"Catégorie\s*:\s*([^|<]+)", text)
    if cat_match:
        category = cat_match.group(1).strip()
        
    # Look for verified date
    # Pattern: Dernière vérification : <YYYY-MM-DD>
    date_match = re.search(r"Dernière vérification\s*:\s*(\d{4}-\d{2}-\d{2})", text)
    if date_match:
        verified_date = date_match.group(1).strip()
        
    return category, verified_date

def main():
    input_file = 'paris_late_night_pois.kml'
    output_file = 'paris_pois.geojson'
    
    print(f"Reading {input_file}...")
    gdf = gpd.read_file(input_file)
    
    print(f"Processing {len(gdf)} features...")
    
    # We need to extract info from 'description' column
    # Note: KML 'description' often contains HTML/CDATA
    
    categories = []
    verified_dates = []
    
    for desc in gdf['description']:
        cat, date = parse_description(desc)
        categories.append(cat)
        verified_dates.append(date)
        
    gdf['category'] = categories
    gdf['verified_date'] = pd.to_datetime(verified_dates, errors='coerce')
    
    # Keep only name, category, verified_date, and geometry
    # The user said "Keep name, include ... category ... and ... verified_date"
    # Usually, keeping the geometry is implied for a GeoJSON.
    # Let's see what columns we have to ensure we don't lose anything vital or keep too much.
    # The prompt asks for "one row per placemark".
    
    # According to the prompt: "Keep name, include the source's classification/grouping as a category column, and pull ... verified_date"
    # We'll select these specifically.
    
    final_gdf = gdf[['name', 'category', 'verified_date', 'geometry']].copy()
    
    # Ensure geometry is valid (KML might have issues)
    final_gdf = final_gdf[final_gdf.geometry.notnull()]

    print(f"Writing to {output_file}...")
    final_gdf.to_file(output_file, driver='GeoJSON')
    print("Done!")

if __name__ == "__main__":
    main()

import geopandas as gpd
import pandas as pd
import re
from bs4 import BeautifulSoup

def parse_description(desc_html):
    """
    Parses the HTML description to extract category and verified_date.
    """
    if not desc_html:
        return None, None
    
    soup = BeautifulSoup(desc_html, 'html.parser')
    text = soup.get_text(separator='|')
    
    category = None
    verified_date = None
    
    # Look for category
    # Pattern: Catégorie : <value>
    cat_match = re.search(r"Catégorie\s*:\s*([^|<]+)", text)
    if cat_match:
        category = cat_match.group(1).strip()
        
    # Look for verified date
    # Pattern: Dernière vérification : <YYYY-MM-DD>
    date_match = re.search(r"Dernière vérification\s*:\s*(\d{4}-\d{2}-\d{2})", text)
    if date_match:
        verified_date = date_match.group(1).strip()
        
    return category, verified_date

def main():
    input_file = 'paris_late_night_pois.kml'
    output_file = 'paris_pois.geojson'
    
    print(f"Reading {input_file}...")
    # The KML has multiple layers. Let's see what they are.
    # We want to process all of them if they contain the POIs.
    # However, the error message said "Cafés ouverts tard" (default) was the first one.
    # Since the goal is to convert the file (which contains multiple groups/layers), 
    # we should probably combine them or decide which one to take.
    # Given the prompt "one row per placemark", and the KML has different layers,
    # we will iterate through layers and combine them.
    
    import fiona
    layers = fiona.listlayers(input_file)
    print(f"Layers found: {layers}")
    
    all_gdfs = []
    for layer in layers:
        print(f"Reading layer: {layer}")
        gdf = gpd.read_file(input_file, layer=layer)
        all_gdfs.append(gdf)
    
    gdf = pd.concat(all_gdfs, ignore_index=True)
    gdf = gpd.GeoDataFrame(gdf, crs=all_gdfs[0].crs)
    
    # Inspect columns
    print(f"Columns available: {gdf.columns.tolist()}")
    
    # The error said 'name' not in index. Let's check case sensitivity or naming.
    # KML 'name' might be 'Name' or something else in the GeoDataFrame.
    # Let's standardize the 'name' column if it's not present.
    
    name_col = None
    for col in ['name', 'Name', 'label', 'Label']:
        if col in gdf.columns:
            name_col = col
            break
            
    if not name_col:
        print("Warning: Could not find a name column. Using index as name.")
        gdf['name'] = gdf.index.astype(str)
    else:
        gdf = gdf.rename(columns={name_col: 'name'})

    print(f"Processing {len(gdf)} features...")
    
    categories = []
    verified_dates = []
    
    # The description column might also be case sensitive
    desc_col = None
    for col in ['description', 'Description', 'desc']:
        if col in gdf.columns:
            desc_col = col
            break
            
    if not desc_col:
        print("Error: Description column not found.")
        return

    for desc in gdf[desc_col]:
        cat, date = parse_description(desc)
        categories.append(cat)
        verified_dates.append(date)
        
    gdf['category'] = categories
    gdf['verified_date'] = pd.to_datetime(verified_dates, errors='coerce')
    
    # Select only required columns
    # We use 'name' as we renamed it above.
    final_gdf = gdf[['name', 'category', 'verified_date', 'geometry']].copy()
    
    # Ensure geometry is valid
    final_gdf = final_gdf[final_gdf.geometry.notnull()]

    print(f"Writing to {output_file}...")
    final_gdf.to_file(output_file, driver='GeoJSON')
    print("Done!")

if __name__ == "__main__":
    main()