fio-l1-paris-kml-pois

Rerun

import geopandas as gpd
import pandas as pd
import re
from bs4 import BeautifulSoup
import json

def parse_description(desc_html):
    if not desc_html:
        return None, None
    
    # Use BeautifulSoup to handle the HTML content
    soup = BeautifulSoup(desc_html, 'html.parser')
    text = soup.get_text(separator=' ')
    
    # The description looks like:
    # <b% Arabica</b><br/>Cat&eacute;gorie&nbsp;: Cafés ouverts tard<br/><a href="...">Voir la fiche</a><br/>Derni&egrave;re v&eacute;rification&nbsp;: 2026-01-01
    
    # Extract category: "Catégorie : ..."
    # Note: The HTML might have entities like &eacute; which BeautifulSoup handles.
    category = None
    cat_match = re.search(r'Catégorie\s*:\s*([^<]+)', text)
    if cat_match:
        category = cat_match.group(1).strip()
        
    # Extract verified_date: "Dernière vérification : YYYY-MM-DD"
    verified_date = None
    date_match = re.search(r'Dernière vérification\s*:\s*(\d{4}-\d{2}-\d{2})', text)
    if date_match:
        verified_date = date_match.group(1).strip()
        
    return category, verified_date

def main():
    input_file = 'paris_late_night_pois.kml'
    output_file = 'paris_pois.geojson'
    
    print(f"Reading {input_file}...")
    gdf = gpd.read_file(input_file)
    
    print("Processing records...")
    # We need to iterate through the rows and extract info from the 'description' column
    # In KML, the description is often stored in a 'description' column in geopandas
    
    categories = []
    verified_dates = []
    
    for desc in gdf['description']:
        cat, v_date = parse_description(desc)
        categories.append(cat)
        verified_dates.append(v_date)
        
    gdf['category'] = categories
    gdf['verified_date'] = pd.to_datetime(verified_dates, errors='coerce')
    
    # Keep only necessary columns: name, category, verified_date, and geometry
    # The prompt says "Keep name, include the source's classification/grouping as a category column, 
    # and pull each entry's 'last verified' date... into a verified_date column"
    # It doesn't explicitly say to drop others, but "one row per placemark" and the specific 
    # instructions suggest a clean output. Let's keep name, category, verified_date, and geometry.
    
    final_gdf = gdf[['name', 'category', 'verified_date', 'geometry']].copy()
    
    # Convert verified_date to ISO string for GeoJSON compatibility if needed, 
    # though geopandas handles datetime objects well.
    # To ensure it's a string in the JSON output:
    final_gdf['verified_date'] = final_gdf['verified_date'].dt.strftime('%Y-%m-%d')

    print(f"Saving to {output_file}...")
    final_gdf.to_file(output_file, driver='GeoJSON')
    print("Done!")

if __name__ == "__main__":
    main()

import geopandas as gpd
import pandas as pd
import re
import json

def parse_description(desc_html):
    if not desc_html:
        return None, None
    
    # Since bs4 is not available, we'll use regex and basic string manipulation.
    # The description format is:
    # <b% Arabica</b><br/>Cat&eacute;gorie&nbsp;: Cafés ouverts tard<br/><a href="...">Voir la fiche</a><br/>Derni&egrave;re v&eacute;rification&nbsp;: 2026-01-01
    
    # 1. Clean up common HTML entities manually
    text = desc_html.replace('&nbsp;', ' ').replace('&eacute;', 'e').replace('&egrave;', 'e')
    
    # 2. Extract category
    # Look for Catégorie : ... until the next < or <br/>
    category = None
    cat_match = re.search(r'Cat[éè]gorie\s*:\s*([^<]+)', text, re.IGNORECASE)
    if cat_match:
        category = cat_match.group(1).strip()
        
    # 3. Extract verified_date
    verified_date = None
    date_match = re.search(r'Derni[èe]re v[éè]rification\s*:\s*(\d{4}-\d{2}-\d{2})', text, re.IGNORECASE)
    if date_match:
        verified_date = date_match.group(1).strip()
        
    return category, verified_date

def main():
    input_file = 'paris_late_night_pois.kml'
    output_file = 'paris_pois.geojson'
    
    print(f"Reading {input_file}...")
    gdf = gpd.read_file(input_file)
    
    print("Processing records...")
    
    categories = []
    verified_dates = []
    
    # Ensure 'description' column exists and handle potential missing values
    if 'description' not in gdf.columns:
        # If description is missing, we can't do much. 
        # Let's check the columns of gdf first.
        print(f"Error: 'description' column not found in {input_file}. Columns: {gdf.columns.tolist()}")
        return

    for desc in gdf['description']:
        cat, v_date = parse_description(desc)
        categories.append(cat)
        verified_dates.append(v_date)
        
    gdf['category'] = categories
    # Use pd.to_datetime to convert. 
    # We'll convert to string later for the GeoJSON.
    gdf['verified_date'] = pd.to_datetime(verified_dates, errors='coerce')
    
    # Keep only requested columns + geometry
    final_gdf = gdf[['name', 'category', 'verified_date', 'geometry']].copy()
    
    # Convert datetime objects to ISO strings for GeoJSON
    # This ensures the column is a string in the final file.
    final_gdf['verified_date'] = final_gdf['verified_date'].dt.strftime('%Y-%m-%d')

    print(f"Saving to {output_file}...")
    final_gdf.to_file(output_file, driver='GeoJSON')
    print("Done!")

if __name__ == "__main__":
    main()

import geopandas as gpd
import pandas as pd
import re

def parse_description(desc_html):
    if not desc_html:
        return None, None
    
    # Clean up common HTML entities manually
    text = desc_html.replace(' ', ' ').replace('é', 'e').replace('è', 'e')
    
    # 2. Extract category
    # Look for Catégorie : ... until the next < or <br/>
    category = None
    cat_match = re.search(r'Cat[éè]gorie\s*:\s*([^<]+)', text, re.IGNORECASE)
    if cat_match:
        category = cat_match.group(1).strip()
        
    # 3. Extract verified_date
    verified_date = None
    date_match = re.search(r'Derni[èe]re v[éè]rification\s*:\s*(\d{4}-\d{2}-\d{2})', text, re.IGNORECASE)
    if date_match:
        verified_date = date_match.group(1).strip()
        
    return category, verified_date

def main():
    input_file = 'paris_late_night_pois.kml'
    output_file = 'paris_pois.geojson'
    
    print(f"Reading {input_file}...")
    # The KML has multiple layers. We want all of them.
    # However, the instructions say "convert it", which usually implies the whole content.
    # Since the user mentioned "one row per placemark", let's read all layers and concatenate them.
    
    import fiona
    layers = fiona.listlayers(input_file)
    print(f"Found layers: {layers}")
    
    all_gdfs = []
    for layer in layers:
        print(f"Reading layer: {layer}")
        gdf = gpd.read_file(input_file, layer=layer)
        all_gdfs.append(gdf)
    
    if not all_gdfs:
        print("No layers found.")
        return

    gdf = pd.concat(all_gdfs, ignore_index=True)
    
    # The columns in the KML are 'Name' (capital N) instead of 'name'
    # Let's normalize it to 'name'
    if 'Name' in gdf.columns and 'name' not in gdf.columns:
        gdf['name'] = gdf['Name']
    
    print("Processing records...")
    
    categories = []
    verified_dates = []
    
    if 'description' not in gdf.columns:
        print(f"Error: 'description' column not found in {input_file}. Columns: {gdf.columns.tolist()}")
        return

    for desc in gdf['description']:
        cat, v_date = parse_description(desc)
        categories.append(cat)
        verified_dates.append(v_date)
        
    gdf['category'] = categories
    gdf['verified_date'] = pd.to_datetime(verified_dates, errors='coerce')
    
    # Keep only requested columns + geometry
    # User asked for: name, category, verified_date
    # We'll use the normalized 'name'
    final_gdf = gdf[['name', 'category', 'verified_date', 'geometry']].copy()
    
    # Convert datetime objects to ISO strings for GeoJSON
    final_gdf['verified_date'] = final_gdf['verified_date'].dt.strftime('%Y-%m-%d')

    print(f"Saving to {output_file}...")
    final_gdf.to_file(output_file, driver='GeoJSON')
    print(f"Success! Saved {len(final_gdf)} rows to {output_file}.")

if __name__ == "__main__":
    main()

import geopandas as gpd
import pandas as pd
import re

def parse_description(desc_html):
    if not desc_html:
        return None, None
    
    # 1. Remove HTML tags
    text = re.sub(r'<[^>]+>', ' ', desc_html)
    
    # 2. Replace common entities
    text = text.replace('&nbsp;', ' ').replace('&eacute;', 'e').replace('&egrave;', 'e').replace('&agrave;', 'a').replace('&icirc;', 'i').replace('&otilde;', 'o').replace('&uacute;', 'u')
    # Let's also do a broader replacement for any common ones if possible, but let's start simple.
    # Actually, let's just normalize the text to lowercase and remove non-ascii for matching if needed, 
    # or just match the patterns with possible variations.
    
    # Use a more robust approach: 
    # The text after stripping tags should look like: "b % Arabica /b Catégorie : Cafés ouverts tard a href=... Voir la fiche Derniere vérification : 2026-01-01"
    # (Note: some characters might still be unicode)
    
    # Let's try to find the category by looking for "Catégorie" or "Categorie" or "Categorie"
    # and the date by looking for "vérification" or "verification"
    
    category = None
    # Regex for Category
    # Matches "Catégorie", "Categorie", "Catégorie", etc. followed by ":"
    cat_pattern = re.compile(r'Cat[eéè]gorie\s*[:\-\s]\s*([^:]+?)(?=\s*<|Derni|$)', re.IGNORECASE)
    # Wait, the stripped text might not have < anymore.
    # Let's use a simpler regex: look for the words and then capture until a known delimiter or end of line.
    
    # Actually, let's just use the original string and search for patterns that are likely to match.
    # Let's try to find the content between "Catégorie :" and the next tag or "Dernière"
    
    # Try matching in the raw HTML first to be safe with entities
    # Category: Cat&eacute;gorie&nbsp;: (.*?) <br/ or <a
    cat_match = re.search(r'Cat[eéè]gorie\s*[:\s\x20\xa0]\s*([^<]+)', desc_html, re.IGNORECASE)
    if not cat_match:
        # Try without the accent in the search
        cat_match = re.search(r'Categorie\s*[:\s\x20\xa0]\s*([^<]+)', desc_html, re.IGNORECASE)
        
    if cat_match:
        category = cat_match.group(1).strip()
        # Clean up entities in the captured group
        category = category.replace('&nbsp;', ' ').replace('&eacute;', 'e').replace('&egrave;', 'e')
        # remove any remaining HTML tags if they were captured
        category = re.sub(r'<[^>]+>', '', category)

    verified_date = None
    # Date: Derni&egrave;re v&eacute;rification&nbsp;: (YYYY-MM-DD)
    date_match = re.search(r'Derni[eèé]re\s+v[eéè]rification\s*[:\s\x20\xa0]\s*(\d{4}-\d{2}-\d{2})', desc_html, re.IGNORECASE)
    if not date_match:
         # Try without accents
         date_match = re.search(r'Derniere\s+verification\s*[:\s\x20\xa0]\s*(\d{4}-\d{2}-\d{2})', desc_html, re.IGNORECASE)

    if date_match:
        verified_date = date_match.group(1).strip()
        
    return category, verified_date

def main():
    input_file = 'paris_late_night_pois.kml'
    output_file = 'paris_pois.geojson'
    
    import fiona
    layers = fiona.listlayers(input_file)
    print(f"Found layers: {layers}")
    
    all_gdfs = []
    for layer in layers:
        print(f"Reading layer: {layer}")
        gdf = gpd.read_file(input_file, layer=layer)
        all_gdfs.append(gdf)
    
    if not all_gdfs:
        print("No layers found.")
        return

    gdf = pd.concat(all_gdfs, ignore_index=True)
    
    if 'Name' in gdf.columns and 'name' not in gdf.columns:
        gdf['name'] = gdf['Name']
    
    print("Processing records...")
    
    categories = []
    verified_dates = []
    
    for desc in gdf['description']:
        cat, v_date = parse_description(desc)
        categories.append(cat)
        verified_dates.append(v_date)
        
    gdf['category'] = categories
    gdf['verified_date'] = pd.to_datetime(verified_dates, errors='coerce')
    
    final_gdf = gdf[['name', 'category', 'verified_date', 'geometry']].copy()
    final_gdf['verified_date'] = final_gdf['verified_date'].dt.strftime('%Y-%m-%d')

    print(f"Saving to {output_file}...")
    final_gdf.to_file(output_file, driver='GeoJSON')
    print(f"Success! Saved {len(final_gdf)} rows to {output_file}.")

if __name__ == "__main__":
    main()

import geopandas as gpd
import pandas as pd
import re

def parse_description(desc_html):
    if not desc_html:
        return None, None
    
    # Try matching the category first. 
    # The pattern in the KML is: Cat&eacute;gorie&nbsp;: Cafés ouverts tard<br/>
    # We'll use a regex that accounts for the possible HTML entities or their decoded versions.
    
    # 1. Extract Category
    # We'll look for 'Catégorie' or 'Categorie' (with or without accents)
    # followed by a colon, then the text until the next '<' or '<br/>'
    category = None
    # This pattern looks for "Catégorie" or "Categorie" (with various accent combinations)
    # then a colon/space/nbsp, then captures everything until the next '<'
    cat_pattern = r'Cat[e\x{00e9}\x{00e8}]gorie\s*[:\s\xa0]*([^<]+)'
    # Since re.search might struggle with unicode in the pattern if not careful, 
    # we'll try a few variations.
    
    for pattern in [
        r'Cat[e\xe9\xe8]gorie\s*[:\s\xa0]*([^<]+)',
        r'Categorie\s*[:\s\xa0]*([^<]+)',
        r'Cat\xe9gorie\s*[:\s\xa0]*([^<]+)'
    ]:
        match = re.search(pattern, desc_html, re.IGNORECASE)
        if match:
            category = match.group(1).strip()
            break
            
    if category:
        # Clean up any remaining HTML entities in the category name
        category = category.replace('&nbsp;', ' ').replace('&eacute;', 'e').replace('&egrave;', 'e')
        # Remove any HTML tags that might have been caught
        category = re.sub(r'<[^>]+>', '', category)
        # Clean up leading/trailing whitespace or common entities left over
        category = category.strip()

    # 2. Extract Date
    # The pattern in the KML is: Derni&egrave;re v&eacute;rification&nbsp;: 2026-01-01
    verified_date = None
    date_pattern = r'Derni[e\xe8\xe9]re\s+v[e\xe9\xe8]rification\s*[:\s\xa0]*(\d{4}-\d{2}-\d{2})'
    
    # Try variations of the date pattern
    for pattern in [
        r'Derni[e\xe8\xe9]re\s+v[e\xe9\xe8]rification\s*[:\s\xa0]*(\d{4}-\d{2}-\d{2})',
        r'Derniere\s+verification\s*[:\s\xa0]*(\d{4}-\d{2}-\d{2})'
    ]:
        match = re.search(pattern, desc_html, re.IGNORECASE)
        if match:
            verified_date = match.group(1).strip()
            break
            
    return category, verified_date

def main():
    input_file = 'paris_late_night_pois.kml'
    output_file = 'paris_pois.geojson'
    
    import fiona
    layers = fiona.listlayers(input_file)
    print(f"Found layers: {layers}")
    
    all_gdfs = []
    for layer in layers:
        print(f"Reading layer: {layer}")
        gdf = gpd.read_file(input_file, layer=layer)
        all_gdfs.append(gdf)
    
    if not all_gdfs:
        print("No layers found.")
        return

    gdf = pd.concat(all_gdfs, ignore_index=True)
    
    if 'Name' in gdf.columns and 'name' not in gdf.columns:
        gdf['name'] = gdf['Name']
    
    print("Processing records...")
    
    categories = []
    verified_dates = []
    
    for desc in gdf['description']:
        cat, v_date = parse_description(desc)
        categories.append(cat)
        verified_dates.append(v_date)
        
    gdf['category'] = categories
    gdf['verified_date'] = pd.to_datetime(verified_dates, errors='coerce')
    
    # Keep only requested columns + geometry
    final_gdf = gdf[['name', 'category', 'verified_date', 'geometry']].copy()
    
    # Convert datetime objects to ISO strings for GeoJSON
    final_gdf['verified_date'] = final_gdf['verified_date'].dt.strftime('%Y-%m-%d')

    print(f"Saving to {output_file}...")
    final_gdf.to_file(output_file, driver='GeoJSON')
    print(f"Success! Saved {len(final_gdf)} rows to {output_file}.")

if __name__ == "__main__":
    main()

import geopandas as gpd
import pandas as pd
import re

def parse_description(desc_html):
    if not desc_html:
        return None, None
    
    # Use the exact patterns found in the file content
    # Looking at the original file:
    # <br/>Cat&eacute;gorie&nbsp;: Cafés ouverts tard<br/>
    # <br/>Derni&egrave;re v&eacute;rification&nbsp;: 2026-01-01
    
    category = None
    # Try to find category by matching the HTML entities directly
    cat_match = re.search(r'Cat&eacute;gorie&nbsp;: (.*?)(?=<br/|$)', desc_html)
    if not cat_match:
        # Fallback to a more generic version if the above fails
        cat_match = re.search(r'Cat[e\xe9\xe8]gorie\s*[:\s\xa0]*([^<]+)', desc_html, re.IGNORECASE)
    
    if cat_match:
        category = cat_match.group(1).strip()
        # Clean up common entities in the result
        category = category.replace('&nbsp;', ' ').replace('&eacute;', 'e').replace('&egrave;', 'e')
        category = re.sub(r'<[^>]+>', '', category)

    verified_date = None
    # Try to find date by matching the HTML entities directly
    date_match = re.search(r'Derni&egrave;re v&eacute;rification&nbsp;: (\d{4}-\d{2}-\d{2})', desc_html)
    if not date_match:
        # Fallback to generic
        date_match = re.search(r'Derni[e\xe8\xe9]re\s+v[e\xe9\xe8]rification\s*[:\s\xa0]*(\d{4}-\d{2}-\d{2})', desc_html, re.IGNORECASE)

    if date_match:
        verified_date = date_match.group(1).strip()
        
    return category, verified_date

def main():
    input_file = 'paris_late_night_pois.kml'
    output_file = 'paris_pois.geojson'
    
    import fiona
    layers = fiona.listlayers(input_file)
    print(f"Found layers: {layers}")
    
    all_gdfs = []
    for layer in layers:
        print(f"Reading layer: {layer}")
        gdf = gpd.read_file(input_file, layer=layer)
        all_gdfs.append(gdf)
    
    if not all_gdfs:
        print("No layers found.")
        return

    gdf = pd.concat(all_gdfs, ignore_index=True)
    
    # The KML uses 'Name'
    if 'Name' in gdf.columns and 'name' not in gdf.columns:
        gdf['name'] = gdf['Name']
    
    print("Processing records...")
    
    categories = []
    verified_dates = []
    
    for desc in gdf['description']:
        cat, v_date = parse_description(desc)
        categories.append(cat)
        verified_dates.append(v_date)
        
    gdf['category'] = categories
    gdf['verified_date'] = pd.to_datetime(verified_dates, errors='coerce')
    
    final_gdf = gdf[['name', 'category', 'verified_date', 'geometry']].copy()
    final_gdf['verified_date'] = final_gdf['verified_date'].dt.strftime('%Y-%m-%d')

    print(f"Saving to {output_file}...")
    final_gdf.to_file(output_file, driver='GeoJSON')
    print(f"Success! Saved {len(final_gdf)} rows to {output_file}.")

if __name__ == "__main__":
    main()