fio-l1-paris-kml-pois

Rerun

import geopandas as gpd

try:
    gdf = gpd.read_file('paris_late_night_pois.kml')
    print("Columns:", gdf.columns.tolist())
    print("First few rows:")
    print(gdf.head())
except Exception as e:
    print(f"Error: {e}")

import geopandas as gpd
import pandas as pd
import re
from datetime import datetime

def extract_date(text):
    if not text or not isinstance(text, str):
        return None
    # Look for "last verified" followed by a date pattern
    # Typical pattern might be: "Last verified: 2023-10-25" or similar.
    # The prompt says "HTML blurb", so it might be something like <p>Last verified: 2023-10-25</p>
    # Or "Last verified on 25 Oct 2023"
    
    # Try to find a date pattern (YYYY-MM-DD or similar)
    # Let's try a broad regex first to see what's in there.
    match = re.search(r'last verified.*?(\d{4}-\d{2}-\d{2})', text, re.IGNORECASE)
    if match:
        return match.group(1)
    
    # Try another pattern: "25 October 2023"
    match = re.search(r'last verified.*?(\d{1,2}\s+\w+\s+\d{4})', text, re.IGNORECASE)
    if match:
        try:
            date_str = match.group(1)
            # This is a bit naive, but let's try parsing common formats
            for fmt in ("%d %B %Y", "%d %b %Y"):
                try:
                    return datetime.strptime(date_str, fmt).strftime('%Y-%m-%d')
                except ValueError:
                    continue
        except:
            pass
            
    return None

def process_kml():
    # Since there are multiple layers, we should probably combine them 
    # or iterate through them to capture the "classification/grouping".
    # The warning suggested: 'Cafés ouverts tard', 'Bibliothèques de nuit', 'Tours et infos touristiques'.
    
    layers = ['Cafés ouverts tard', 'Bibliothèques de nuit', 'Tours et infos touristiques']
    all_gdfs = []
    
    for layer in layers:
        try:
            gdf = gpd.read_file('paris_late_night_pois.kml', layer=layer)
            gdf['category'] = layer
            all_gdfs.append(gdf)
            print(f"Loaded layer: {layer} with {len(gdf)} rows")
        except Exception as e:
            print(f"Could not load layer {layer}: {e}")

    if not all_gdfs:
        print("No layers loaded.")
        return

    combined_gdf = pd.concat(all_gdfs, ignore_index=True)
    
    # Rename 'Name' to 'name' as requested
    if 'Name' in combined_gdf.columns:
        combined_gdf = combined_gdf.rename(columns={'Name': 'name'})
    
    # Extract verified_date from description
    if 'description' in combined_gdf.columns:
        combined_gdf['verified_date'] = combined_gdf['description'].apply(extract_date)
    else:
        combined_gdf['verified_date'] = None

    # Keep only requested columns + geometry
    # requested: name, category, verified_date
    cols_to_keep = ['name', 'category', 'verified_date', 'geometry']
    # Ensure columns exist before subsetting
    existing_cols = [c for c in cols_to_keep if c in combined_gdf.columns]
    final_gdf = combined_gdf[existing_cols]

    # Convert to GeoJSON
    final_gdf.to_file('paris_pois.geojson', driver='GeoJSON')
    print("Saved to paris_pois.geojson")
    print(final_gdf.head())

if __name__ == "__main__":
    process_kml()

import geopandas as gpd
import pandas as pd
import re
from datetime import datetime

def extract_date(text):
    if not text or not isinstance(text, str):
        return None
    
    # Remove HTML tags to avoid interference
    clean_text = re.sub('<[^<]+?>', ' ', text)
    
    # Try to find a date pattern: "Last verified: 2023-10-25"
    # We'll search for anything that looks like a date near "last verified"
    # Patterns to look for:
    # 2023-10-25
    # 25/10/2023
    # 25 Oct 2023
    # 25 October 2023
    
    # Let's find the position of "last verified"
    match_keyword = re.search(r'last verified', clean_text, re.IGNORECASE)
    if not match_keyword:
        return None
    
    # Take a substring starting from "last verified"
    sub_text = clean_text[match_keyword.start():]
    
    # 1. Try ISO format YYYY-MM-DD
    iso_match = re.search(r'(\d{4}-\d{2}-\d{2})', sub_text)
    if iso_match:
        return iso_match.group(1)
    
    # 2. Try DD/MM/YYYY
    slash_match = re.search(r'(\d{1,2}/\d{1,2}/\d{4})', sub_text)
    if slash_match:
        try:
            dt = datetime.strptime(slash_match.group(1), '%d/%m/%Y')
            return dt.strftime('%Y-%m-%d')
        except ValueError:
            pass

    # 3. Try DD Month YYYY (e.g., 25 October 2023 or 25 Oct 2023)
    # This regex looks for a digit, then a word (month), then a digit
    word_date_match = re.search(r'(\d{1,2})\s+([A-Za-z]+)\s+(\d{4})', sub_text)
    if word_date_match:
        day = word_date_match.group(1)
        month = word_date_match.group(2)
        year = word_date_match.group(3)
        
        for fmt in ("%d %B %Y", "%d %b %Y"):
            try:
                dt = datetime.strptime(f"{day} {month} {year}", fmt)
                return dt.strftime('%Y-%m-%d')
            except ValueError:
                continue

    return None

def process_kml():
    layers = ['Cafés ouverts tard', 'Bibliothèques de nuit', 'Tours et infos touristiques']
    all_gdfs = []
    
    for layer in layers:
        try:
            gdf = gpd.read_file('paris_late_night_pois.kml', layer=layer)
            gdf['category'] = layer
            all_gdfs.append(gdf)
            print(f"Loaded layer: {layer} with {len(gdf)} rows")
        except Exception as e:
            print(f"Could not load layer {layer}: {e}")

    if not all_gdfs:
        print("No layers loaded.")
        return

    combined_gdf = pd.concat(all_gdfs, ignore_index=True)
    
    if 'Name' in combined_gdf.columns:
        combined_gdf = combined_gdf.rename(columns={'Name': 'name'})
    
    if 'description' in combined_gdf.columns:
        combined_gdf['verified_date'] = combined_gdf['description'].apply(extract_date)
    else:
        combined_gdf['verified_date'] = None

    cols_to_keep = ['name', 'category', 'verified_date', 'geometry']
    existing_cols = [c for c in cols_to_keep if c in combined_gdf.columns]
    final_gdf = combined_gdf[existing_cols].copy()

    # Convert to GeoJSON
    final_gdf.to_file('paris_pois.geojson', driver='GeoJSON')
    print("Saved to paris_pois.geojson")
    
    # Verify the dates
    print("\nVerification check (sample of extracted dates):")
    print(final_gdf[['name', 'verified_date']].dropna().head(10))
    print("\nCounts of missing dates:")
    print(final_gdf['verified_date'].isna().value_counts())

if __name__ == "__main__":
    process_kml()

import geopandas as gpd

def inspect_descriptions():
    try:
        gdf = gpd.read_file('paris_late_night_pois.kml', layer='Cafés ouverts tard')
        print("Description sample 1:")
        print(gdf['description'].iloc[0])
        print("\nDescription sample 2:")
        print(gdf['description'].iloc[1])
        print("\nDescription sample 3:")
        print(gdf['description'].iloc[2])
    except Exception as e:
        print(f"Error: {e}")

if __name__ == "__main__":
    inspect_descriptions()

import geopandas as gpd
import pandas as pd
import re
from datetime import datetime

def extract_date(text):
    if not text or not isinstance(text, str):
        return None
    
    # The text is: <b>% Arabica</b><br/>Cat&eacute;gorie&nbsp;: Cafés ouverts tard<br/><a href="https://example.org/poi/0">Voir la fiche</a><br/>Derni&egrave;re v&eacute;rification&nbsp;: 2026-01-01
    # The phrase "Derni&egrave;re v&eacute;rification" translates to "Last verification"
    # But it's HTML encoded.
    
    # Let's search for the pattern of the date itself or the encoded phrase
    # Pattern: Derni&egrave;re v&eacute;rification&nbsp;: 2026-01-01
    # Or just look for any YYYY-MM-DD after some text.
    
    # Simplified: look for YYYY-MM-DD after "v&eacute;rification" or "verification"
    match = re.search(r'v(?:&eacute;|e)rification.*?(\d{4}-\d{2}-\d{2})', text, re.IGNORECASE)
    if match:
        return match.group(1)
    
    # Fallback: just find the last YYYY-MM-DD in the string
    matches = re.findall(r'(\d{4}-\d{2}-\d{2})', text)
    if matches:
        return matches[-1]
        
    return None

def process_kml():
    layers = ['Cafés ouverts tard', 'Bibliothèques de nuit', 'Tours et infos touristiques']
    all_gdfs = []
    
    for layer in layers:
        try:
            gdf = gpd.read_file('paris_late_night_pois.kml', layer=layer)
            gdf['category'] = layer
            all_gdfs.append(gdf)
            print(f"Loaded layer: {layer} with {len(gdf)} rows")
        except Exception as e:
            print(f"Could not load layer {layer}: {e}")

    if not all_gdfs:
        print("No layers loaded.")
        return

    combined_gdf = pd.concat(all_gdfs, ignore_index=True)
    
    if 'Name' in combined_gdf.columns:
        combined_gdf = combined_gdf.rename(columns={'Name': 'name'})
    
    if 'description' in combined_gdf.columns:
        combined_gdf['verified_date'] = combined_gdf['description'].apply(extract_date)
    else:
        combined_gdf['verified_date'] = None

    cols_to_keep = ['name', 'category', 'verified_date', 'geometry']
    existing_cols = [c for c in cols_to_keep if c in combined_gdf.columns]
    final_gdf = combined_gdf[existing_cols].copy()

    # Convert to GeoJSON
    final_gdf.to_file('paris_pois.geojson', driver='GeoJSON')
    print("Saved to paris_pois.geojson")
    
    # Verify the dates
    print("\nVerification check (sample of extracted dates):")
    print(final_gdf[['name', 'verified_date']].dropna().head(10))
    print("\nCounts of missing dates:")
    print(final_gdf['verified_date'].isna().value_counts())

if __name__ == "__main__":
    process_kml()