dd-l2-tokyo-overture-schools

Rerun

# /// script
# requires-python = ">=3.12"
# dependencies = ["geopandas", "pandas", "pyarrow", "shapely", "pyogrio"]
# ///

import geopandas as gpd
import pandas as pd

def main():
    # Load the wards bbox
    wards_bbox = gpd.read_file("tokyo_23wards_bbox.geojson")
    
    # Load the places (it's a parquet file, but contains spatial data)
    # Since it might be a GeoParquet or just a Parquet with geometry column, 
    # geopandas.read_parquet is the way to go.
    places = gpd.read_parquet("tokyo_places.parquet")
    
    # Filter for schools relevant to age 8-14.
    # Usually, this involves checking 'category' or 'type' or 'tags'.
    # I'll inspect the columns first in a separate step if needed, 
    # but let's try to find relevant rows.
    # I'll assume there's a column that helps identify schools.
    
    # First, let's see what columns we have to identify "schools" and "age"
    # Actually, let's print columns to the console for debugging if it fails.
    print("Places columns:", places.columns.tolist())
    
    # We need to filter for schools. 
    # I'll search for "school" in column names or values.
    # Often in OSM-derived data, 'category' or 'class' or similar is used.
    
    # For now, let's assume we need to filter by a column that indicates the type.
    # Let's check unique values in common columns if they exist.
    
    # I will write a script that first inspects and then filters.
    # Since I can't interactively, I'll write a robust script.

if __name__ == "__main__":
    main()

# /// script
# requires-python = ">=3.12"
# dependencies = ["geopandas", "pandas", "pyarrow", "shapely", "pyogrio"]
# ///

import geopandas as gpd
import pandas as pd
import json

def main():
    # Load the wards bbox
    wards_bbox = gpd.read_file("tokyo_23wards_bbox.geojson")
    
    # Load the places
    places = gpd.read_parquet("tokyo_places.parquet")
    
    # 1. Filter by geometry: inside the wards bbox
    # The bbox is likely a single polygon or a collection of them.
    # We use the spatial join or intersection.
    places_in_wards = gpd.sjoin(places, wards_bbox, predicate='within')

    # 2. Filter for schools relevant to age 8-14.
    # We need to inspect 'categories' and 'names' and 'addresses'.
    # 'categories' might be a list or a string of tags.
    # 'names' might be a list or a dict.
    # 'addresses' might be a list or a dict.
    
    # Let's inspect some sample categories and names to understand the structure.
    # But I'll go straight to a filtering logic.
    
    # Schools for 8-14 are typically Elementary (shogakko) and Junior High (chugakko).
    # I'll check the 'categories' column for "school", "elementary_school", "junior_high_school", etc.
    
    def is_relevant_school(row):
        cats = row.get('categories', [])
        # If cats is a string, make it a list
        if isinstance(cats, str):
            cats = [c.strip().lower() for c in cats.split(',')]
        elif not isinstance(cats, (list, set)):
            cats = []
        else:
            cats = [str(c).lower() for c in cats]
            
        school_keywords = {'school', 'elementary_school', 'junior_high_school', 'middle_school', 'shogakko', 'chugakko'}
        # Check if any keyword is in the categories
        if any(k in cats or k in str(cats) for k in school_keywords):
             return True
        return False

    # In many datasets, 'categories' is a JSON-like string or list.
    # Let's try to filter based on the presence of school-related strings.
    # Since I don't know the exact format, I'll use a flexible string match on the column.
    
    # A safer way: Convert categories to string and check for keywords.
    mask = places_in_wards['categories'].astype(str).str.contains('school|shogakko|chugakko', case=False, na=False)
    schools = places_in_wards[mask].copy()

    # 3. Format the output.
    # Required keys: id, name, confidence, address_freeform, address_locality, address_postcode
    
    # We need to extract these from 'names', 'confidence', and 'addresses'.
    # Let's assume 'names' is a list of strings or a dict, and 'addresses' is a list of dicts or a dict.
    # Based on the previous 'inspect_cols.py' output, they are just columns.
    
    # I'll write a helper to extract values safely.
    
    def extract_name(val):
        if isinstance(val, list) and len(val) > 0: return str(val[0])
        if isinstance(val, dict): return str(list(val.values())[0]) if val else ""
        return str(val)

    def extract_address_field(addr_val, field_name):
        # addr_val might be a list of dicts or a dict
        if isinstance(addr_val, list) and len(addr_val) > 0:
            item = addr_val[0]
            if isinstance(item, dict):
                return str(item.get(field_name, ""))
            return ""
        if isinstance(addr_val, dict):
            return str(addr_val.get(field_name, ""))
        return ""

    # Before applying, let's see a sample of the data to be sure.
    # I will print the first row of 'schools' to see the structure of 'names' and 'addresses'
    if not schools.empty:
        print("Sample row after school filtering:")
        print(schools.iloc[0].to_dict())
    else:
        print("No schools found in wards.")
        return

    # Redefining main logic with the actual structure found.
    # (I'll refine this after running the script)

if __name__ == "__main__":
    main()

# /// script
# requires-python = ">=3.12"
# dependencies = ["geopandas", "pandas", "pyarrow", "shapely", "pyogrio"]
# ///

import geopandas as gpd
import pandas as pd
import numpy as np

def main():
    # Load the wards bbox
    wards_bbox = gpd.read_file("tokyo_23wards_bbox.geojson")
    
    # Load the places
    places = gpd.read_parquet("tokyo_places.parquet")
    
    # Ensure CRS match to avoid warning and errors
    if places.crs != wards_bbox.crs:
        wards_bbox = wards_bbox.to_crs(places.crs)

    # 1. Spatial Join: Find places within the wards
    # We use 'within' to ensure the point is inside the ward polygon.
    places_in_wards = gpd.sjoin(places, wards_bbox, predicate='within')

    # 2. Filter for schools relevant to age 8-14.
    # Based on sample: 'categories' is a dict with 'primary' key.
    # 'names' is a dict with 'primary' key.
    # 'addresses' is a numpy array of dicts.
    
    def is_relevant_school(row):
        cats = row.get('categories')
        if isinstance(cats, dict):
            primary = str(cats.get('primary', '')).lower()
            # Check for school-related terms in primary category
            # Also check for specific subtypes if needed.
            # 'school' is good, but we might want to exclude 'university' if possible.
            # However, the prompt asks for schools for age 8-14.
            # Let's check if 'school' is in the primary category.
            if 'school' in primary:
                # Check if it's not a university (though 'school' might be part of it)
                if 'university' not in primary and 'college' not in primary:
                    return True
        return False

    # Apply filtering
    # Note: we want to be inclusive enough for "schools" but avoid universities.
    # The sample showed 'primary': 'school'.
    
    mask = places_in_wards['categories'].apply(is_relevant_school)
    schools = places_in_wards[mask].copy()

    if schools.empty:
        print("No schools found matching criteria.")
        return

    # 3. Format the properties for the output GeoJSON
    # Required keys: id, name, confidence, address_freeform, address_locality, address_postcode

    def get_name(row):
        names = row.get('names')
        if isinstance(names, dict):
            return str(names.get('primary', ''))
        return ""

    def get_address_field(row, field):
        addresses = row.get('addresses')
        if isinstance(addresses, np.ndarray) and len(addresses) > 0:
            addr_dict = addresses[0]
            if isinstance(addr_dict, dict):
                return str(addr_dict.get(field, ""))
        elif isinstance(addresses, list) and len(addresses) > 0:
            addr_dict = addresses[0]
            if isinstance(addr_dict, dict):
                return str(addr_dict.get(field, ""))
        elif isinstance(addresses, dict):
            return str(addresses.get(field, ""))
        return ""

    # Create the new dataframe with exact keys
    output_df = gpd.GeoDataFrame(geometry=schools.geometry, crs=schools.crs)
    output_df['id'] = schools['id']
    output_df['name'] = schools.apply(get_name, axis=1)
    output_df['confidence'] = schools['confidence']
    output_df['address_freeform'] = schools.apply(lambda r: get_address_field(r, 'freeform'), axis=1)
    output_df['address_locality'] = schools.apply(lambda r: get_address_field(r, 'locality'), axis=1)
    output_df['address_postcode'] = schools.apply(lambda r: get_address_field(r, 'postcode'), axis=1)

    # Ensure we only keep the required columns
    # The GeoJSON format will include 'geometry' as well.
    final_cols = ['id', 'name', 'confidence', 'address_freeform', 'address_locality', 'address_postcode', 'geometry']
    output_df = output_df[final_cols]

    # Save to GeoJSON
    output_df.to_file("tokyo_schools.geojson", driver="GeoJSON")
    print(f"Successfully saved {len(output_df)} schools to tokyo_schools.geojson")

if __name__ == "__main__":
    main()

# /// script
# requires-python = ">=3.12"
# dependencies = ["geopandas", "pandas", "pyarrow", "shapely", "pyogrio"]
# ///

import geopandas as gpd
import pandas as pd
import numpy as np

def main():
    # Load the wards bbox
    wards_bbox = gpd.read_file("tokyo_23wards_bbox.geojson")
    
    # Load the places
    places = gpd.read_parquet("tokyo_places.parquet")
    
    # Ensure CRS match
    if places.crs != wards_bbox.crs:
        wards_bbox = wards_bbox.to_crs(places.crs)

    # 1. Spatial Join: Find places within the wards
    places_in_wards = gpd.sjoin(places, wards_bbox, predicate='within')

    # 2. Filter for schools relevant to age 8-14.
    # Categories might be None or other types.
    def is_relevant_school(cats):
        if cats is None or not isinstance(cats, dict):
            return False
        
        primary = str(cats.get('primary', '')).lower()
        
        # Check for school-related terms in primary category
        # We specifically want elementary/junior high. 
        # The prompt mentions 8-14, which covers these.
        # We check if 'school' is present but not 'university' or 'college'.
        if 'school' in primary:
            if 'university' not in primary and 'college' not in primary:
                return True
        return False

    # Apply filtering safely using .apply() on the series directly
    mask = places_in_wards['categories'].apply(is_relevant_school)
    schools = places_in_wards[mask].copy()

    if schools.empty:
        print("No schools found matching criteria.")
        return

    # 3. Format the properties for the output GeoJSON
    def get_name(names):
        if isinstance(names, dict):
            return str(names.get('primary', ''))
        return ""

    def get_address_field(addresses, field):
        if isinstance(addresses, (np.ndarray, list)) and len(addresses) > 0:
            addr_item = addresses[0]
            if isinstance(addr_item, dict):
                return str(addr_item.get(field, ""))
        elif isinstance(addresses, dict):
            return str(addresses.get(field, ""))
        return ""

    # Create the new dataframe with exact keys
    output_df = gpd.GeoDataFrame(geometry=schools.geometry, crs=schools.crs)
    output_df['id'] = schools['id']
    output_df['name'] = schools['names'].apply(get_name)
    output_df['confidence'] = schools['confidence']
    output_df['address_freeform'] = schools['addresses'].apply(lambda x: get_address_field(x, 'freeform'))
    output_df['address_locality'] = schools['addresses'].apply(lambda x: get_address_field(x, 'locality'))
    output_df['address_postcode'] = schools['addresses'].apply(lambda x: get_address_field(x, 'postcode'))

    # Ensure we only keep the required columns
    final_cols = ['id', 'name', 'confidence', 'address_freeform', 'address_locality', 'address_postcode', 'geometry']
    output_df = output_df[final_cols]

    # Save to GeoJSON
    output_df.to_file("tokyo_schools.geojson", driver="GeoJSON")
    print(f"Successfully saved {len(output_df)} schools to tokyo_schools.geojson")

if __name__ == "__main__":
    main()

# /// script
# requires-python = ">=3.12"
# dependencies = ["geopandas", "pyogrio"]
# ///

import geopandas as gpd

def main():
    gdf = gpd.read_file("tokyo_schools.geojson")
    print("Columns:", gdf.columns.tolist())
    print("First row:\n", gdf.head(1))
    
    expected_keys = ['id', 'name', 'confidence', 'address_freeform', 'address_locality', 'address_postcode']
    for key in expected_keys:
        if key not in gdf.columns:
            print(f"MISSING KEY: {key}")
        else:
            print(f"FOUND KEY: {key}")

if __name__ == "__main__":
    main()