dd-l2-tokyo-overture-schools

Rerun

import geopandas as gpd
import pandas as pd
import json

# Load the bounding box for the 23 wards
wards_bbox = gpd.read_file('tokyo_23wards_bbox.geojson')

# We need to combine the bounding box into a single geometry to check intersection
# or just check if points are within the union of the bounding boxes.
# Actually, the instruction says "inside the wards rectangle in tokyo_23wards_bbox".
# This implies the wards_bbox file contains the geometry of the 23 wards.
# Let's merge them into one polygon to represent the area.
wards_union = wards_bbox.union_all()

# Load the places
# Since it's a parquet file, we'll use geopandas.read_parquet if possible, 
# or read with pandas and convert.
df = pd.read_parquet('tokyo_places.parquet')

# Convert to GeoDataFrame
# The geometry column in parquet might be in a format that pandas/geopandas 
# handles if it was saved as a GeoParquet, but if it's just bytes, we need to handle it.
# However, standard geopandas.read_parquet usually handles it.
gdf = gpd.GeoDataFrame(df, geometry='geometry')

# Filter for schools
# Based on the preview, categories is a dict with 'primary' and 'alternate'
# We want primary == 'school' or 'education' in alternate
def is_school(cat):
    if isinstance(cat, dict):
        if cat.get('primary') == 'school':
            return True
        alternates = cat.get('alternate', [])
        if 'school' in alternates or 'education' in alternates:
            return True
    return False

# Let's check how 'categories' is stored. In the preview it looks like a dict.
# But in the parquet it might be a string or a JSON-like object.
# Let's refine the filtering logic.

# Filtering for schools based on categories
# We'll also filter for the age range 8-14. 
# The prompt says "every school relevant to that age range". 
# Usually, this means Elementary (6-12) and Junior High (12-15).
# However, the dataset doesn't seem to have 'age' properties explicitly.
# Looking at the prompt: "every school relevant to that age range".
# This might mean I need to look for keywords in names or categories?
# Or maybe 'school' is enough if the dataset only contains certain types.
# Actually, in Overture/OpenStreetMap, "school" covers all.
# If there's no age metadata, I'll assume 'school' is the target.
# Wait, "relevant to that age range" might imply I should filter by school type if possible.
# Let's look at the data again.

mask = gdf['categories'].apply(lambda x: (
    (isinstance(x, dict) and x.get('primary') == 'school') or
    (isinstance(x, dict) and 'school' in x.get('alternate', [])) or
    (isinstance(x, dict) and 'education' in x.get('alternate', []))
))

schools = gdf[mask].copy()

# Spatial filter: inside the wards rectangle
schools = schools[schools.intersects(wards_union)]

# Now, the specific requirement for age 8-14.
# If I can't find age in the data, I'll assume 'school' covers it, 
# but I should check if there's any other category like 'elementary_school' or 'junior_high_school'.
# Given the constraints of the prompt, I will proceed with 'school' and 
# check if any other info is available.

# Prepare output columns
# id, name, confidence, address_freeform, address_locality, address_postcode

def extract_address(addr_array):
    # addr_array is an array of dicts
    if isinstance(addr_array, (list, pd.Series, type(None))) and len(addr_array) > 0:
        first_addr = addr_array[0]
        return (
            first_addr.get('freeform'),
            first_addr.get('locality'),
            first_addr.get('postcode')
        )
    return None, None, None

# The 'names' column is a dict. 'names.get("primary")'
# The 'addresses' column is an array of dicts.

results = []
for _, row in schools.iterrows():
    addr_free, addr_loc, addr_post = extract_address(row['addresses'])
    
    name = None
    if isinstance(row['names'], dict):
        name = row['names'].get('primary')
    
    results.append({
        'id': row['id'],
        'name': name,
        'confidence': row['confidence'],
        'address_freeform': addr_free,
        'address_locality': addr_loc,
        'address_postcode': addr_post
    })

output_gdf = gpd.GeoDataFrame(results, geometry=schools['geometry'], crs=schools.crs)

# Ensure the geometry is correct for the output
# Output columns must be exact.
# The columns in results are: id, name, confidence, address_freeform, address_locality, address_postcode
# The geometry is also there.

# Save to geojson
output_gdf.to_file('tokyo_schools.geojson', driver='GeoJSON')

import geopandas as gpd
import pandas as pd
import numpy as np

# Load the bounding box for the 23 wards
wards_bbox = gpd.read_file('tokyo_23wards_bbox.geojson')
wards_union = wards_bbox.union_all()

# Load the places
df = pd.read_parquet('tokyo_places.parquet')
gdf = gpd.GeoDataFrame(df, geometry='geometry')

# Ensure CRS is consistent
if gdf.crs is None:
    gdf.set_crs(epsg=4326, inplace=True)
if wards_union.crs is None:
    wards_union.set_crs(epsg=4326, inplace=True)
else:
    wards_union = wards_union.to_crs(gdf.crs)

def is_school(cat):
    if isinstance(cat, dict):
        if cat.get('primary') == 'school':
            return True
        alternates = cat.get('alternate', [])
        if isinstance(alternates, (list, np.ndarray)):
            if 'school' in alternates or 'education' in alternates:
                return True
    return False

mask = gdf['categories'].apply(is_school)
schools = gdf[mask].copy()

# Spatial filter: inside the wards rectangle
schools = schools[schools.intersects(wards_union)]

def extract_address_info(addr_array):
    if isinstance(addr_array, (list, np.ndarray)) and len(addr_array) > 0:
        first_addr = addr_array[0]
        if isinstance(first_addr, dict):
            return (
                first_addr.get('freeform'),
                first_addr.get('locality'),
                first_addr.get('postcode')
            )
    return None, None, None

results_data = []
for _, row in schools.iterrows():
    addr_free, addr_loc, addr_post = extract_address_info(row['addresses'])
    
    name = None
    if isinstance(row['names'], dict):
        name = row['names'].get('primary')
    
    results_data.append({
        'id': row['id'],
        'name': name,
        'confidence': float(row['confidence']),
        'address_freeform': addr_free,
        'address_locality': addr_loc,
        'address_postcode': addr_post,
        'geometry': row['geometry']
    })

output_gdf = gpd.GeoDataFrame(results_data, crs=gdf.crs)
output_gdf = output_gdf[['id', 'name', 'confidence', 'address_freeform', 'address_locality', 'address_postcode', 'geometry']]

output_gdf.to_file('tokyo_schools.geojson', driver='GeoJSON')
print(f"Successfully saved {len(output_gdf)} schools to tokyo_schools.geojson")

import geopandas as gpd
import pandas as pd
import numpy as np
from shapely import wkb

# Load the bounding box for the 23 wards
wards_bbox = gpd.read_file('tokyo_23wards_bbox.geojson')
wards_union = wards_bbox.union_all()

# Load the places
df = pd.read_parquet('tokyo_places.parquet')

# The 'geometry' column contains WKB bytes. We need to convert them.
def convert_wkb(wkb_bytes):
    if isinstance(wkb_bytes, bytes):
        try:
            return wkb.loads(wkb_bytes)
        except Exception:
            return None
    return None

df['geometry'] = df['geometry'].apply(convert_wkb)

# Drop rows where geometry conversion failed
df = df.dropna(subset=['geometry'])

# Convert to GeoDataFrame
gdf = gpd.GeoDataFrame(df, geometry='geometry')

# Ensure CRS is consistent
if gdf.crs is None:
    gdf.set_crs(epsg=4326, inplace=True)
if wards_union.crs is None:
    wards_union.set_crs(epsg=4326, inplace=True)
else:
    wards_union = wards_union.to_crs(gdf.crs)

def is_school(cat):
    if isinstance(cat, dict):
        if cat.get('primary') == 'school':
            return True
        alternates = cat.get('alternate', [])
        if isinstance(alternates, (list, np.ndarray)):
            if 'school' in alternates or 'education' in alternates:
                return True
    return False

mask = gdf['categories'].apply(is_school)
schools = gdf[mask].copy()

# Spatial filter: inside the wards rectangle
schools = schools[schools.intersects(wards_union)]

def extract_address_info(addr_array):
    if isinstance(addr_array, (list, np.ndarray)) and len(addr_array) > 0:
        first_addr = addr_array[0]
        if isinstance(first_addr, dict):
            return (
                first_addr.get('freeform'),
                first_addr.get('locality'),
                first_addr.get('postcode')
            )
    return None, None, None

results_data = []
for _, row in schools.iterrows():
    addr_free, addr_loc, addr_post = extract_address_info(row['addresses'])
    
    name = None
    if isinstance(row['names'], dict):
        name = row['names'].get('primary')
    
    results_data.append({
        'id': row['id'],
        'name': name,
        'confidence': float(row['confidence']),
        'address_freeform': addr_free,
        'address_locality': addr_loc,
        'address_postcode': addr_post,
        'geometry': row['geometry']
    })

output_gdf = gpd.GeoDataFrame(results_data, crs=gdf.crs)
output_gdf = output_gdf[['id', 'name', 'confidence', 'address_freeform', 'address_locality', 'address_postcode', 'geometry']]

output_gdf.to_file('tokyo_schools.geojson', driver='GeoJSON')
print(f"Successfully saved {len(output_gdf)} schools to tokyo_schools.geojson")

import geopandas as gpd
import pandas as pd
import numpy as np
from shapely import wkb

# Load the bounding box for the 23 wards
wards_bbox = gpd.read_file('tokyo_23wards_bbox.geojson')
wards_union = wards_bbox.union_all()

# Load the places
df = pd.read_parquet('tokyo_places.parquet')

# The 'geometry' column contains WKB bytes. We need to convert them.
def convert_wkb(wkb_bytes):
    if isinstance(wkb_bytes, bytes):
        try:
            return wkb.loads(wkb_bytes)
        except Exception:
            return None
    return None

df['geometry'] = df['geometry'].apply(convert_wkb)

# Drop rows where geometry conversion failed
df = df.dropna(subset=['geometry'])

# Convert to GeoDataFrame
gdf = gpd.GeoDataFrame(df, geometry='geometry')

# Ensure CRS is consistent
if gdf.crs is None:
    gdf.set_crs(epsg=4326, inplace=True)

# wards_union is a shapely geometry, it doesn't have a CRS. 
# We need to make sure wards_bbox (the GeoDataFrame) is in the same CRS as gdf.
wards_bbox = wards_bbox.to_crs(gdf.crs)
wards_union = wards_bbox.union_all()

def is_school(cat):
    if isinstance(cat, dict):
        if cat.get('primary') == 'school':
            return True
        alternates = cat.get('alternate', [])
        if isinstance(alternates, (list, np.ndarray)):
            if 'school' in alternates or 'education' in alternates:
                return True
    return False

mask = gdf['categories'].apply(is_school)
schools = gdf[mask].copy()

# Spatial filter: inside the wards rectangle
schools = schools[schools.intersects(wards_union)]

def extract_address_info(addr_array):
    if isinstance(addr_array, (list, np.ndarray)) and len(addr_array) > 0:
        first_addr = addr_array[0]
        if isinstance(first_addr, dict):
            return (
                first_addr.get('freeform'),
                first_addr.get('locality'),
                first_addr.get('postcode')
            )
    return None, None, None

results_data = []
for _, row in schools.iterrows():
    addr_free, addr_loc, addr_post = extract_address_info(row['addresses'])
    
    name = None
    if isinstance(row['names'], dict):
        name = row['names'].get('primary')
    
    results_data.append({
        'id': row['id'],
        'name': name,
        'confidence': float(row['confidence']),
        'address_freeform': addr_free,
        'address_locality': addr_loc,
        'address_postcode': addr_post,
        'geometry': row['geometry']
    })

output_gdf = gpd.GeoDataFrame(results_data, crs=gdf.crs)
output_gdf = output_gdf[['id', 'name', 'confidence', 'address_freeform', 'address_locality', 'address_postcode', 'geometry']]

output_gdf.to_file('tokyo_schools.geojson', driver='GeoJSON')
print(f"Successfully saved {len(output_gdf)} schools to tokyo_schools.geojson")