spa-l2-cairo-shop-knn

Rerun

import geopandas as gpd

# Inspect the layers and schema
layers = gpd.list_layers('cairo_retail.gpkg')
print("Layers in GPKG:", layers)

for layer in layers:
    print(f"\n--- Layer: {layer['name']} ---")
    gdf = gpd.read_file('cairo_retail.gpkg', layer=layer['name'])
    print(f"Columns: {gdf.columns.tolist()}")
    print(f"CRS: {gdf.crs}")
    print(f"Head:\n{gdf.head(3)}")

import geopandas as gpd

# Inspect the layers and schema
layers = gpd.list_layers('cairo_retail.gpkg')
print("Layers in GPKG:", layers)

for layer_name in layers:
    print(f"\n--- Layer: {layer_name} ---")
    gdf = gpd.read_file('cairo_retail.gpkg', layer=layer_name)
    print(f"Columns: {gdf.columns.tolist()}")
    print(f"CRS: {gdf.crs}")
    print(f"Head:\n{gdf.head(3)}")

import geopandas as gpd

# Inspect the layers and schema
layers = gpd.list_layers('cairo_retail.gpkg')
print("Layers in GPKG:", layers)

# In the previous run, it seemed layers was a DataFrame? 
# Let's check what layers is.
print(f"Type of layers: {type(layers)}")
if hasattr(layers, 'columns'):
    print(layers)
else:
    print(layers)

# If layers is a DataFrame, the loop should be:
if hasattr(layers, 'columns'):
    for layer_name in layers['name']:
        print(f"\n--- Layer: {layer_name} ---")
        gdf = gpd.read_file('cairo_retail.gpkg', layer=layer_name)
        print(f"Columns: {gdf.columns.tolist()}")
        print(f"CRS: {gdf.crs}")
        print(f"Head:\n{gdf.head(3)}")
else:
    for layer_name in layers:
        print(f"\n--- Layer: {layer_name} ---")
        gdf = gpd.read_file('cairo_retail.gpkg', layer=layer_name)
        print(f"Columns: {gdf.columns.tolist()}")
        print(f"CRS: {gdf.crs}")
        print(f"Head:\n{gdf.head(3)}")

import geopandas as gpd
import pandas as pd
import json
from shapely.ops import nearest_points
from scipy.spatial import cKDTree
import numpy as np

def normalize_name(name):
    if not isinstance(name, str):
        return ""
    # Basic normalization: uppercase, strip whitespace, remove common extra characters
    n = name.upper().strip()
    # For the purpose of this task, we'll use a simple approach to group names.
    # In a real scenario, more sophisticated fuzzy matching might be needed.
    # We'll try to remove common suffixes/prefixes if they seem to be the issue.
    return n

def solve():
    # Load data
    shops = gpd.read_file('cairo_retail.gpkg', layer='shops')
    anchors = gpd.read_file('cairo_retail.gpkg', layer='anchors')

    # Normalizing names for shops
    # We need to ensure all shops of the same chain share the same canonical spelling.
    # The prompt says: "the same shop_id always carries the same normalised_name, 
    # and all shops belonging to one chain share a single canonical spelling."
    # This implies we should group by some similarity.
    # Let's try a simple approach first: uppercase and strip.
    
    # For anchors, we also need to tidy names.
    anchors['anchor_name_normalised'] = anchors['anchor_name'].str.upper().str.strip()

    # Let's define a grouping mechanism for shop names.
    # We'll create a mapping: raw_name -> canonical_name
    # Given the requirement, we can group by a "base" name.
    # Let's try to find a way to consolidate.
    
    def get_base_name(name):
        # Very simple normalization for grouping
        return " ".join(name.upper().split())

    shops['base_name'] = shops['raw_name'].apply(get_base_name)
    
    # Now we need to pick ONE canonical name for each base_name.
    # Let's pick the most frequent one or just the first one encountered.
    name_mapping = {}
    unique_bases = shops['base_name'].unique()
    
    # To handle "inconsistent spellings" better, we might want to group 
    # similar names. But since we don't have a dictionary, 
    # let's assume 'base_name' is a good proxy for now, 
    # or let's try to group names that are very similar.
    
    # Let's use a simple clustering or just unique base names.
    # Actually, let's try to group by the first word if it's a brand.
    # Or better, let's see the data if possible. 
    # Since I can't see all data, I'll use base_name.
    
    for base in unique_bases:
        # Find all raw names that map to this base
        subset = shops[shops['base_name'] == base]['raw_name']
        # Pick the most common raw name as the canonical one
        canonical = subset.mode()[0] if not subset.empty else base
        name_mapping[base] = canonical

    shops['normalised_name'] = shops['base_name'].map(name_mapping)

    # Prepare spatial indexing for KNN
    # Both are in EPSG:22992 (metric)
    shop_coords = np.array(list(shops.geometry.apply(lambda x: (x.x, x.y))))
    anchor_coords = np.array(list(anchors.geometry.apply(lambda x: (x.x, x.y))))
    
    shop_tree = cKDTree(shop_coords)
    anchor_tree = cKDTree(anchor_coords)

    output = []

    for i, anchor in anchors.iterrows():
        anchor_id = anchor['anchor_id']
        anchor_name_norm = anchor['anchor_name_normalised']
        anchor_pos = anchor.geometry.coords[0]

        # 1. Find 5 nearest shops
        # dists, indices = shop_tree.query(anchor_pos, k=5)
        # Note: query might return the anchor itself if it's in the shop list, 
        # but shops and anchors are separate layers.
        dists, shop_indices = shop_tree.query(anchor_pos, k=5)
        
        # Ensure we have exactly 5 (if there are fewer shops, though unlikely)
        if len(shop_indices) < 5:
             # This case shouldn't happen based on problem description
             pass

        knn_list = []
        for d, idx in zip(dists, shop_indices):
            shop = shops.iloc[idx]
            knn_list.append({
                "shop_id": shop['shop_id'],
                "normalised_name": shop['normalised_name'],
                "distance_m": float(d),
                "within_1km": bool(d <= 1000.0)
            })
            
        # 2. Find 3 closest sibling anchors
        # A sibling anchor is an anchor other than the current one.
        # k=4 because the closest will be the anchor itself.
        dist_siblings, sibling_indices = anchor_tree.query(anchor_pos, k=4)
        
        # Filter out the current anchor
        sibling_data = []
        for d, idx in zip(dist_siblings, sibling_indices):
            if anchors.iloc[idx]['anchor_id'] != anchor_id:
                sibling_data.append((d, idx))
        
        # Sort by distance and take top 3
        sibling_data.sort(key=lambda x: x[0])
        top_3_siblings = sibling_data[:3]
        
        # If we have fewer than 3 siblings, we still need a 5x3 matrix. 
        # The prompt says "3 closest sibling anchors". 
        # Let's assume there are at least 3.
        
        # 3. 5x3 distance matrix
        # Rows: knn order (5 shops)
        # Cols: sibling order (3 anchors)
        # Matrix[shop_idx][sibling_idx] = distance from shop to sibling anchor
        
        matrix = []
        for knn_entry in knn_list:
            row = []
            shop_geom = shops.iloc[shop_indices[knn_list.index(knn_entry)]].geometry
            # Wait, knn_list is already in order. 
            # Let's use the shop's index from the original shops dataframe.
            # But we need to find the index in 'shops' that corresponds to the knn_entry.
            # Let's refactor the loop to be cleaner.
            pass

        # Refactored loop for matrix
        matrix = []
        for idx_in_knn, (d_shop_anchor, idx_shop) in enumerate(zip(dists, shop_indices)):
            row = []
            shop_geom = shops.iloc[idx_shop].geometry
            for d_sib_anchor, idx_sib in top_3_siblings:
                sib_geom = anchors.iloc[idx_sib].geometry
                # Distance between shop and sibling anchor
                dist_shop_sib = shop_geom.distance(sib_geom)
                row.append(float(dist_shop_sib))
            matrix.append(row)

        output.append({
            "anchor_id": anchor_id,
            "anchor_name_normalised": anchor_name_norm,
            "knn": knn_list,
            "full_distance_matrix_m": matrix
        })

    # Write to JSON
    with open('market_neighbourhoods.json', 'w') as f:
        json.dump(output, f, indent=2)

if __name__ == "__main__":
    solve()

import geopandas as gpd
import pandas as pd
import json
import numpy as np
import re
from scipy.spatial import cKDTree

def solve():
    # Load data
    shops = gpd.read_file('cairo_retail.gpkg', layer='shops')
    anchors = gpd.read_file('cairo_retail.gpkg', layer='anchors')

    # 1. Normalize names
    # For anchors, just tidy up.
    anchors['anchor_name_normalised'] = anchors['anchor_name'].str.upper().str.strip()

    # For shops: "the same shop_id always carries the same normalised_name, 
    # and all shops belonging to one chain share a single canonical spelling."
    
    def clean_for_grouping(name):
        if not isinstance(name, str): return ""
        # Remove special characters and extra spaces, uppercase
        n = re.sub(r'[^a-zA-Z0-9\s]', '', name).upper()
        return " ".join(n.split())

    shops['group_name'] = shops['raw_name'].apply(clean_for_grouping)
    
    # For each group, pick the most frequent 'raw_name' as the canonical spelling
    canonical_map = {}
    for group, group_df in shops.groupby('group_name'):
        # Most frequent raw_name in this group
        if not group_df.empty:
            most_frequent = group_df['raw_name'].mode()[0]
            canonical_map[group] = most_frequent
        else:
            canonical_map[group] = group

    # Apply canonical name to all shops in that group
    shops['normalised_name'] = shops['group_name'].map(lambda x: canonical_map.get(x, ""))

    # 2. Spatial Analysis
    # Use the metric CRS (EPSG:22992)
    shop_coords = np.array(list(shops.geometry.apply(lambda x: (x.x, x.y))))
    anchor_coords = np.array(list(anchors.geometry.apply(lambda x: (x.x, x.y))))
    
    shop_tree = cKDTree(shop_coords)
    anchor_tree = cKDTree(anchor_coords)

    output = []

    for i, anchor in anchors.iterrows():
        anchor_id = anchor['anchor_id']
        anchor_name_norm = anchor['anchor_name_normalised']
        anchor_pos = anchor.geometry.coords[0]

        # Find 5 nearest shops
        dists, shop_indices = shop_tree.query(anchor_pos, k=5)
        
        # Handle k=1 case (though k=5 is requested)
        if isinstance(dists, float):
            dists = np.array([dists])
            shop_indices = np.array([shop_indices])
        elif isinstance(dists, (int, np.integer)):
            dists = np.array([dists])
            shop_indices = np.array([shop_indices])

        knn_list = []
        for d, s_idx in zip(dists, shop_indices):
            shop = shops.iloc[s_idx]
            knn_list.append({
                "shop_id": shop['shop_id'],
                "normalised_name": shop['normalised_name'],
                "distance_m": float(d),
                "within_1km": bool(d <= 1000.0)
            })
            
        # Find 3 closest sibling anchors
        # Use min to avoid error if there are fewer than 4 anchors total
        k_siblings = min(4, len(anchors))
        d_sibs, s_idx_sibs = anchor_tree.query(anchor_pos, k=k_siblings)
        
        if isinstance(d_sibs, float):
            d_sibs = np.array([d_sibs])
            s_idx_sibs = np.array([s_idx_sibs])
        elif isinstance(d_sibs, (int, np.integer)):
            d_sibs = np.array([d_sibs])
            s_idx_sibs = np.array([s_idx_sibs])

        sibling_data = []
        for d, idx in zip(d_sibs, s_idx_sibs):
            # Use anchor_id for comparison to be safe
            if anchors.iloc[idx]['anchor_id'] != anchor_id:
                sibling_data.append((d, idx))
        
        # Sort by distance and take top 3
        sibling_data.sort(key=lambda x: x[0])
        top_3_siblings = sibling_data[:3]
        
        # 3. Distance Matrix (5x3)
        # Rows: knn (5 shops)
        # Cols: sibling anchors (up to 3)
        matrix = []
        for s_idx in shop_indices:
            row = []
            shop_geom = shops.iloc[s_idx].geometry
            for _, sib_idx in top_3_siblings:
                sib_geom = anchors.iloc[sib_idx].geometry
                dist_val = shop_geom.distance(sib_geom)
                row.append(float(dist_val))
            matrix.append(row)

        output.append({
            "anchor_id": anchor_id,
            "anchor_name_normalised": anchor_name_norm,
            "knn": knn_list,
            "full_distance_matrix_m": matrix
        })

    # Write to JSON
    with open('market_neighbourhoods.json', 'w') as f:
        json.dump(output, f, indent=2)

if __name__ == "__main__":
    solve()