spa-l2-cairo-shop-knn

Rerun

import geopandas as gpd

# Load the layers
gdf = gpd.read_file('cairo_retail.gpkg', layer='shops')
anchors = gpd.read_file('cairo_retail.gpkg', layer='anchors')

print("Shops info:")
print(gdf.head())
print(gdf.info())

print("\nAnchors info:")
print(anchors.head())
print(anchors.info())

import geopandas as gpd
import pandas as pd
import numpy as np
from scipy.spatial import cKDTree
import json

def solve():
    # Load data
    shops = gpd.read_file('cairo_retail.gpkg', layer='shops')
    anchors = gpd.read_file('cairo_retail.gpkg', layer='anchors')

    # 1. Normalize names
    # The prompt says "the same shop_id always carries the same normalised_name, 
    # and all shops belonging to one chain share a single canonical spelling."
    # It also says "Tidy the anchor names too."
    
    # For shops, let's assume a common pattern: we need to group by similar names.
    # Since we don't have a explicit "chain" column, I will use a simple string normalization
    # (lowercase, stripped, alphanumeric only) to cluster names.
    # But actually, the instruction implies that the data has inconsistencies.
    # Let's check unique names in shops first to see the variety.
    
    def normalize_string(s):
        if not isinstance(s, str): return ""
        import re
        s = s.strip().upper()
        # Remove non-alphanumeric for grouping (keep spaces to distinguish)
        s = re.sub(r'[^A-Z0-9\s]', '', s)
        return " ".join(s.split())

    # To find "chains", we'll group names that are very similar.
    # However, let's first implement the requirements logically.
    # I'll create a mapping: raw_name -> normalized_name
    
    # Step 1: Create a mapping for shops
    # We'll use a simple approach: group by normalized version.
    shops['norm_temp'] = shops['raw_name'].apply(normalize_string)
    
    # For each normalized group, we pick the most frequent raw name as the canonical one.
    # This handles "inconsistent spellings" by picking the "most common" spelling.
    name_map = {}
    for group_name, group in shops.groupby('norm_temp'):
        canonical_name = group['raw_name'].mode()[0] if not group['raw_name'].mode().empty else group['raw_name'].iloc[0]
        for raw in group['raw_name'].unique():
            name_map[raw] = canonical_name
            
    shops['normalised_name'] = shops['raw_name'].map(name_map)

    # Step 2: Tidy anchor names
    anchors['anchor_name_normalised'] = anchors['anchor_name'].apply(lambda x: normalize_string(x))

    # 2. Proximity analysis
    # Need distances in meters. Check CRS.
    # Assuming the CRS is metric (looking at coordinates 637478, 814957).
    # If it's not, we should reproject to a local UTM (Egypt is UTM 36N).
    
    if shops.crs.is_geographic:
        shops = shops.to_crs(epsg=32636)
        anchors = anchors.to_crs(epsg=32636)
    
    shop_coords = np.array(list(zip(shops.geometry.x, shops.geometry.y)))
    anchor_coords = np.array(list(zip(anchors.geometry.x, anchors.geometry.y)))
    
    shop_tree = cKDTree(shop_coords)
    anchor_tree = cKDTree(anchor_coords)

    output = []

    for i, anchor in anchors.iterrows():
        a_id = anchor['anchor_id']
        a_name = anchor['anchor_name_normalised']
        a_geom = anchor.geometry
        a_coord = np.array([a_geom.x, a_geom.y])

        # KNN shops (5 nearest)
        # Note: distance might be 0 if an anchor is at a shop, but kdtree handles it.
        # We need 5 nearest shops.
        dists_shops, idxs_shops = shop_tree.query(a_coord, k=5)
        
        # If there are fewer than 5 shops total, kdtree might fail or return fewer.
        # But the prompt implies we can get 5.
        
        knn_list = []
        for d, idx in zip(dists_shops, idxs_shops):
            shop = shops.iloc[idx]
            knn_list.append({
                "shop_id": str(shop['shop_id']),
                "normalised_name": str(shop['normalised_name']),
                "distance_m": float(d),
                "within_1km": bool(d <= 1000.0)
            })
        
        # Sort knn_list by distance_m (ascending) - though cKDTree does this
        knn_list = sorted(knn_list, key=lambda x: x['distance_m'])

        # 3. Sibling anchors (3 closest sibling anchors)
        # "Sibling anchors" usually means anchors other than itself.
        # We find the 4 nearest anchors and take the top 3 (excluding self).
        dists_anchors, idxs_anchors = anchor_tree.query(a_coord, k=4)
        
        # Filter out the current anchor itself. 
        # Since the anchor is in the tree, the closest is likely the anchor itself (dist 0).
        sibling_idxs = []
        for idx in idxs_anchors:
            if anchors.iloc[idx]['anchor_id'] != a_id:
                sibling_idxs.append(idx)
            if len(sibling_idxs) == 3:
                break
        
        # If we can't find 3 siblings, we might need to pad or just take what we have.
        # But we need a 5x3 matrix.
        # Let's ensure we have exactly 3 sibling indices for the matrix calculation.
        # If we don't have 3, we'll pad with -1 or something, but let's assume there are enough.
        
        # The matrix: 5 rows (knn shops) by 3 columns (sibling anchors).
        # Column order: sibling anchors in distance-ascending order.
        
        matrix = []
        if len(sibling_idxs) > 0:
            # Get coordinates of sibling anchors
            sibling_coords = anchor_coords[sibling_idxs]
            
            # For each shop in knn, calculate distance to each sibling anchor
            for shop_entry in knn_list:
                row = []
                # We need the actual shop coordinates for the distance calculation
                # to be precise, although the prompt asks for the matrix based on distances.
                # Let's use the shop's geometry.
                s_idx = shops.index[shops['shop_id'] == shop_entry['shop_id']][0]
                s_coord = shop_coords[s_idx]
                
                for s_idx_anchor in sibling_idxs:
                    s_anc_coord = anchor_coords[s_idx_anchor]
                    dist = np.linalg.norm(s_coord - s_anc_coord)
                    row.append(float(dist))
                matrix.append(row)
        
        # If the matrix is not 5x3 (e.g. fewer than 3 siblings), we need to pad?
        # The prompt says "a 5x3 distance matrix". It assumes 3 siblings exist.
        # Let's ensure it's 5x3 by padding with 0.0 or something if needed, 
        # but usually in these tasks, the data is sufficient.
        
        # To be safe, if matrix is smaller than 5x3, pad it.
        while len(matrix) < 5:
            matrix.append([0.0, 0.0, 0.0])
        while len(matrix[0]) < 3:
            for r in range(len(matrix)):
                matrix[r].append(0.0)

        output.append({
            "anchor_id": a_id,
            "anchor_name_normalised": a_name,
            "knn": knn_list,
            "full_distance_matrix_m": matrix
        })

    with open('market_neighbourhoods.json', 'w') as f:
        json.dump(output, f, indent=2)

if __name__ == "__main__":
    solve()

import geopandas as gpd
import pandas as pd
import numpy as np
from scipy.spatial import cKDTree
import json
import re

def normalize_string(s):
    if not isinstance(s, str): return ""
    s = s.strip().upper()
    # Remove non-alphanumeric for grouping (keep spaces to distinguish)
    s = re.sub(r'[^A-Z0-9\s]', '', s)
    return " ".join(s.split())

def solve():
    # Load data
    shops = gpd.read_file('cairo_retail.gpkg', layer='shops')
    anchors = gpd.read_file('cairo_retail.gpkg', layer='anchors')

    # 1. Normalize names
    # For shops: Group by normalized version and pick the mode (most common) as the canonical name
    shops['norm_temp'] = shops['raw_name'].apply(normalize_string)
    
    # Create a mapping: raw_name -> canonical_name
    # To ensure "the same shop_id always carries the same normalised_name",
    # we calculate it once.
    name_map = {}
    for _, group in shops.groupby('norm_temp'):
        # Mode might return multiple if tied, take the first one
        modes = group['raw_name'].mode()
        canonical_name = modes[0] if not modes.empty else group['raw_name'].iloc[0]
        for raw in group['raw_name'].unique():
            name_map[raw] = canonical_name
            
    shops['normalised_name'] = shops['raw_name'].map(name_map)

    # Tidy anchor names
    anchors['anchor_name_normalised'] = anchors['anchor_name'].apply(normalize_string)

    # 2. Proximity analysis
    # Convert to metric CRS if needed. We'll use UTM 36N (EPSG:32636) which is appropriate for Cairo.
    if shops.crs.is_geographic:
        shops = shops.to_crs(epsg=32636)
        anchors = anchors.to_crs(epsg=32636)
    
    shop_coords = np.array(list(zip(shops.geometry.x, shops.geometry.y)))
    anchor_coords = np.array(list(zip(anchors.geometry.x, anchors.geometry.y)))
    
    shop_tree = cKDTree(shop_coords)
    anchor_tree = cKDTree(anchor_coords)

    output = []

    for i, anchor in anchors.iterrows():
        a_id = anchor['anchor_id']
        a_name = anchor['anchor_name_normalised']
        a_coord = np.array([anchor.geometry.x, anchor.geometry.y])

        # KNN shops (5 nearest)
        dists_shops, idxs_shops = shop_tree.query(a_coord, k=5)
        
        # Handle case where k might be larger than available points (though unlikely here)
        if len(dists_shops) < 5:
            # This part might need adjustment if k=5 is not possible
            pass

        knn_list = []
        for d, idx in zip(dists_shops, idxs_shops):
            shop = shops.iloc[idx]
            knn_list.append({
                "shop_id": str(shop['shop_id']),
                "normalised_name": str(shop['normalised_name']),
                "distance_m": float(d),
                "within_1km": bool(d <= 1000.0)
            })
        
        # Ensure knn is sorted by distance (cKDTree query usually returns sorted, but let's be explicit)
        knn_list = sorted(knn_list, key=lambda x: x['distance_m'])

        # 3. Sibling anchors (3 closest sibling anchors)
        # We need the 3 closest anchors that are NOT the current anchor.
        # Query k=4 (to account for the anchor itself)
        dists_anchors, idxs_anchors = anchor_tree.query(a_coord, k=4)
        
        sibling_idxs = []
        for idx in idxs_anchors:
            if anchors.iloc[idx]['anchor_id'] != a_id:
                sibling_idxs.append(idx)
            if len(sibling_idxs) == 3:
                break
        
        # Fill sibling_idxs if we have fewer than 3
        # In a real scenario we'd handle this, here we'll assume the data permits it or pad.
        
        # full_distance_matrix_m: 5 rows (knn shops) by 3 columns (sibling anchors)
        matrix = []
        for shop_entry in knn_list:
            # Find the shop coordinate
            # Optimization: use a dictionary for index lookup if performance was an issue
            s_idx = shops.index[shops['shop_id'] == shop_entry['shop_id']][0]
            s_coord = shop_coords[s_idx]
            
            row = []
            for s_idx_anchor in sibling_idxs:
                anc_coord = anchor_coords[s_idx_anchor]
                dist = np.linalg.norm(s_coord - anc_coord)
                row.append(float(dist))
            
            # Pad row if fewer than 3 siblings found
            while len(row) < 3:
                row.append(0.0)
            matrix.append(row)
            
        # Pad matrix if fewer than 5 shops in knn (shouldn't happen with k=5)
        while len(matrix) < 5:
            matrix.append([0.0, 0.0, 0.0])

        output.append({
            "anchor_id": a_id,
            "anchor_name_normalised": a_name,
            "knn": knn_list,
            "full_distance_matrix_m": matrix
        })

    with open('market_neighbourhoods.json', 'w') as f:
        json.dump(output, f, indent=2)

if __name__ == "__main__":
    solve()