spa-l2-cairo-shop-knn

Rerun

import geopandas as gpd

# Load the data
gdf = gpd.read_file('cairo_retail.gpkg')

# Check the layers and contents
print("Layers in GPKG:", gpd.io.file.list_layers('cairo_retail.gpkg'))

# Let's see the columns for each layer
shops = gpd.read_file('cairo_retail.gpkg', layer='shops')
anchors = gpd.read_file('cairo_retail.gpkg', layer='anchors')

print("\nShops columns:", shops.columns.tolist())
print("Anchors columns:", anchors.columns.tolist())

print("\nFirst few shops:")
print(shops.head())

print("\nFirst few anchors:")
print(anchors.head())

import geopandas as gpd

# Load the data
try:
    import fiona
    layers = fiona.listlayers('cairo_retail.gpkg')
    print("Layers in GPKG:", layers)
except Exception as e:
    print("Error listing layers:", e)

# Let's see the columns for each layer
shops = gpd.read_file('cairo_retail.gpkg', layer='shops')
anchors = gpd.read_file('cairo_retail.gpkg', layer='anchors')

print("\nShops columns:", shops.columns.tolist())
print("Anchors columns:", anchors.columns.tolist())

print("\nFirst few shops:")
print(shops.head())

print("\nFirst few anchors:")
print(anchors.head())

import geopandas as gpd
import pandas as pd
import numpy as np
import json
from sklearn.neighbors import BallTree

def solve():
    # Load the data
    shops = gpd.read_file('cairo_retail.gpkg', layer='shops')
    anchors = gpd.read_file('cairo_retail.gpkg', layer='anchors')

    # 1. Normalize names
    # For shops: group by name (case insensitive, strip whitespace)
    # However, the instruction says: "the same shop_id always carries the same normalised_name, 
    # and all shops belonging to one chain share a single canonical spelling."
    # This implies a group-by logic.
    
    def get_canonical_names(df, name_col):
        # Strip and upper case for grouping
        df['temp_name'] = df[name_col].str.strip().str.upper()
        # Use the first occurrence as canonical name for each group
        mapping = df.groupby('temp_name')[name_col].first().to_dict()
        return df['temp_name'].map(mapping)

    # Let's try a more robust normalization:
    # A "chain" is identified by the cleaned version of the raw name.
    def clean_name(name):
        if not name: return ""
        return " ".join(name.upper().split())

    shops['normalised_name'] = shops['raw_name'].apply(clean_name)
    anchors['anchor_name_normalised'] = anchors['anchor_name'].apply(clean_name)

    # 2. Distance calculation using BallTree (more efficient for spatial lookups)
    # Ensure we use a projected CRS for metres. The coordinates look like UTM (600k, 800k).
    # Let's check CRS.
    if shops.crs is None:
        print("Warning: Shops CRS is None. Assuming EPSG:3857 or similar.")
        # In reality, we should find the correct CRS. For now, we'll assume it's metric.
    
    # If CRS is not set, let's try to detect or set it if we know it's Cairo. 
    # Looking at the coordinates (637478, 814957), it's likely a local UTM or similar.
    # For the sake of the task, we will proceed assuming the geometry is already in metres.
    
    # Convert geometries to numpy arrays for BallTree
    shop_coords = np.array(list(zip(shops.geometry.x, shops.geometry.y)))
    anchor_coords = np.array(list(zip(anchors.geometry.x, anchors.geometry.y)))

    # KNN for shops (5 nearest for each anchor)
    tree_shops = BallTree(shop_coords)
    # Query 5 nearest neighbors for each anchor
    # k=5
    dist_shops, ind_shops = tree_shops.query(anchor_coords, k=5)

    # KNN for sibling anchors (3 nearest siblings for each anchor)
    # "sibling anchors" usually means anchors excluding itself.
    # Since we need 3, we query k=4 and discard the first one (distance 0).
    tree_anchors = BallTree(anchor_coords)
    dist_siblings, ind_siblings = tree_anchors.query(anchor_coords, k=4)

    # 3. Build the result list
    results = []
    
    for i in range(len(anchors)):
        anchor_id = anchors.iloc[i]['anchor_id']
        anchor_name_norm = anchors.iloc[i]['anchor_name_normalised']
        
        # KNN shops
        knn_list = []
        for d, idx in zip(dist_shops[i], ind_shops[i]):
            shop = shops.iloc[idx]
            knn_list.append({
                "shop_id": str(shop['shop_id']),
                "normalised_name": shop['normalised_name'],
                "distance_m": float(d),
                "within_1km": bool(d <= 1000)
            })
        
        # 3 Closest siblings (excluding self)
        # dist_siblings[i, 0] is 0 (self). We take indices 1, 2, 3.
        sibling_indices = ind_siblings[i, 1:4]
        sibling_dists = dist_siblings[i, 1:4]
        
        # Distance matrix (5x3): 5 shops (rows) to 3 sibling anchors (cols)
        # rows follow knn order
        # columns correspond to sibling anchors in distance-ascending order
        matrix = []
        for s_idx in range(5):
            row = []
            shop_coord = shop_coords[ind_shops[i, s_idx]]
            for sib_idx in sibling_indices:
                sib_coord = anchor_coords[sib_idx]
                # Euclidean distance for the matrix (since we are in metric space)
                # Or we can use the distance calculated by BallTree if it was just between anchors, 
                # but we need distance between SHOP and SIBLING ANCHOR.
                d_shop_sib = np.linalg.norm(shop_coord - sib_coord)
                row.append(float(d_shop_sib))
            matrix.append(row)
            
        results.append({
            "anchor_id": str(anchor_id),
            "anchor_name_normalised": anchor_name_norm,
            "knn": knn_list,
            "full_distance_matrix_m": matrix
        })

    # Write output
    with open('market_neighbourhoods.json', 'w') as f:
        json.dump(results, f, indent=2)

if __name__ == "__main__":
    solve()

import geopandas as gpd
import pandas as pd
import numpy as np
import json
from scipy.spatial import cKDTree

def solve():
    # Load the data
    shops = gpd.read_file('cairo_retail.gpkg', layer='shops')
    anchors = gpd.read_file('cairo_retail.gpkg', layer='anchors')

    # 1. Normalize names
    def clean_name(name):
        if not name: return ""
        return " ".join(name.upper().split())

    # Requirement: "the same shop_id always carries the same normalised_name, 
    # and all shops belonging to one chain share a single canonical spelling."
    # Let's normalize raw names to find groups.
    shops['clean_raw'] = shops['raw_name'].apply(clean_name)
    anchors['anchor_name_normalised'] = anchors['anchor_name'].apply(clean_name)

    # Group shops by their cleaned name to find a canonical version.
    # Using the first occurrence of the raw name for each cleaned name.
    canonical_map = shops.groupby('clean_raw')['raw_name'].first().to_dict()
    shops['normalised_name'] = shops['clean_raw'].map(lambda x: clean_name(canonical_map[x]))

    # 2. Distance calculation using cKDTree (standard in scipy)
    shop_coords = np.array(list(zip(shops.geometry.x, shops.geometry.y)))
    anchor_coords = np.array(list(zip(anchors.geometry.x, anchors.geometry.y)))

    # KNN for shops (5 nearest for each anchor)
    tree_shops = cKDTree(shop_coords)
    dist_shops, ind_shops = tree_shops.query(anchor_coords, k=5)

    # KNN for sibling anchors (3 nearest siblings for each anchor)
    tree_anchors = cKDTree(anchor_coords)
    # Query k=4 to skip the anchor itself
    dist_siblings, ind_siblings = tree_anchors.query(anchor_coords, k=4)

    # 3. Build the result list
    results = []
    
    for i in range(len(anchors)):
        anchor_id = str(anchors.iloc[i]['anchor_id'])
        anchor_name_norm = anchors.iloc[i]['anchor_name_normalised']
        
        # KNN shops
        knn_list = []
        for d, idx in zip(dist_shops[i], ind_shops[i]):
            shop = shops.iloc[idx]
            knn_list.append({
                "shop_id": str(shop['shop_id']),
                "normalised_name": shop['normalised_name'],
                "distance_m": float(d),
                "within_1km": bool(d <= 1000.000001) # floating point buffer
            })
        
        # 3 Closest siblings (excluding self)
        # index 0 is self, so take 1, 2, 3
        sibling_indices = ind_siblings[i, 1:4]
        
        # Distance matrix (5x3): 5 shops (rows) to 3 sibling anchors (cols)
        # rows follow knn order
        # columns correspond to sibling anchors in distance-ascending order
        matrix = []
        for s_idx in range(5):
            row = []
            shop_coord = shop_coords[ind_shops[i, s_idx]]
            for sib_idx in sibling_indices:
                sib_coord = anchor_coords[sib_idx]
                # Distance between shop and sibling anchor
                d_shop_sib = np.linalg.norm(shop_coord - sib_coord)
                row.append(float(d_shop_sib))
            matrix.append(row)
            
        results.append({
            "anchor_id": anchor_id,
            "anchor_name_normalised": anchor_name_norm,
            "knn": knn_list,
            "full_distance_matrix_m": matrix
        })

    # Write output
    with open('market_neighbourhoods.json', 'w') as f:
        json.dump(results, f, indent=2)

if __name__ == "__main__":
    solve()

import geopandas as gpd
import pandas as pd
import numpy as np
import json

def solve():
    # Load the data
    shops = gpd.read_file('cairo_retail.gpkg', layer='shops')
    anchors = gpd.read_file('cairo_retail.gpkg', layer='anchors')

    # 1. Normalize names
    def clean_name(name):
        if not name: return ""
        return " ".join(name.upper().split())

    # Requirement: "the same shop_id always carries the same normalised_name, 
    # and all shops belonging to one chain share a single canonical spelling."
    # Let's normalize raw names to find groups.
    shops['clean_raw'] = shops['raw_name'].apply(clean_name)
    anchors['anchor_name_normalised'] = anchors['anchor_name'].apply(clean_name)

    # Group shops by their cleaned name to find a canonical version.
    # Using the first occurrence of the raw name for each cleaned name.
    # We'll map everything to the cleaned version to be safe, or use first raw name.
    # "all shops belonging to one chain share a single canonical spelling"
    # Let's use the cleaned version itself as the normalized name.
    shops['normalised_name'] = shops['clean_raw']

    # 2. Distance calculation using manual calculation for simplicity (given size constraints)
    # Since we don't have scipy/sklearn, we'll use numpy broadcasting if the size allows,
    # or just simple loops for each anchor.
    
    shop_coords = np.array(list(zip(shops.geometry.x, shops.geometry.y)))
    anchor_coords = np.array(list(zip(anchors.geometry.x, anchors.geometry.y)))

    # Pre-calculate all anchor-to-anchor distances to find siblings
    # dist_anchors[i, j] is distance between anchor i and anchor j
    diff_anchors = anchor_coords[:, np.newaxis, :] - anchor_coords[np.newaxis, :, :]
    dist_anchors_matrix = np.sqrt(np.sum(diff_anchors**2, axis=-1))

    # 3. Build the result list
    results = []
    
    for i in range(len(anchors)):
        anchor_id = str(anchors.iloc[i]['anchor_id'])
        anchor_name_norm = anchors.iloc[i]['anchor_name_normalised']
        anchor_pos = anchor_coords[i]

        # Finding 5 nearest shops for this anchor
        diff_shops = shop_coords - anchor_pos
        dist_to_shops = np.sqrt(np.sum(diff_shops**2, axis=1))
        
        # Get indices of 5 nearest shops
        # argsort is reliable
        nearest_shop_indices = np.argsort(dist_to_shops)[:5]
        
        knn_list = []
        for idx in nearest_shop_indices:
            d = dist_to_shops[idx]
            shop = shops.iloc[idx]
            knn_list.append({
                "shop_id": str(shop['shop_id']),
                "normalised_name": shop['normalised_name'],
                "distance_m": float(d),
                "within_1km": bool(d <= 1000.000001)
            })
        
        # Finding 3 closest siblings (excluding self)
        # We already have dist_anchors_matrix
        # For anchor i, distances to others are dist_anchors_matrix[i]
        # We need to ignore the 0 distance to itself
        
        # Set distance to self as infinity so it's not selected in top 3
        dist_to_others = dist_anchors_matrix[i].copy()
        dist_to_others[i] = np.inf
        
        nearest_sibling_indices = np.argsort(dist_to_others)[:3]
        
        # Distance matrix (5x3): 5 shops (rows) to 3 sibling anchors (cols)
        # rows follow knn order
        # columns correspond to sibling anchors in distance-ascending order
        matrix = []
        for s_idx in nearest_shop_indices:
            row = []
            shop_pos = shop_coords[s_idx]
            for sib_idx in nearest_sibling_indices:
                sib_pos = anchor_coords[sib_idx]
                d_shop_sib = np.linalg.norm(shop_pos - sib_pos)
                row.append(float(d_shop_sib))
            matrix.append(row)
            
        results.append({
            "anchor_id": anchor_id,
            "anchor_name_normalised": anchor_name_norm,
            "knn": knn_list,
            "full_distance_matrix_m": matrix
        })

    # Write output
    with open('market_neighbourhoods.json', 'w') as f:
        json.dump(results, f, indent=2)

if __name__ == "__main__":
    solve()

import geopandas as gpd
import pandas as pd
import numpy as np
import json

def solve():
    # Load the data
    shops = gpd.read_file('cairo_retail.gpkg', layer='shops')
    anchors = gpd.read_file('cairo_retail.gpkg', layer='anchors')

    # 1. Normalize names
    def clean_name(name):
        if not name: return ""
        return " ".join(name.upper().split())

    # Requirement: "the same shop_id always carries the same normalised_name, 
    # and all shops belonging to one chain share a single canonical spelling."
    
    # First, create a cleaned version to group by
    shops['clean_raw'] = shops['raw_name'].apply(clean_name)
    
    # Determine canonical name for each chain. 
    # We'll use the first original name (from the input) that corresponds to the cleaned name.
    # Let's sort them by shop_id to ensure consistency if we were picking "first".
    shops = shops.sort_values('shop_id')
    canonical_map = shops.groupby('clean_raw')['raw_name'].first().to_dict()
    
    # Now, the normalized name is the cleaned version of the canonical name.
    shops['normalised_name'] = shops['clean_raw'].map(lambda x: clean_name(canonical_map[x]))
    
    # Anchors normalization
    anchors['anchor_name_normalised'] = anchors['anchor_name'].apply(clean_name)

    # 2. Distance calculation
    shop_coords = np.array(list(zip(shops.geometry.x, shops.geometry.y)))
    anchor_coords = np.array(list(zip(anchors.geometry.x, anchors.geometry.y)))

    # Pre-calculate all anchor-to-anchor distances to find siblings
    diff_anchors = anchor_coords[:, np.newaxis, :] - anchor_coords[np.newaxis, :, :]
    dist_anchors_matrix = np.sqrt(np.sum(diff_anchors**2, axis=-1))

    # 3. Build the result list
    results = []
    
    for i in range(len(anchors)):
        anchor_id = str(anchors.iloc[i]['anchor_id'])
        anchor_name_norm = anchors.iloc[i]['anchor_name_normalised']
        anchor_pos = anchor_coords[i]

        # Finding 5 nearest shops for this anchor
        diff_shops = shop_coords - anchor_pos
        dist_to_shops = np.sqrt(np.sum(diff_shops**2, axis=1))
        
        # Get indices of 5 nearest shops
        nearest_shop_indices = np.argsort(dist_to_shops)[:5]
        
        knn_list = []
        for idx in nearest_shop_indices:
            d = dist_to_shops[idx]
            shop = shops.iloc[idx]
            knn_list.append({
                "shop_id": str(shop['shop_id']),
                "normalised_name": shop['normalised_name'],
                "distance_m": float(d),
                "within_1km": bool(d <= 1000.000001)
            })
        
        # Finding 3 closest siblings (excluding self)
        dist_to_others = dist_anchors_matrix[i].copy()
        dist_to_others[i] = np.inf
        nearest_sibling_indices = np.argsort(dist_to_others)[:3]
        
        # Distance matrix (5x3): 5 shops (rows) to 3 sibling anchors (cols)
        matrix = []
        for s_idx in nearest_shop_indices:
            row = []
            shop_pos = shop_coords[s_idx]
            for sib_idx in nearest_sibling_indices:
                sib_pos = anchor_coords[sib_idx]
                d_shop_sib = np.linalg.norm(shop_pos - sib_pos)
                row.append(float(d_shop_sib))
            matrix.append(row)
            
        results.append({
            "anchor_id": anchor_id,
            "anchor_name_normalised": anchor_name_norm,
            "knn": knn_list,
            "full_distance_matrix_m": matrix
        })

    # Write output
    with open('market_neighbourhoods.json', 'w') as f:
        json.dump(results, f, indent=2)

if __name__ == "__main__":
    solve()

import geopandas as gpd
import pandas as pd
import numpy as np
import json

def solve():
    # Load the data
    shops = gpd.read_file('cairo_retail.gpkg', layer='shops')
    anchors = gpd.read_file('cairo_retail.gpkg', layer='anchors')

    # 1. Normalize names
    def clean_name(name):
        if not name: return ""
        return " ".join(name.upper().split())

    # Requirement: "the same shop_id always carries the same normalised_name, 
    # and all shops belonging to one chain share a single canonical spelling."
    
    # We need to group names that are "the same chain".
    # A common way to do this for retail is to look at the first word 
    # OR the cleaned version. 
    # Given the Seoudi example, 'SEOUDI MARKET' and 'SEOUDI' should likely be the same.
    # However, 'LOCAL SHOP 00001' and 'LOCAL SHOP 00002' should NOT be the same.
    
    # Let's use a logic: if the cleaned names share the same first word 
    # AND that word is not 'LOCAL', 'THE', 'A', etc., they might be the same.
    # But that's too complex and risky.
    
    # Let's try a different approach: 
    # Create 'clean_raw' as the baseline.
    shops['clean_raw'] = shops['raw_name'].apply(clean_name)
    
    # To handle 'SEOUDI' vs 'SEOUDI MARKET', we can group by the first word 
    # but only if the word is not 'LOCAL' or 'SHOP'.
    def get_chain_id(name):
        parts = name.split()
        if not parts: return ""
        first_word = parts[0]
        # If it's a generic word, the chain id is the whole name
        if first_word in ['LOCAL', 'SHOP', 'THE', 'A', 'AN']:
            return name
        return first_word

    shops['chain_id'] = shops['clean_raw'].apply(get_chain_id)
    
    # Now, for each chain_id, pick a canonical name.
    # We'll pick the most frequent 'clean_raw' within that chain_id.
    def get_canonical(group):
        return group['clean_raw'].value_counts().idxmax()

    canonical_map = shops.groupby('chain_id').apply(get_canonical).to_dict()
    
    # Mapping the chain_id to the canonical clean_raw
    shops['normalised_name'] = shops['chain_id'].map(canonical_map)
    
    # Anchors normalization
    anchors['anchor_name_normalised'] = anchors['anchor_name'].apply(clean_name)

    # 2. Distance calculation
    shop_coords = np.array(list(zip(shops.geometry.x, shops.geometry.y)))
    anchor_coords = np.array(list(zip(anchors.geometry.x, anchors.geometry.y)))

    # Pre-calculate all anchor-to-anchor distances to find siblings
    diff_anchors = anchor_coords[:, np.newaxis, :] - anchor_coords[np.newaxis, :, :]
    dist_anchors_matrix = np.sqrt(np.sum(diff_anchors**2, axis=-1))

    # 3. Build the result list
    results = []
    
    for i in range(len(anchors)):
        anchor_id = str(anchors.iloc[i]['anchor_id'])
        anchor_name_norm = anchors.iloc[i]['anchor_name_normalised']
        anchor_pos = anchor_coords[i]

        # Finding 5 nearest shops for this anchor
        diff_shops = shop_coords - anchor_pos
        dist_to_shops = np.sqrt(np.sum(diff_shops**2, axis=1))
        
        # Get indices of 5 nearest shops
        nearest_shop_indices = np.argsort(dist_to_shops)[:5]
        
        knn_list = []
        for idx in nearest_shop_indices:
            d = dist_to_shops[idx]
            shop = shops.iloc[idx]
            knn_list.append({
                "shop_id": str(shop['shop_id']),
                "normalised_name": shop['normalised_name'],
                "distance_m": float(d),
                "within_1km": bool(d <= 1000.000001)
            })
        
        # Finding 3 closest siblings (excluding self)
        dist_to_others = dist_anchors_matrix[i].copy()
        dist_to_others[i] = np.inf
        nearest_sibling_indices = np.argsort(dist_to_others)[:3]
        
        # Distance matrix (5x3): 5 shops (rows) to 3 sibling anchors (cols)
        # rows follow knn order
        # columns correspond to sibling anchors in distance-ascending order
        matrix = []
        for s_idx in nearest_shop_indices:
            row = []
            shop_pos = shop_coords[s_idx]
            for sib_idx in nearest_sibling_indices:
                sib_pos = anchor_coords[sib_idx]
                d_shop_sib = np.linalg.norm(shop_pos - sib_pos)
                row.append(float(d_shop_sib))
            matrix.append(row)
            
        results.append({
            "anchor_id": anchor_id,
            "anchor_name_normalised": anchor_name_norm,
            "knn": knn_list,
            "full_distance_matrix_m": matrix
        })

    # Write output
    with open('market_neighbourhoods.json', 'w') as f:
        json.dump(results, f, indent=2)

if __name__ == "__main__":
    solve()