dd-l2-bangkok-multicrs-audit

Rerun

import geopandas as gpd
import fiona
import pandas as pd
import numpy as np

def detect_encoding(gdf):
    """
    Heuristic to detect if text columns contain mojibake.
    We check string columns for common patterns of mojibake (e.g., weird character sequences).
    However, a simpler way is to check if we can decode/encode without error 
    or if there are high counts of replacement characters.
    """
    # In the context of this prompt, it asks to look for 'latin1-mojibake' or 'utf-8'.
    # A common way to detect mojibake is checking if 'Ã' or similar characters appear 
    # where they shouldn't, or if string encoding/decoding cycles produce artifacts.
    
    # Let's check all object columns
    str_cols = gdf.select_dtypes(include=['object']).columns
    if len(str_cols) == 0:
        return "utf-8" # Default if no text
    
    for col in str_cols:
        # Get all unique non-null values
        samples = gdf[col].dropna().unique()
        for s in samples:
            if not isinstance(s, str):
                continue
            # If it's mojibake, it often contains characters like Ã, ©, etc. 
            # resulting from interpreting latin1 as utf-8.
            # A common check: if we try to encode as utf-8 and then decode as latin1,
            # does it look like gibberish? 
            # Actually, the prompt suggests looking for 'anomalies'.
            # Let's try to see if there are "replacement characters" or common mojibake markers.
            if "Ã" in s or "Â" in s:
                return "latin1-mojibake"
                
    return "utf-8"

def audit_layers(file_path):
    layers = fiona.listlayers(file_path)
    audit_results = []

    for layer in layers:
        gdf = gpd.read_file(file_path, layer=layer)
        
        # layer_name
        layer_name = layer
        
        # declared_crs (formatted as EPSG:NNNN)
        crs = gdf.crs
        if crs is not None:
            if crs.is_epsg:
                declared_crs = f"EPSG:{crs.to_epsg()}"
            else:
                # Fallback for non-standard or projected CRS without EPSG code
                declared_crs = str(crs)
        else:
            declared_crs = "None"
            
        # geometry_type
        # gdf.geometry.type returns the type of the geometry series
        if not gdf.empty:
            geometry_type = gdf.geometry.type.iloc[0]
        else:
            geometry_type = "None"
            
        # feature_count
        feature_count = len(gdf)
        
        # sample_x and sample_y
        sample_x = None
        sample_y = None
        if feature_count > 0:
            # Get the first geometry's representative point or centroid
            sample_geom = gdf.geometry.iloc[0]
            if sample_geom is not None:
                # Use centroid to ensure we get a point
                centroid = sample_geom.centroid
                sample_x = centroid.x
                sample_y = centroid.y
                
        # encoding_detected
        encoding_detected = detect_encoding(gdf)
        
        audit_results.append({
            'layer_name': layer_name,
            'declared_crs': declared_crs,
            'geometry_type': geometry_type,
            'feature_count': feature_count,
            'sample_x': sample_x,
            'sample_y': sample_y,
            'encoding_detected': encoding_detected
        })

    return pd.DataFrame(audit_results)

if __name__ == "__main__":
    file_path = 'bangkok_contractor_delivery.gpkg'
    df = audit_layers(file_path)
    df.to_csv('crs_audit.csv', index=False)
    print("Audit complete. Results written to crs_audit.csv")

import geopandas as gpd
import fiona
import pandas as pd

def detect_encoding(gdf):
    str_cols = gdf.select_dtypes(include=['object']).columns
    if len(str_cols) == 0:
        return "utf-8"
    
    for col in str_cols:
        samples = gdf[col].dropna().unique()
        for s in samples:
            if not isinstance(s, str):
                continue
            # Common mojibake sequences for latin1 interpreted as utf-8
            # e.g. 'Ã' followed by other symbols
            if any(char in s for char in ["Ã", "Â", "©", "®", "´", "ì", "í", "î", "ï"]):
                # A very basic heuristic as requested
                # Let's check if we see the characteristic à patterns
                import re
                if re.search(r'[ÃÂ][\w\d]', s):
                    return "latin1-mojibake"
    return "utf-8"

def audit_layers(file_path):
    layers = fiona.listlayers(file_path)
    audit_results = []

    for layer in layers:
        gdf = gpd.read_file(file_path, layer=layer)
        
        layer_name = layer
        
        crs = gdf.crs
        if crs is not None:
            epsg_code = crs.to_epsg()
            if epsg_code:
                declared_crs = f"EPSG:{epsg_code}"
            else:
                declared_crs = str(crs)
        else:
            declared_crs = "None"
            
        if not gdf.empty:
            # Using the first geometry directly as requested ("sample coordinate taken from the layer as it sits on disk")
            # We'll take the first feature's centroid to ensure we have a point (x, y) 
            # even if the geometry is a Polygon or LineString
            sample_geom = gdf.geometry.iloc[0]
            if sample_geom is not None:
                # Get a point from the geometry. For polygons, centroid is a safe representative.
                # For lines, centroid is also a point.
                # Using centroid to represent the "coordinate" of the feature.
                point = sample_geom.centroid
                sample_x = point.x
                sample_y = point.y
            else:
                sample_x = None
                sample_y = None
            geometry_type = gdf.geometry.type.iloc[0]
        else:
            geometry_type = "None"
            sample_x = None
            sample_y = None
            
        feature_count = len(gdf)
        encoding_detected = detect_encoding(gdf)
        
        audit_results.append({
            'layer_name': layer_name,
            'declared_crs': declared_crs,
            'geometry_type': geometry_type,
            'feature_count': feature_count,
            'sample_x': sample_x,
            'sample_y': sample_y,
            'encoding_detected': encoding_detected
        })

    return pd.DataFrame(audit_results)

if __name__ == "__main__":
    file_path = 'bangkok_contractor_delivery.gpkg'
    df = audit_layers(file_path)
    df.to_csv('crs_audit.csv', index=False)
    print("Audit complete. Results written to crs_audit.csv")

import geopandas as gpd
import fiona
import pandas as pd
import re

def detect_encoding(gdf):
    """
    Detects if text columns contain mojibake.
    In the provided sample, 'แปลงที่ดิน' is a clear sign of 
    UTF-8 bytes being interpreted as Latin-1 (ISO-8859-1).
    """
    str_cols = gdf.select_dtypes(include=['object']).columns
    if len(str_cols) == 0:
        return "utf-8"
    
    for col in str_cols:
        samples = gdf[col].dropna().unique()
        for s in samples:
            if not isinstance(s, str):
                continue
            
            # Check for mojibake patterns: 
            # A high frequency of characters like 'à', '¹', '¸', etc. 
            # which occur when UTF-8 is read as Latin-1.
            # Or specifically looking for the 'à' + 'special char' pattern.
            mojibake_pattern = r'[àâäãåæ]'
            if re.search(mojibake_pattern, s):
                return "latin1-mojibake"
                
    return "utf-8"

def audit_layers(file_path):
    layers = fiona.listlayers(file_path)
    audit_results = []

    for layer in layers:
        gdf = gpd.read_file(file_path, layer=layer)
        
        layer_name = layer
        
        crs = gdf.crs
        if crs is not None:
            epsg_code = crs.to_epsg()
            if epsg_code:
                declared_crs = f"EPSG:{epsg_code}"
            else:
                declared_crs = str(crs)
        else:
            declared_crs = "None"
            
        if not gdf.empty:
            # Use the first geometry's centroid for a representative sample coordinate
            sample_geom = gdf.geometry.iloc[0]
            if sample_geom is not None:
                point = sample_geom.centroid
                sample_x = point.x
                sample_y = point.y
            else:
                sample_x = None
                sample_y = None
            geometry_type = gdf.geometry.type.iloc[0]
            feature_count = len(gdf)
        else:
            geometry_type = "None"
            sample_x = None
            sample_y = None
            feature_count = 0
            
        encoding_detected = detect_encoding(gdf)
        
        audit_results.append({
            'layer_name': layer_name,
            'declared_crs': declared_crs,
            'geometry_type': geometry_type,
            'feature_count': feature_count,
            'sample_x': sample_x,
            'sample_y': sample_y,
            'encoding_detected': encoding_detected
        })

    return pd.DataFrame(audit_results)

if __name__ == "__main__":
    file_path = 'bangkok_contractor_delivery.gpkg'
    df = audit_layers(file_path)
    df.to_csv('crs_audit.csv', index=False)
    print("Audit complete. Results written to crs_audit.csv")

import geopandas as gpd
import fiona
import pandas as pd
import re

def detect_encoding(gdf):
    """
    Detects if text columns contain mojibake.
    Mojibake (UTF-8 interpreted as Latin-1) often manifests as sequences 
    of characters like 'à', '¹', '¸', etc.
    """
    str_cols = gdf.select_dtypes(include=['object']).columns
    if len(str_cols) == 0:
        return "utf-8"
    
    for col in str_cols:
        # We check all non-null values in the column
        samples = gdf[col].dropna().unique()
        for s in samples:
            if not isinstance(s, str):
                continue
            
            # Heuristic: Many UTF-8 characters in Thai/other languages 
            # start with 'à' (0xC3) or 'â' (0xC2) when read as Latin-1.
            # We check for common mojibake marker characters.
            if re.search(r'[àâäãåæ]', s):
                return "latin1-mojibake"
                
    return "utf-8"

def audit_layers(file_path):
    layers = fiona.listlayers(file_path)
    audit_results = []

    for layer in layers:
        gdf = gpd.read_file(file_path, layer=layer)
        
        layer_name = layer
        
        # CRS handling
        crs = gdf.crs
        if crs is not None:
            epsg_code = crs.to_epsg()
            if epsg_code:
                declared_crs = f"EPSG:{epsg_code}"
            else:
                declared_crs = str(crs)
        else:
            declared_crs = "None"
            
        # Geometry and features
        if not gdf.empty:
            # Using the first feature's centroid as the sample coordinate
            sample_geom = gdf.geometry.iloc[0]
            if sample_geom is not None:
                point = sample_geom.centroid
                sample_x = point.x
                sample_y = point.y
            else:
                sample_x = None
                sample_y = None
            geometry_type = gdf.geometry.type.iloc[0]
            feature_count = len(gdf)
        else:
            geometry_type = "None"
            sample_x = None
            sample_y = None
            feature_count = 0
            
        # Encoding detection
        encoding_detected = detect_encoding(gdf)
        
        audit_results.append({
            'layer_name': layer_name,
            'declared_crs': declared_crs,
            'geometry_type': geometry_type,
            'feature_count': feature_count,
            'sample_x': sample_x,
            'sample_y': sample_y,
            'encoding_detected': encoding_detected
        })

    return pd.DataFrame(audit_results)

if __name__ == "__main__":
    file_path = 'bangkok_contractor_delivery.gpkg'
    df = audit_layers(file_path)
    df.to_csv('crs_audit.csv', index=False)
    print("Audit complete. Results written to crs_audit.csv")