dd-l2-bangkok-multicrs-audit

Rerun

import geopandas as gpd
import fiona
import sys

layers = fiona.listlayers('bangkok_contractor_delivery.gpkg')

for lyr in layers:
    gdf = gpd.read_file('bangkok_contractor_delivery.gpkg', layer=lyr)
    print(f'=== {lyr} ===')
    for col in gdf.columns:
        if gdf[col].dtype == 'object':
            val = gdf[col].iloc[0]
            print(f'  {col}: {val!r}')
            # Try to interpret as latin1 mojibake
            try:
                # If it looks like mojibake (e.g. à¹...) try decoding as latin1 then re-encode
                raw_bytes = val.encode('latin-1')
                decoded = raw_bytes.decode('utf-8', errors='replace')
                print(f'    latin1->utf8: {decoded!r}')
            except:
                pass
    print()

import geopandas as gpd
import fiona
import sys

layers = fiona.listlayers('bangkok_contractor_delivery.gpkg')

with open('/work/encoding_check.txt', 'w', encoding='utf-8') as f:
    for lyr in layers:
        gdf = gpd.read_file('bangkok_contractor_delivery.gpkg', layer=lyr)
        f.write(f'=== {lyr} ===\n')
        for col in gdf.columns:
            if gdf[col].dtype == 'object':
                val = gdf[col].iloc[0]
                f.write(f'  {col}: {val!r}\n')
                # Try to interpret as latin1 mojibake
                try:
                    raw_bytes = val.encode('latin-1')
                    decoded = raw_bytes.decode('utf-8', errors='replace')
                    f.write(f'    latin1->utf8: {decoded!r}\n')
                except:
                    pass
        f.write('\n')

import fiona
import csv
import math

layers = fiona.listlayers('bangkok_contractor_delivery.gpkg')

rows = []

for lyr in layers:
    with fiona.open('bangkok_contractor_delivery.gpkg', layer=lyr) as src:
        # Get CRS
        crs = src.crs
        if crs is None:
            declared_crs = 'EPSG:0'
        else:
            # Extract EPSG code
            epsg = crs.to_epsg()
            if epsg:
                declared_crs = f'EPSG:{epsg}'
            else:
                # Try to extract from dict
                if 'init' in crs:
                    declared_crs = str(crs['init']).upper()
                elif 'epsg' in str(crs).lower():
                    # Try to extract number
                    import re
                    m = re.search(r'(\d+)', str(crs))
                    if m:
                        declared_crs = f'EPSG:{m.group(1)}'
                    else:
                        declared_crs = str(crs)
                else:
                    declared_crs = str(crs)

        # Get geometry type and count
        geom_type = None
        feature_count = 0
        sample_geom = None

        for i, feat in enumerate(src):
            if i == 0:
                geom = feat['geometry']
                if geom and geom['type']:
                    geom_type = geom['type']
                    # Get sample coordinate
                    coords = geom['coordinates']
                    if geom['type'] == 'Point':
                        sample_geom = coords
                    elif geom['type'] in ('MultiPoint',):
                        sample_geom = coords[0]
                    elif geom['type'] in ('LineString',):
                        sample_geom = coords[0]
                    elif geom['type'] in ('MultiLineString',):
                        sample_geom = coords[0][0]
                    elif geom['type'] in ('Polygon',):
                        sample_geom = coords[0][0]
                    elif geom['type'] in ('MultiPolygon',):
                        sample_geom = coords[0][0][0]
            feature_count = i + 1

        # Determine sample_x and sample_y
        if sample_geom:
            sample_x = sample_geom[0]
            sample_y = sample_geom[1]
        else:
            sample_x = None
            sample_y = None

        # Check encoding: look at first text attribute
        encoding_detected = 'utf-8'
        # Re-read to check text
        with fiona.open('bangkok_contractor_delivery.gpkg', layer=lyr) as src2:
            first_feat = next(iter(src2))
            for key, val in first_feat['properties'].items():
                if val and isinstance(val, str) and len(val) > 1:
                    # Check for mojibake: try to encode as latin-1 then decode as utf-8
                    try:
                        raw_bytes = val.encode('latin-1')
                        # If it can be encoded as latin-1 and the result looks like valid UTF-8 sequences
                        # containing bytes 0xc0-0xff (common in Thai UTF-8), it's mojibake
                        high_bytes = sum(1 for b in raw_bytes if b > 0x7f)
                        if high_bytes > 0:
                            # Try to decode those bytes as UTF-8
                            try:
                                decoded = raw_bytes.decode('utf-8')
                                # If it decodes successfully, the original was UTF-8 bytes 
                                # mis-interpreted as latin-1
                                # Check if the decoded text contains Thai characters
                                has_thai = any(ord(c) > 0x0E00 and ord(c) < 0x10000 for c in decoded)
                                if has_thai:
                                    encoding_detected = 'latin1-mojibake'
                                    break
                            except (UnicodeDecodeError, UnicodeEncodeError):
                                pass
                    except (UnicodeEncodeError, UnicodeDecodeError):
                        pass
                    break

        rows.append({
            'layer_name': lyr,
            'declared_crs': declared_crs,
            'geometry_type': geom_type,
            'feature_count': feature_count,
            'sample_x': sample_x,
            'sample_y': sample_y,
            'encoding_detected': encoding_detected
        })

# Write CSV
with open('/work/crs_audit.csv', 'w', newline='', encoding='utf-8') as f:
    writer = csv.DictWriter(f, fieldnames=['layer_name', 'declared_crs', 'geometry_type', 'feature_count', 'sample_x', 'sample_y', 'encoding_detected'])
    writer.writeheader()
    writer.writerows(rows)

print('CSV written successfully')
for r in rows:
    print(r)

import fiona

with fiona.open('bangkok_contractor_delivery.gpkg', layer='parcels') as src:
    feat = next(iter(src))
    val = feat['properties']['name_th']
    print(f'val: {val!r}')
    print(f'len: {len(val)}')
    for i, ch in enumerate(val[:20]):
        print(f'  char[{i}]: U+{ord(ch):04X} {ch!r}')
    # Encode as latin-1 to get the raw bytes
    raw = val.encode('latin-1')
    print(f'raw bytes: {raw.hex()}')
    print(f'raw bytes repr: {list(raw)[:30]}')
    # Decode as UTF-8
    try:
        decoded = raw.decode('utf-8')
        print(f'decoded as utf-8: {decoded!r}')
    except Exception as e:
        print(f'Cannot decode as utf-8: {e}')
    
print()

with fiona.open('bangkok_contractor_delivery.gpkg', layer='markets') as src:
    feat = next(iter(src))
    val = feat['properties']['name_th']
    print(f'val: {val!r}')
    for i, ch in enumerate(val[:10]):
        print(f'  char[{i}]: U+{ord(ch):04X} {ch!r}')
    raw = val.encode('latin-1')
    print(f'raw bytes: {raw.hex()}')
    try:
        decoded = raw.decode('utf-8')
        print(f'decoded as utf-8: {decoded!r}')
    except Exception as e:
        print(f'Cannot decode as utf-8: {e}')

import fiona
import csv

layers = fiona.listlayers('bangkok_contractor_delivery.gpkg')

rows = []

for lyr in layers:
    with fiona.open('bangkok_contractor_delivery.gpkg', layer=lyr) as src:
        # Get CRS
        crs = src.crs
        if crs is None:
            declared_crs = 'EPSG:0'
        else:
            epsg = crs.to_epsg()
            if epsg:
                declared_crs = f'EPSG:{epsg}'
            else:
                # Fallback: extract from string
                import re
                m = re.search(r'(\d+)', str(crs))
                declared_crs = f'EPSG:{m.group(1)}' if m else str(crs)

        # Get geometry type, feature count, and sample coordinates
        geom_type = None
        feature_count = 0
        sample_x = None
        sample_y = None

        for i, feat in enumerate(src):
            if i == 0:
                geom = feat['geometry']
                if geom and geom.get('type'):
                    geom_type = geom['type']
                    coords = geom['coordinates']
                    if geom_type == 'Point':
                        sample_x, sample_y = coords[0], coords[1]
                    elif geom_type == 'MultiPoint':
                        sample_x, sample_y = coords[0][0], coords[0][1]
                    elif geom_type == 'LineString':
                        sample_x, sample_y = coords[0][0], coords[0][1]
                    elif geom_type == 'MultiLineString':
                        sample_x, sample_y = coords[0][0][0], coords[0][0][1]
                    elif geom_type == 'Polygon':
                        sample_x, sample_y = coords[0][0][0], coords[0][0][1]
                    elif geom_type == 'MultiPolygon':
                        sample_x, sample_y = coords[0][0][0][0], coords[0][0][0][1]
            feature_count = i + 1

        # Detect encoding from first text attribute
        encoding_detected = 'utf-8'
        with fiona.open('bangkok_contractor_delivery.gpkg', layer=lyr) as src2:
            first_feat = next(iter(src2))
            for key, val in first_feat['properties'].items():
                if val and isinstance(val, str) and len(val) > 0:
                    # Check if this is latin1-mojibake:
                    # The string contains only chars in range U+0000-U+00FF (latin-1 encodable)
                    # AND when encoded as latin-1 the resulting bytes decode as valid UTF-8
                    try:
                        raw_bytes = val.encode('latin-1')
                        # If we got here, all chars are in latin-1 range
                        # Now try to decode those bytes as UTF-8
                        try:
                            decoded = raw_bytes.decode('utf-8')
                            # Check if decoded text contains non-ASCII characters (e.g. Thai)
                            has_high = any(ord(c) > 0x7f for c in decoded)
                            if has_high:
                                encoding_detected = 'latin1-mojibake'
                                break
                        except UnicodeDecodeError:
                            pass
                    except UnicodeEncodeError:
                        # Contains characters outside latin-1 range -> proper Unicode
                        pass
                    break  # Only check first text attribute

        rows.append({
            'layer_name': lyr,
            'declared_crs': declared_crs,
            'geometry_type': geom_type,
            'feature_count': feature_count,
            'sample_x': sample_x,
            'sample_y': sample_y,
            'encoding_detected': encoding_detected
        })

# Write CSV
with open('/work/crs_audit.csv', 'w', newline='', encoding='utf-8') as f:
    writer = csv.DictWriter(f, fieldnames=[
        'layer_name', 'declared_crs', 'geometry_type',
        'feature_count', 'sample_x', 'sample_y', 'encoding_detected'
    ])
    writer.writeheader()
    writer.writerows(rows)

print('crs_audit.csv written successfully')
for r in rows:
    print(r)

import fiona

with fiona.open('bangkok_contractor_delivery.gpkg', layer='parcels') as src:
    first_feat = next(iter(src))
    for key, val in first_feat['properties'].items():
        if val and isinstance(val, str) and len(val) > 0:
            print(f'key={key}, val={val!r}')
            try:
                raw_bytes = val.encode('latin-1')
                print(f'  encoded to latin-1 OK, bytes={raw_bytes[:20]}')
                try:
                    decoded = raw_bytes.decode('utf-8')
                    print(f'  decoded as utf-8: {decoded!r}')
                    has_high = any(ord(c) > 0x7f for c in decoded)
                    print(f'  has_high={has_high}')
                except UnicodeDecodeError as e:
                    print(f'  utf-8 decode failed: {e}')
            except UnicodeEncodeError as e:
                print(f'  latin-1 encode failed: {e}')
            break

import fiona

# Check which columns have Thai/mojibake content
for lyr in ['parcels', 'roads', 'markets']:
    with fiona.open('bangkok_contractor_delivery.gpkg', layer=lyr) as src:
        first_feat = next(iter(src))
        print(f'=== {lyr} ===')
        for key, val in first_feat['properties'].items():
            print(f'  {key}: {val!r}')
            if val and isinstance(val, str) and len(val) > 0:
                try:
                    raw = val.encode('latin-1')
                    try:
                        decoded = raw.decode('utf-8')
                        has_high = any(ord(c) > 0x7f for c in decoded)
                        print(f'    -> latin1->utf8: {decoded!r} (has_high={has_high})')
                    except:
                        print(f'    -> latin1 bytes but not valid utf-8')
                except:
                    print(f'    -> not latin-1 encodable (proper unicode)')
        print()