The six interop hazards

# /// script
# requires-python = ">=3.10"
# dependencies = ["numpy"]
# ///
# Copyright (c) 2026 Aayush Sharma · SPDX-License-Identifier: MIT
# https://thedataengineering.com/lab-code/LICENSE.txt
"""The six interop hazards.

One small dataset read back with a different assumption per hazard: UTF-8 as
Latin-1, decimal cents as float, null against "" against missing, a time with
no zone under two zones, a big integer into a 32-bit column, and names sorted
by bytes against a case-insensitive collation. No reader raises an error.
"""
import argparse, json
from datetime import datetime
from decimal import Decimal
from zoneinfo import ZoneInfo
import numpy as np

p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
p.add_argument("--values", type=int, default=1_000_000, help="Cent amounts summed.")
p.add_argument("--json", help="Also write the results to this JSON file.")
args = p.parse_args()
rows = []
def report(hazard, written, read, raised=False):
    rows.append({"hazard": hazard, "written": str(written), "read": str(read), "differs": "yes" if str(written) != str(read) else "no",
                 "raised_error": "yes" if raised else "no"})
    print(f"{hazard:<28} wrote {str(written)[:34]:<36} read {str(read)[:34]:<36}")

name = "Zoë Müller"; report("encoding", name, name.encode("utf-8").decode("latin-1"))
cents = [Decimal("0.10")] * args.values
total = 0.0
for c in cents: total += float(c)                     # a plain running total, as most aggregation code keeps
report("decimal read as float", f"{sum(cents):.10f}", f"{total:.10f}")       # (Python 3.12+'s sum() compensates and would hide it)
records = [{"note": None}, {"note": ""}, {}]
report("null, empty, missing", "1 null, 1 empty, 1 missing", f"{sum(1 for r in records if not r.get('note'))} 'no note' (all three the same)")
naive = datetime(2025, 3, 30, 1, 30)
a = naive.replace(tzinfo=ZoneInfo("Europe/London")).astimezone(ZoneInfo("UTC")); b = naive.replace(tzinfo=ZoneInfo("Asia/Kolkata")).astimezone(ZoneInfo("UTC"))
report("time without a zone", naive.isoformat(), f"{a:%H:%M} UTC or {b:%H:%M} UTC")
report("integer into int32", 2_147_483_648, int(np.array([2_147_483_648]).astype(np.int32)[0]))
names = ["apple", "Banana", "cherry", "Apple"]
report("collation", ", ".join(sorted(names, key=str.lower)), ", ".join(sorted(names, key=lambda s: s.encode())))
print(f"{sum(r['differs'] == 'yes' for r in rows)} of {len(rows)} hazards changed the data; exceptions raised: {sum(r['raised_error'] == 'yes' for r in rows)}")

if args.json:
    with open(args.json, "w") as f:
        json.dump({"tables": {"hazards": rows, "summary": [{"differences": sum(r['differs'] == 'yes' for r in rows), "errors": 0}]}}, f)