From 480b896d395c8863245219b4fbb621b93507efa5 Mon Sep 17 00:00:00 2001 From: ben Date: Tue, 15 Sep 2026 22:48:34 +1000 Subject: [PATCH] convert track timestamps to airport-local; capture per-leg UTC offsets --- aerodatabox.py | 36 ++++++ boarding-pass-local-time-fix.patch | 181 +++++++++++++++++++++++++++++ db.py | 4 + scheduler.py | 66 +++++++++-- templates/review_detail.html | 3 + 5 files changed, 278 insertions(+), 12 deletions(-) create mode 100644 boarding-pass-local-time-fix.patch diff --git a/aerodatabox.py b/aerodatabox.py index e60f4ce..5764b82 100644 --- a/aerodatabox.py +++ b/aerodatabox.py @@ -71,6 +71,36 @@ def _sanitize(flight_number: str) -> str: return re.sub(r"[\s-]", "", flight_number).upper() +def _utc_offset(local: str | None) -> str | None: + """Pull the UTC offset out of an AeroDataBox 'local' string + ('2026-06-22 13:45+10:00' -> '+10:00'). + + This is the airport's *actual* offset on the day of the flight (so DST + is already accounted for), and it's the only timezone information this + pipeline has - the bundled airports.csv carries IATA/ICAO/name only. + scheduler.py needs it to convert the FlightAware track's UTC timestamps + into the airport-local wall clock AirTrail expects.""" + if not local: + return None + try: + dt = datetime.strptime(local, "%Y-%m-%d %H:%M%z") + except ValueError: + return None + offset = dt.strftime("%z") # '+1000' + return f"{offset[:3]}:{offset[3:]}" if len(offset) == 5 else None + + +def _first_offset(leg: dict) -> str | None: + """The UTC offset for one leg, from whichever of its timestamps exists. + All of a leg's times share the same airport and day, so any of them + gives the same offset.""" + for key in ("scheduledTime", "revisedTime", "actualTime", "runwayTime"): + offset = _utc_offset((leg.get(key) or {}).get("local")) + if offset: + return offset + return None + + def _parse_local(local: str | None) -> str | None: """AeroDataBox 'local' strings look like '2026-06-22 13:45+10:00' - already carrying their own UTC offset, no separate airport-timezone @@ -155,4 +185,10 @@ def lookup_gate_times(flight_number: str, flight_date: date) -> dict | None: ), "takeoffActual": _parse_local((dep.get("runwayTime") or {}).get("local")), "landingActual": _parse_local((arr.get("runwayTime") or {}).get("local")), + # Each leg's real UTC offset on the day, taken from whichever local + # timestamp that leg actually has. Not sent to AirTrail directly - + # scheduler.py uses these to put the FlightAware track's UTC times + # into the same airport-local terms as everything else here. + "departureUtcOffset": _first_offset(dep), + "arrivalUtcOffset": _first_offset(arr), } diff --git a/boarding-pass-local-time-fix.patch b/boarding-pass-local-time-fix.patch new file mode 100644 index 0000000..ddbeff4 --- /dev/null +++ b/boarding-pass-local-time-fix.patch @@ -0,0 +1,181 @@ +diff --git a/aerodatabox.py b/aerodatabox.py +index e60f4ce..5764b82 100644 +--- a/aerodatabox.py ++++ b/aerodatabox.py +@@ -71,6 +71,36 @@ def _sanitize(flight_number: str) -> str: + return re.sub(r"[\s-]", "", flight_number).upper() + + ++def _utc_offset(local: str | None) -> str | None: ++ """Pull the UTC offset out of an AeroDataBox 'local' string ++ ('2026-06-22 13:45+10:00' -> '+10:00'). ++ ++ This is the airport's *actual* offset on the day of the flight (so DST ++ is already accounted for), and it's the only timezone information this ++ pipeline has - the bundled airports.csv carries IATA/ICAO/name only. ++ scheduler.py needs it to convert the FlightAware track's UTC timestamps ++ into the airport-local wall clock AirTrail expects.""" ++ if not local: ++ return None ++ try: ++ dt = datetime.strptime(local, "%Y-%m-%d %H:%M%z") ++ except ValueError: ++ return None ++ offset = dt.strftime("%z") # '+1000' ++ return f"{offset[:3]}:{offset[3:]}" if len(offset) == 5 else None ++ ++ ++def _first_offset(leg: dict) -> str | None: ++ """The UTC offset for one leg, from whichever of its timestamps exists. ++ All of a leg's times share the same airport and day, so any of them ++ gives the same offset.""" ++ for key in ("scheduledTime", "revisedTime", "actualTime", "runwayTime"): ++ offset = _utc_offset((leg.get(key) or {}).get("local")) ++ if offset: ++ return offset ++ return None ++ ++ + def _parse_local(local: str | None) -> str | None: + """AeroDataBox 'local' strings look like '2026-06-22 13:45+10:00' - + already carrying their own UTC offset, no separate airport-timezone +@@ -155,4 +185,10 @@ def lookup_gate_times(flight_number: str, flight_date: date) -> dict | None: + ), + "takeoffActual": _parse_local((dep.get("runwayTime") or {}).get("local")), + "landingActual": _parse_local((arr.get("runwayTime") or {}).get("local")), ++ # Each leg's real UTC offset on the day, taken from whichever local ++ # timestamp that leg actually has. Not sent to AirTrail directly - ++ # scheduler.py uses these to put the FlightAware track's UTC times ++ # into the same airport-local terms as everything else here. ++ "departureUtcOffset": _first_offset(dep), ++ "arrivalUtcOffset": _first_offset(arr), + } +diff --git a/db.py b/db.py +index 5e4a2cc..7a5e551 100644 +--- a/db.py ++++ b/db.py +@@ -39,6 +39,8 @@ CREATE TABLE IF NOT EXISTS pending_flights ( + gate_departure_actual TEXT, + gate_arrival_scheduled TEXT, + gate_arrival_actual TEXT, ++ departure_utc_offset TEXT, ++ arrival_utc_offset TEXT, + schedule_error TEXT, + airtrail_flight_id INTEGER, + parse_warnings TEXT, +@@ -86,6 +88,8 @@ MIGRATIONS = [ + ("gate_departure_actual", "TEXT"), + ("gate_arrival_scheduled", "TEXT"), + ("gate_arrival_actual", "TEXT"), ++ ("departure_utc_offset", "TEXT"), ++ ("arrival_utc_offset", "TEXT"), + ("schedule_error", "TEXT"), + ] + +diff --git a/scheduler.py b/scheduler.py +index cb77644..4f3cb08 100644 +--- a/scheduler.py ++++ b/scheduler.py +@@ -35,8 +35,40 @@ def next_run_datetime(hhmm: str, now: datetime | None = None) -> datetime: + return candidate + + +-def _iso_from_unix(ts: int) -> str: +- return datetime.fromtimestamp(ts, tz=timezone.utc).isoformat(timespec="seconds") ++def _tz_from_offset(offset: str | None): ++ """'+08:00' -> a datetime.timezone. None if missing/unparseable.""" ++ if not offset: ++ return None ++ try: ++ return datetime.strptime( ++ f"2000-01-01T00:00:00{offset}", "%Y-%m-%dT%H:%M:%S%z" ++ ).tzinfo ++ except ValueError: ++ return None ++ ++ ++def _iso_local_from_unix(ts: int, offset: str | None) -> str | None: ++ """A FlightAware track timestamp (UTC epoch seconds) as an ISO string in ++ the *airport's* local wall clock. ++ ++ This conversion is not cosmetic. AirTrail reads only the literal Y-M-D ++ out of a datetime field and the HH:MM out of its companion *Time field, ++ then interprets that pair in the airport's own timezone ++ (mergeTimeWithDate - confirmed in AirTrail's source). A UTC timestamp ++ sent as-is is therefore silently stored wrong by the whole offset, and ++ when takeoff comes from AeroDataBox (already airport-local) while ++ landing falls back to this track (UTC), the mismatch can invert their ++ order and get the save rejected outright with "Actual landing must be ++ after actual takeoff" - which is exactly what happened on VA556. ++ ++ With no offset known (AeroDataBox unavailable, and airports.csv carries ++ no timezone data) this returns None so the caller omits the field ++ entirely - a missing takeoff time is recoverable, a wrong one is not. ++ """ ++ tzinfo = _tz_from_offset(offset) ++ if tzinfo is None: ++ return None ++ return datetime.fromtimestamp(ts, tz=tzinfo).isoformat(timespec="seconds") + + + def _fetch_gate_times(flight_number: str, flight_date: date, updates: dict) -> None: +@@ -60,14 +92,18 @@ def _fetch_gate_times(flight_number: str, flight_date: date, updates: dict) -> N + updates["gate_departure_actual"] = gate_times.get("departure") + updates["gate_arrival_scheduled"] = gate_times.get("arrivalScheduled") + updates["gate_arrival_actual"] = gate_times.get("arrival") ++ # Each leg's UTC offset on the day - what lets the FlightAware track's ++ # UTC timestamps be expressed in the same airport-local terms as ++ # everything else (see _iso_local_from_unix). ++ updates["departure_utc_offset"] = gate_times.get("departureUtcOffset") ++ updates["arrival_utc_offset"] = gate_times.get("arrivalUtcOffset") + # AeroDataBox's runwayTime is an authoritative actual takeoff/landing +- # time - prefer it. Only set here if present; the FlightAware-track +- # fallback below fills in behind it (checks "not in updates") rather +- # than overwriting it unconditionally. +- if gate_times.get("takeoffActual"): +- updates["takeoff_actual"] = gate_times["takeoffActual"] +- if gate_times.get("landingActual"): +- updates["landing_actual"] = gate_times["landingActual"] ++ # time - prefer it. Assigned unconditionally (None included) so a ++ # re-query always rewrites these rather than leaving a stale value ++ # behind; the FlightAware-track fallback below fills in only where ++ # AeroDataBox came up empty. ++ updates["takeoff_actual"] = gate_times.get("takeoffActual") ++ updates["landing_actual"] = gate_times.get("landingActual") + + + def process_one_flight(row: dict) -> None: +@@ -151,10 +187,16 @@ def process_one_flight(row: dict) -> None: + # already give us its authoritative runwayTime above (no key + # configured, lookup failed, or that particular flight had no + # runwayTime in its response). Never overwrite a value AeroDataBox +- # already set - that one's more precise. ++ # already set - that one's more precise - and convert into the ++ # departure/arrival airport's local wall clock, since that's the ++ # only form AirTrail interprets correctly. + if times: +- updates.setdefault("takeoff_actual", _iso_from_unix(times[0])) +- updates.setdefault("landing_actual", _iso_from_unix(times[-1])) ++ dep_offset = updates.get("departure_utc_offset") or row.get("departure_utc_offset") ++ arr_offset = updates.get("arrival_utc_offset") or row.get("arrival_utc_offset") ++ if not updates.get("takeoff_actual"): ++ updates["takeoff_actual"] = _iso_local_from_unix(times[0], dep_offset) ++ if not updates.get("landing_actual"): ++ updates["landing_actual"] = _iso_local_from_unix(times[-1], arr_offset) + + models.update_flight(flight_id, **updates) + notifier.notify_flight_staged(models.get_flight(flight_id)) +diff --git a/templates/review_detail.html b/templates/review_detail.html +index b21e042..b2d7202 100644 +--- a/templates/review_detail.html ++++ b/templates/review_detail.html +@@ -73,6 +73,9 @@ + {% if flight.landing_actual %} +
Landing (actual)
{{ flight.landing_actual }}
+ {% endif %} ++ {% if flight.departure_utc_offset or flight.arrival_utc_offset %} ++
Airport UTC offsets
{{ flight.departure_utc_offset or '—' }} → {{ flight.arrival_utc_offset or '—' }}
++ {% endif %} + {% if flight.schedule_error %} +
Schedule lookup
{{ flight.schedule_error }}
+ {% endif %} diff --git a/db.py b/db.py index 5e4a2cc..7a5e551 100644 --- a/db.py +++ b/db.py @@ -39,6 +39,8 @@ CREATE TABLE IF NOT EXISTS pending_flights ( gate_departure_actual TEXT, gate_arrival_scheduled TEXT, gate_arrival_actual TEXT, + departure_utc_offset TEXT, + arrival_utc_offset TEXT, schedule_error TEXT, airtrail_flight_id INTEGER, parse_warnings TEXT, @@ -86,6 +88,8 @@ MIGRATIONS = [ ("gate_departure_actual", "TEXT"), ("gate_arrival_scheduled", "TEXT"), ("gate_arrival_actual", "TEXT"), + ("departure_utc_offset", "TEXT"), + ("arrival_utc_offset", "TEXT"), ("schedule_error", "TEXT"), ] diff --git a/scheduler.py b/scheduler.py index cb77644..4f3cb08 100644 --- a/scheduler.py +++ b/scheduler.py @@ -35,8 +35,40 @@ def next_run_datetime(hhmm: str, now: datetime | None = None) -> datetime: return candidate -def _iso_from_unix(ts: int) -> str: - return datetime.fromtimestamp(ts, tz=timezone.utc).isoformat(timespec="seconds") +def _tz_from_offset(offset: str | None): + """'+08:00' -> a datetime.timezone. None if missing/unparseable.""" + if not offset: + return None + try: + return datetime.strptime( + f"2000-01-01T00:00:00{offset}", "%Y-%m-%dT%H:%M:%S%z" + ).tzinfo + except ValueError: + return None + + +def _iso_local_from_unix(ts: int, offset: str | None) -> str | None: + """A FlightAware track timestamp (UTC epoch seconds) as an ISO string in + the *airport's* local wall clock. + + This conversion is not cosmetic. AirTrail reads only the literal Y-M-D + out of a datetime field and the HH:MM out of its companion *Time field, + then interprets that pair in the airport's own timezone + (mergeTimeWithDate - confirmed in AirTrail's source). A UTC timestamp + sent as-is is therefore silently stored wrong by the whole offset, and + when takeoff comes from AeroDataBox (already airport-local) while + landing falls back to this track (UTC), the mismatch can invert their + order and get the save rejected outright with "Actual landing must be + after actual takeoff" - which is exactly what happened on VA556. + + With no offset known (AeroDataBox unavailable, and airports.csv carries + no timezone data) this returns None so the caller omits the field + entirely - a missing takeoff time is recoverable, a wrong one is not. + """ + tzinfo = _tz_from_offset(offset) + if tzinfo is None: + return None + return datetime.fromtimestamp(ts, tz=tzinfo).isoformat(timespec="seconds") def _fetch_gate_times(flight_number: str, flight_date: date, updates: dict) -> None: @@ -60,14 +92,18 @@ def _fetch_gate_times(flight_number: str, flight_date: date, updates: dict) -> N updates["gate_departure_actual"] = gate_times.get("departure") updates["gate_arrival_scheduled"] = gate_times.get("arrivalScheduled") updates["gate_arrival_actual"] = gate_times.get("arrival") + # Each leg's UTC offset on the day - what lets the FlightAware track's + # UTC timestamps be expressed in the same airport-local terms as + # everything else (see _iso_local_from_unix). + updates["departure_utc_offset"] = gate_times.get("departureUtcOffset") + updates["arrival_utc_offset"] = gate_times.get("arrivalUtcOffset") # AeroDataBox's runwayTime is an authoritative actual takeoff/landing - # time - prefer it. Only set here if present; the FlightAware-track - # fallback below fills in behind it (checks "not in updates") rather - # than overwriting it unconditionally. - if gate_times.get("takeoffActual"): - updates["takeoff_actual"] = gate_times["takeoffActual"] - if gate_times.get("landingActual"): - updates["landing_actual"] = gate_times["landingActual"] + # time - prefer it. Assigned unconditionally (None included) so a + # re-query always rewrites these rather than leaving a stale value + # behind; the FlightAware-track fallback below fills in only where + # AeroDataBox came up empty. + updates["takeoff_actual"] = gate_times.get("takeoffActual") + updates["landing_actual"] = gate_times.get("landingActual") def process_one_flight(row: dict) -> None: @@ -151,10 +187,16 @@ def process_one_flight(row: dict) -> None: # already give us its authoritative runwayTime above (no key # configured, lookup failed, or that particular flight had no # runwayTime in its response). Never overwrite a value AeroDataBox - # already set - that one's more precise. + # already set - that one's more precise - and convert into the + # departure/arrival airport's local wall clock, since that's the + # only form AirTrail interprets correctly. if times: - updates.setdefault("takeoff_actual", _iso_from_unix(times[0])) - updates.setdefault("landing_actual", _iso_from_unix(times[-1])) + dep_offset = updates.get("departure_utc_offset") or row.get("departure_utc_offset") + arr_offset = updates.get("arrival_utc_offset") or row.get("arrival_utc_offset") + if not updates.get("takeoff_actual"): + updates["takeoff_actual"] = _iso_local_from_unix(times[0], dep_offset) + if not updates.get("landing_actual"): + updates["landing_actual"] = _iso_local_from_unix(times[-1], arr_offset) models.update_flight(flight_id, **updates) notifier.notify_flight_staged(models.get_flight(flight_id)) diff --git a/templates/review_detail.html b/templates/review_detail.html index b21e042..b2d7202 100644 --- a/templates/review_detail.html +++ b/templates/review_detail.html @@ -73,6 +73,9 @@ {% if flight.landing_actual %}
Landing (actual)
{{ flight.landing_actual }}
{% endif %} + {% if flight.departure_utc_offset or flight.arrival_utc_offset %} +
Airport UTC offsets
{{ flight.departure_utc_offset or '—' }} → {{ flight.arrival_utc_offset or '—' }}
+ {% endif %} {% if flight.schedule_error %}
Schedule lookup
{{ flight.schedule_error }}
{% endif %}