diff --git a/aerodatabox.py b/aerodatabox.py index 5764b82..94accf3 100644 --- a/aerodatabox.py +++ b/aerodatabox.py @@ -43,6 +43,7 @@ wheels-up/touchdown). import json import re import urllib.error +import urllib.parse import urllib.request from datetime import date, datetime @@ -90,6 +91,64 @@ def _utc_offset(local: str | None) -> str | None: return f"{offset[:3]}:{offset[3:]}" if len(offset) == 5 else None +def _get(url: str): + """GET an AeroDataBox endpoint and return the decoded JSON, or None for + a "no content" answer. Raises AeroDataBoxError for anything retryable. + + A default urllib request (no User-Agent) gets HTTP 403 from AeroDataBox's + Cloudflare-fronted gateway even with a perfectly valid, subscribed key - + confirmed live, side by side, same key/URL: bare urllib -> 403, same + request + this UA -> 200 with real data. See config.BROWSER_USER_AGENT.""" + req = urllib.request.Request( + url, + headers={ + "x-rapidapi-key": config.AERODATABOX_API_KEY, + "User-Agent": config.BROWSER_USER_AGENT, + }, + ) + try: + with urllib.request.urlopen(req, timeout=20) as resp: + if resp.status == 204: + return None + raw = resp.read() + except urllib.error.HTTPError as e: + if e.code == 204: + return None + raise AeroDataBoxError(f"HTTP {e.code} from AeroDataBox") from e + except urllib.error.URLError as e: + raise AeroDataBoxError(f"error reaching AeroDataBox: {e}") from e + + if not raw: + return None + try: + return json.loads(raw.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError) as e: + raise AeroDataBoxError(f"bad AeroDataBox response: {e}") from e + + +def lookup_aircraft_icao(registration: str) -> str | None: + """Registration (e.g. 'VH-8VE') -> ICAO type code (e.g. 'B38M'). + + Costs one extra API unit per flight. Needed because AirTrail's + `aircraft` field is matched against the `icao` column of its own + aircraft table (getAircraftByIcao, confirmed in its source), so the + plain model string AeroDataBox puts on the flight record + ('Boeing 737 MAX 8') is not something AirTrail can resolve.""" + if not config.AERODATABOX_API_KEY or not registration: + return None + + data = _get(f"{BASE_URL}/aircrafts/reg/{urllib.parse.quote(registration)}") + if not isinstance(data, dict): + return None + # AirTrail tries model before icaoCode against the same column, so both + # are worth passing on in that order. + for key in ("icaoCode", "model"): + value = (data.get(key) or "").strip() + if value: + return value + return None + + def _first_offset(leg: dict) -> str | None: """The UTC offset for one leg, from whichever of its timestamps exists. All of a leg's times share the same airport and day, so any of them @@ -126,35 +185,10 @@ def lookup_gate_times(flight_number: str, flight_date: date) -> dict | None: return None cleaned = _sanitize(flight_number) - url = ( + data = _get( f"{BASE_URL}/flights/number/{cleaned}/{flight_date.isoformat()}" "?dateLocalRole=Both&withAircraftImage=false&withLocation=false" ) - # A default urllib request (no User-Agent) gets HTTP 403 from AeroDataBox's - # Cloudflare-fronted gateway even with a perfectly valid, subscribed key - - # confirmed live, side by side, same key/URL: bare urllib -> 403, same - # request + this UA -> 200 with real data. See config.BROWSER_USER_AGENT. - req = urllib.request.Request( - url, - headers={ - "x-rapidapi-key": config.AERODATABOX_API_KEY, - "User-Agent": config.BROWSER_USER_AGENT, - }, - ) - try: - with urllib.request.urlopen(req, timeout=20) as resp: - raw = resp.read() - except urllib.error.HTTPError as e: - if e.code == 204: - return None - raise AeroDataBoxError(f"HTTP {e.code} from AeroDataBox") from e - except urllib.error.URLError as e: - raise AeroDataBoxError(f"error reaching AeroDataBox: {e}") from e - - try: - data = json.loads(raw.decode("utf-8")) - except (json.JSONDecodeError, UnicodeDecodeError) as e: - raise AeroDataBoxError(f"bad AeroDataBox response: {e}") from e if not isinstance(data, list) or not data: return None @@ -191,4 +225,12 @@ def lookup_gate_times(flight_number: str, flight_date: date) -> dict | None: # into the same airport-local terms as everything else here. "departureUtcOffset": _first_offset(dep), "arrivalUtcOffset": _first_offset(arr), + # Straight off the flight record, no extra call. aircraftReg is a + # free-text field in AirTrail; the ICAO *type* code needs a separate + # registration lookup (see lookup_aircraft_icao). + "aircraftReg": ((flight.get("aircraft") or {}).get("reg") or "").strip() or None, + "departureTerminal": (dep.get("terminal") or "").strip() or None, + "departureGate": (dep.get("gate") or "").strip() or None, + "arrivalTerminal": (arr.get("terminal") or "").strip() or None, + "arrivalGate": (arr.get("gate") or "").strip() or None, } diff --git a/airtrail_client.py b/airtrail_client.py index afc19b8..aaf138a 100644 --- a/airtrail_client.py +++ b/airtrail_client.py @@ -131,8 +131,14 @@ def build_save_payload(row: dict, existing_id: int | None) -> dict: "datePrecision": "day", "flightNumber": flight_number or None, "airline": row.get("operating_carrier_icao"), - "aircraft": None, - "aircraftReg": None, + # ICAO *type* code (e.g. 'B38M'); AirTrail validates it against its + # own aircraft table and 500s the whole save if it doesn't know it, + # so save_flight() drops it and retries rather than losing the import. + "aircraft": row.get("aircraft_icao"), + "aircraftReg": row.get("aircraft_reg"), + # Seat position and flight reason are deliberately left unset - the + # BCBP carries neither and no API knows them, so they stay a manual + # choice in AirTrail rather than a guess made here. "flightReason": None, "note": f"Imported via boarding-pass-pipeline (PNR {row.get('pnr') or 'unknown'})", "seats": [ @@ -171,6 +177,15 @@ def build_save_payload(row: dict, existing_id: int | None) -> dict: payload["landingActual"] = row["landing_actual"] payload["landingActualTime"] = _hhmm(row["landing_actual"]) + for payload_key, row_key in ( + ("departureTerminal", "departure_terminal"), + ("departureGate", "departure_gate"), + ("arrivalTerminal", "arrival_terminal"), + ("arrivalGate", "arrival_gate"), + ): + if row.get(row_key): + payload[payload_key] = row[row_key] + coordinates = row.get("track_coordinates") if coordinates: times = row.get("track_times") @@ -192,7 +207,19 @@ def build_save_payload(row: dict, existing_id: int | None) -> dict: def save_flight(payload: dict) -> dict: - return _request("POST", "/api/flight/save", payload) + try: + return _request("POST", "/api/flight/save", payload) + except AirTrailError as e: + # AirTrail resolves `aircraft` against the `icao` column of its own + # aircraft table and fails the ENTIRE save if the type code isn't + # there (apiError('Invalid aircraft'), HTTP 500). The type is a + # nicety; the flight, its times and its track are not. Drop it and + # retry once rather than losing the import over an unknown variant. + if "Invalid aircraft" in str(e) and payload.get("aircraft"): + retry = dict(payload) + retry["aircraft"] = None + return _request("POST", "/api/flight/save", retry) + raise def approve_flight(flight_id: int) -> tuple[bool, str]: diff --git a/boarding-pass-aircraft-terminal-fix.patch b/boarding-pass-aircraft-terminal-fix.patch new file mode 100644 index 0000000..66d169d --- /dev/null +++ b/boarding-pass-aircraft-terminal-fix.patch @@ -0,0 +1,259 @@ +diff --git a/aerodatabox.py b/aerodatabox.py +index 5764b82..94accf3 100644 +--- a/aerodatabox.py ++++ b/aerodatabox.py +@@ -43,6 +43,7 @@ wheels-up/touchdown). + import json + import re + import urllib.error ++import urllib.parse + import urllib.request + from datetime import date, datetime + +@@ -90,6 +91,64 @@ def _utc_offset(local: str | None) -> str | None: + return f"{offset[:3]}:{offset[3:]}" if len(offset) == 5 else None + + ++def _get(url: str): ++ """GET an AeroDataBox endpoint and return the decoded JSON, or None for ++ a "no content" answer. Raises AeroDataBoxError for anything retryable. ++ ++ A default urllib request (no User-Agent) gets HTTP 403 from AeroDataBox's ++ Cloudflare-fronted gateway even with a perfectly valid, subscribed key - ++ confirmed live, side by side, same key/URL: bare urllib -> 403, same ++ request + this UA -> 200 with real data. See config.BROWSER_USER_AGENT.""" ++ req = urllib.request.Request( ++ url, ++ headers={ ++ "x-rapidapi-key": config.AERODATABOX_API_KEY, ++ "User-Agent": config.BROWSER_USER_AGENT, ++ }, ++ ) ++ try: ++ with urllib.request.urlopen(req, timeout=20) as resp: ++ if resp.status == 204: ++ return None ++ raw = resp.read() ++ except urllib.error.HTTPError as e: ++ if e.code == 204: ++ return None ++ raise AeroDataBoxError(f"HTTP {e.code} from AeroDataBox") from e ++ except urllib.error.URLError as e: ++ raise AeroDataBoxError(f"error reaching AeroDataBox: {e}") from e ++ ++ if not raw: ++ return None ++ try: ++ return json.loads(raw.decode("utf-8")) ++ except (json.JSONDecodeError, UnicodeDecodeError) as e: ++ raise AeroDataBoxError(f"bad AeroDataBox response: {e}") from e ++ ++ ++def lookup_aircraft_icao(registration: str) -> str | None: ++ """Registration (e.g. 'VH-8VE') -> ICAO type code (e.g. 'B38M'). ++ ++ Costs one extra API unit per flight. Needed because AirTrail's ++ `aircraft` field is matched against the `icao` column of its own ++ aircraft table (getAircraftByIcao, confirmed in its source), so the ++ plain model string AeroDataBox puts on the flight record ++ ('Boeing 737 MAX 8') is not something AirTrail can resolve.""" ++ if not config.AERODATABOX_API_KEY or not registration: ++ return None ++ ++ data = _get(f"{BASE_URL}/aircrafts/reg/{urllib.parse.quote(registration)}") ++ if not isinstance(data, dict): ++ return None ++ # AirTrail tries model before icaoCode against the same column, so both ++ # are worth passing on in that order. ++ for key in ("icaoCode", "model"): ++ value = (data.get(key) or "").strip() ++ if value: ++ return value ++ return None ++ ++ + def _first_offset(leg: dict) -> str | None: + """The UTC offset for one leg, from whichever of its timestamps exists. + All of a leg's times share the same airport and day, so any of them +@@ -126,35 +185,10 @@ def lookup_gate_times(flight_number: str, flight_date: date) -> dict | None: + return None + + cleaned = _sanitize(flight_number) +- url = ( ++ data = _get( + f"{BASE_URL}/flights/number/{cleaned}/{flight_date.isoformat()}" + "?dateLocalRole=Both&withAircraftImage=false&withLocation=false" + ) +- # A default urllib request (no User-Agent) gets HTTP 403 from AeroDataBox's +- # Cloudflare-fronted gateway even with a perfectly valid, subscribed key - +- # confirmed live, side by side, same key/URL: bare urllib -> 403, same +- # request + this UA -> 200 with real data. See config.BROWSER_USER_AGENT. +- req = urllib.request.Request( +- url, +- headers={ +- "x-rapidapi-key": config.AERODATABOX_API_KEY, +- "User-Agent": config.BROWSER_USER_AGENT, +- }, +- ) +- try: +- with urllib.request.urlopen(req, timeout=20) as resp: +- raw = resp.read() +- except urllib.error.HTTPError as e: +- if e.code == 204: +- return None +- raise AeroDataBoxError(f"HTTP {e.code} from AeroDataBox") from e +- except urllib.error.URLError as e: +- raise AeroDataBoxError(f"error reaching AeroDataBox: {e}") from e +- +- try: +- data = json.loads(raw.decode("utf-8")) +- except (json.JSONDecodeError, UnicodeDecodeError) as e: +- raise AeroDataBoxError(f"bad AeroDataBox response: {e}") from e + + if not isinstance(data, list) or not data: + return None +@@ -191,4 +225,12 @@ def lookup_gate_times(flight_number: str, flight_date: date) -> dict | None: + # into the same airport-local terms as everything else here. + "departureUtcOffset": _first_offset(dep), + "arrivalUtcOffset": _first_offset(arr), ++ # Straight off the flight record, no extra call. aircraftReg is a ++ # free-text field in AirTrail; the ICAO *type* code needs a separate ++ # registration lookup (see lookup_aircraft_icao). ++ "aircraftReg": ((flight.get("aircraft") or {}).get("reg") or "").strip() or None, ++ "departureTerminal": (dep.get("terminal") or "").strip() or None, ++ "departureGate": (dep.get("gate") or "").strip() or None, ++ "arrivalTerminal": (arr.get("terminal") or "").strip() or None, ++ "arrivalGate": (arr.get("gate") or "").strip() or None, + } +diff --git a/airtrail_client.py b/airtrail_client.py +index afc19b8..aaf138a 100644 +--- a/airtrail_client.py ++++ b/airtrail_client.py +@@ -131,8 +131,14 @@ def build_save_payload(row: dict, existing_id: int | None) -> dict: + "datePrecision": "day", + "flightNumber": flight_number or None, + "airline": row.get("operating_carrier_icao"), +- "aircraft": None, +- "aircraftReg": None, ++ # ICAO *type* code (e.g. 'B38M'); AirTrail validates it against its ++ # own aircraft table and 500s the whole save if it doesn't know it, ++ # so save_flight() drops it and retries rather than losing the import. ++ "aircraft": row.get("aircraft_icao"), ++ "aircraftReg": row.get("aircraft_reg"), ++ # Seat position and flight reason are deliberately left unset - the ++ # BCBP carries neither and no API knows them, so they stay a manual ++ # choice in AirTrail rather than a guess made here. + "flightReason": None, + "note": f"Imported via boarding-pass-pipeline (PNR {row.get('pnr') or 'unknown'})", + "seats": [ +@@ -171,6 +177,15 @@ def build_save_payload(row: dict, existing_id: int | None) -> dict: + payload["landingActual"] = row["landing_actual"] + payload["landingActualTime"] = _hhmm(row["landing_actual"]) + ++ for payload_key, row_key in ( ++ ("departureTerminal", "departure_terminal"), ++ ("departureGate", "departure_gate"), ++ ("arrivalTerminal", "arrival_terminal"), ++ ("arrivalGate", "arrival_gate"), ++ ): ++ if row.get(row_key): ++ payload[payload_key] = row[row_key] ++ + coordinates = row.get("track_coordinates") + if coordinates: + times = row.get("track_times") +@@ -192,7 +207,19 @@ def build_save_payload(row: dict, existing_id: int | None) -> dict: + + + def save_flight(payload: dict) -> dict: +- return _request("POST", "/api/flight/save", payload) ++ try: ++ return _request("POST", "/api/flight/save", payload) ++ except AirTrailError as e: ++ # AirTrail resolves `aircraft` against the `icao` column of its own ++ # aircraft table and fails the ENTIRE save if the type code isn't ++ # there (apiError('Invalid aircraft'), HTTP 500). The type is a ++ # nicety; the flight, its times and its track are not. Drop it and ++ # retry once rather than losing the import over an unknown variant. ++ if "Invalid aircraft" in str(e) and payload.get("aircraft"): ++ retry = dict(payload) ++ retry["aircraft"] = None ++ return _request("POST", "/api/flight/save", retry) ++ raise + + + def approve_flight(flight_id: int) -> tuple[bool, str]: +diff --git a/db.py b/db.py +index 7a5e551..5db1d96 100644 +--- a/db.py ++++ b/db.py +@@ -41,6 +41,12 @@ CREATE TABLE IF NOT EXISTS pending_flights ( + gate_arrival_actual TEXT, + departure_utc_offset TEXT, + arrival_utc_offset TEXT, ++ aircraft_reg TEXT, ++ aircraft_icao TEXT, ++ departure_terminal TEXT, ++ departure_gate TEXT, ++ arrival_terminal TEXT, ++ arrival_gate TEXT, + schedule_error TEXT, + airtrail_flight_id INTEGER, + parse_warnings TEXT, +@@ -90,6 +96,12 @@ MIGRATIONS = [ + ("gate_arrival_actual", "TEXT"), + ("departure_utc_offset", "TEXT"), + ("arrival_utc_offset", "TEXT"), ++ ("aircraft_reg", "TEXT"), ++ ("aircraft_icao", "TEXT"), ++ ("departure_terminal", "TEXT"), ++ ("departure_gate", "TEXT"), ++ ("arrival_terminal", "TEXT"), ++ ("arrival_gate", "TEXT"), + ("schedule_error", "TEXT"), + ] + +diff --git a/scheduler.py b/scheduler.py +index 4f3cb08..ad309c7 100644 +--- a/scheduler.py ++++ b/scheduler.py +@@ -104,6 +104,22 @@ def _fetch_gate_times(flight_number: str, flight_date: date, updates: dict) -> N + # AeroDataBox came up empty. + updates["takeoff_actual"] = gate_times.get("takeoffActual") + updates["landing_actual"] = gate_times.get("landingActual") ++ updates["departure_terminal"] = gate_times.get("departureTerminal") ++ updates["departure_gate"] = gate_times.get("departureGate") ++ updates["arrival_terminal"] = gate_times.get("arrivalTerminal") ++ updates["arrival_gate"] = gate_times.get("arrivalGate") ++ ++ registration = gate_times.get("aircraftReg") ++ updates["aircraft_reg"] = registration ++ updates["aircraft_icao"] = None ++ if registration: ++ # One extra API unit, and only worth spending once we actually have a ++ # registration to look up. A failure here is not worth losing the rest ++ # of the lookup over - the registration alone still reaches AirTrail. ++ try: ++ updates["aircraft_icao"] = aerodatabox.lookup_aircraft_icao(registration) ++ except aerodatabox.AeroDataBoxError as e: ++ log.warning("aircraft type lookup failed for %s: %s", registration, e) + + + def process_one_flight(row: dict) -> None: +diff --git a/templates/review_detail.html b/templates/review_detail.html +index b2d7202..d4dffd7 100644 +--- a/templates/review_detail.html ++++ b/templates/review_detail.html +@@ -73,6 +73,12 @@ + {% if flight.landing_actual %} +
Landing (actual)
{{ flight.landing_actual }}
+ {% endif %} ++ {% if flight.aircraft_reg or flight.aircraft_icao %} ++
Aircraft
{{ flight.aircraft_icao or '—' }} · {{ flight.aircraft_reg or '—' }}
++ {% endif %} ++ {% if flight.departure_terminal or flight.departure_gate or flight.arrival_terminal or flight.arrival_gate %} ++
Terminal / gate
{{ flight.departure_terminal or '—' }}/{{ flight.departure_gate or '—' }} → {{ flight.arrival_terminal or '—' }}/{{ flight.arrival_gate or '—' }}
++ {% endif %} + {% if flight.departure_utc_offset or flight.arrival_utc_offset %} +
Airport UTC offsets
{{ flight.departure_utc_offset or '—' }} → {{ flight.arrival_utc_offset or '—' }}
+ {% endif %} diff --git a/db.py b/db.py index 7a5e551..5db1d96 100644 --- a/db.py +++ b/db.py @@ -41,6 +41,12 @@ CREATE TABLE IF NOT EXISTS pending_flights ( gate_arrival_actual TEXT, departure_utc_offset TEXT, arrival_utc_offset TEXT, + aircraft_reg TEXT, + aircraft_icao TEXT, + departure_terminal TEXT, + departure_gate TEXT, + arrival_terminal TEXT, + arrival_gate TEXT, schedule_error TEXT, airtrail_flight_id INTEGER, parse_warnings TEXT, @@ -90,6 +96,12 @@ MIGRATIONS = [ ("gate_arrival_actual", "TEXT"), ("departure_utc_offset", "TEXT"), ("arrival_utc_offset", "TEXT"), + ("aircraft_reg", "TEXT"), + ("aircraft_icao", "TEXT"), + ("departure_terminal", "TEXT"), + ("departure_gate", "TEXT"), + ("arrival_terminal", "TEXT"), + ("arrival_gate", "TEXT"), ("schedule_error", "TEXT"), ] diff --git a/scheduler.py b/scheduler.py index 4f3cb08..ad309c7 100644 --- a/scheduler.py +++ b/scheduler.py @@ -104,6 +104,22 @@ def _fetch_gate_times(flight_number: str, flight_date: date, updates: dict) -> N # AeroDataBox came up empty. updates["takeoff_actual"] = gate_times.get("takeoffActual") updates["landing_actual"] = gate_times.get("landingActual") + updates["departure_terminal"] = gate_times.get("departureTerminal") + updates["departure_gate"] = gate_times.get("departureGate") + updates["arrival_terminal"] = gate_times.get("arrivalTerminal") + updates["arrival_gate"] = gate_times.get("arrivalGate") + + registration = gate_times.get("aircraftReg") + updates["aircraft_reg"] = registration + updates["aircraft_icao"] = None + if registration: + # One extra API unit, and only worth spending once we actually have a + # registration to look up. A failure here is not worth losing the rest + # of the lookup over - the registration alone still reaches AirTrail. + try: + updates["aircraft_icao"] = aerodatabox.lookup_aircraft_icao(registration) + except aerodatabox.AeroDataBoxError as e: + log.warning("aircraft type lookup failed for %s: %s", registration, e) def process_one_flight(row: dict) -> None: diff --git a/templates/review_detail.html b/templates/review_detail.html index b2d7202..d4dffd7 100644 --- a/templates/review_detail.html +++ b/templates/review_detail.html @@ -73,6 +73,12 @@ {% if flight.landing_actual %}
Landing (actual)
{{ flight.landing_actual }}
{% endif %} + {% if flight.aircraft_reg or flight.aircraft_icao %} +
Aircraft
{{ flight.aircraft_icao or '—' }} · {{ flight.aircraft_reg or '—' }}
+ {% endif %} + {% if flight.departure_terminal or flight.departure_gate or flight.arrival_terminal or flight.arrival_gate %} +
Terminal / gate
{{ flight.departure_terminal or '—' }}/{{ flight.departure_gate or '—' }} → {{ flight.arrival_terminal or '—' }}/{{ flight.arrival_gate or '—' }}
+ {% endif %} {% if flight.departure_utc_offset or flight.arrival_utc_offset %}
Airport UTC offsets
{{ flight.departure_utc_offset or '—' }} → {{ flight.arrival_utc_offset or '—' }}
{% endif %}