diff --git a/aerodatabox.py b/aerodatabox.py index 5764b82..94accf3 100644 --- a/aerodatabox.py +++ b/aerodatabox.py @@ -43,6 +43,7 @@ wheels-up/touchdown). import json import re import urllib.error +import urllib.parse import urllib.request from datetime import date, datetime @@ -90,6 +91,64 @@ def _utc_offset(local: str | None) -> str | None: return f"{offset[:3]}:{offset[3:]}" if len(offset) == 5 else None +def _get(url: str): + """GET an AeroDataBox endpoint and return the decoded JSON, or None for + a "no content" answer. Raises AeroDataBoxError for anything retryable. + + A default urllib request (no User-Agent) gets HTTP 403 from AeroDataBox's + Cloudflare-fronted gateway even with a perfectly valid, subscribed key - + confirmed live, side by side, same key/URL: bare urllib -> 403, same + request + this UA -> 200 with real data. See config.BROWSER_USER_AGENT.""" + req = urllib.request.Request( + url, + headers={ + "x-rapidapi-key": config.AERODATABOX_API_KEY, + "User-Agent": config.BROWSER_USER_AGENT, + }, + ) + try: + with urllib.request.urlopen(req, timeout=20) as resp: + if resp.status == 204: + return None + raw = resp.read() + except urllib.error.HTTPError as e: + if e.code == 204: + return None + raise AeroDataBoxError(f"HTTP {e.code} from AeroDataBox") from e + except urllib.error.URLError as e: + raise AeroDataBoxError(f"error reaching AeroDataBox: {e}") from e + + if not raw: + return None + try: + return json.loads(raw.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError) as e: + raise AeroDataBoxError(f"bad AeroDataBox response: {e}") from e + + +def lookup_aircraft_icao(registration: str) -> str | None: + """Registration (e.g. 'VH-8VE') -> ICAO type code (e.g. 'B38M'). + + Costs one extra API unit per flight. Needed because AirTrail's + `aircraft` field is matched against the `icao` column of its own + aircraft table (getAircraftByIcao, confirmed in its source), so the + plain model string AeroDataBox puts on the flight record + ('Boeing 737 MAX 8') is not something AirTrail can resolve.""" + if not config.AERODATABOX_API_KEY or not registration: + return None + + data = _get(f"{BASE_URL}/aircrafts/reg/{urllib.parse.quote(registration)}") + if not isinstance(data, dict): + return None + # AirTrail tries model before icaoCode against the same column, so both + # are worth passing on in that order. + for key in ("icaoCode", "model"): + value = (data.get(key) or "").strip() + if value: + return value + return None + + def _first_offset(leg: dict) -> str | None: """The UTC offset for one leg, from whichever of its timestamps exists. All of a leg's times share the same airport and day, so any of them @@ -126,35 +185,10 @@ def lookup_gate_times(flight_number: str, flight_date: date) -> dict | None: return None cleaned = _sanitize(flight_number) - url = ( + data = _get( f"{BASE_URL}/flights/number/{cleaned}/{flight_date.isoformat()}" "?dateLocalRole=Both&withAircraftImage=false&withLocation=false" ) - # A default urllib request (no User-Agent) gets HTTP 403 from AeroDataBox's - # Cloudflare-fronted gateway even with a perfectly valid, subscribed key - - # confirmed live, side by side, same key/URL: bare urllib -> 403, same - # request + this UA -> 200 with real data. See config.BROWSER_USER_AGENT. - req = urllib.request.Request( - url, - headers={ - "x-rapidapi-key": config.AERODATABOX_API_KEY, - "User-Agent": config.BROWSER_USER_AGENT, - }, - ) - try: - with urllib.request.urlopen(req, timeout=20) as resp: - raw = resp.read() - except urllib.error.HTTPError as e: - if e.code == 204: - return None - raise AeroDataBoxError(f"HTTP {e.code} from AeroDataBox") from e - except urllib.error.URLError as e: - raise AeroDataBoxError(f"error reaching AeroDataBox: {e}") from e - - try: - data = json.loads(raw.decode("utf-8")) - except (json.JSONDecodeError, UnicodeDecodeError) as e: - raise AeroDataBoxError(f"bad AeroDataBox response: {e}") from e if not isinstance(data, list) or not data: return None @@ -191,4 +225,12 @@ def lookup_gate_times(flight_number: str, flight_date: date) -> dict | None: # into the same airport-local terms as everything else here. "departureUtcOffset": _first_offset(dep), "arrivalUtcOffset": _first_offset(arr), + # Straight off the flight record, no extra call. aircraftReg is a + # free-text field in AirTrail; the ICAO *type* code needs a separate + # registration lookup (see lookup_aircraft_icao). + "aircraftReg": ((flight.get("aircraft") or {}).get("reg") or "").strip() or None, + "departureTerminal": (dep.get("terminal") or "").strip() or None, + "departureGate": (dep.get("gate") or "").strip() or None, + "arrivalTerminal": (arr.get("terminal") or "").strip() or None, + "arrivalGate": (arr.get("gate") or "").strip() or None, } diff --git a/airtrail_client.py b/airtrail_client.py index afc19b8..aaf138a 100644 --- a/airtrail_client.py +++ b/airtrail_client.py @@ -131,8 +131,14 @@ def build_save_payload(row: dict, existing_id: int | None) -> dict: "datePrecision": "day", "flightNumber": flight_number or None, "airline": row.get("operating_carrier_icao"), - "aircraft": None, - "aircraftReg": None, + # ICAO *type* code (e.g. 'B38M'); AirTrail validates it against its + # own aircraft table and 500s the whole save if it doesn't know it, + # so save_flight() drops it and retries rather than losing the import. + "aircraft": row.get("aircraft_icao"), + "aircraftReg": row.get("aircraft_reg"), + # Seat position and flight reason are deliberately left unset - the + # BCBP carries neither and no API knows them, so they stay a manual + # choice in AirTrail rather than a guess made here. "flightReason": None, "note": f"Imported via boarding-pass-pipeline (PNR {row.get('pnr') or 'unknown'})", "seats": [ @@ -171,6 +177,15 @@ def build_save_payload(row: dict, existing_id: int | None) -> dict: payload["landingActual"] = row["landing_actual"] payload["landingActualTime"] = _hhmm(row["landing_actual"]) + for payload_key, row_key in ( + ("departureTerminal", "departure_terminal"), + ("departureGate", "departure_gate"), + ("arrivalTerminal", "arrival_terminal"), + ("arrivalGate", "arrival_gate"), + ): + if row.get(row_key): + payload[payload_key] = row[row_key] + coordinates = row.get("track_coordinates") if coordinates: times = row.get("track_times") @@ -192,7 +207,19 @@ def build_save_payload(row: dict, existing_id: int | None) -> dict: def save_flight(payload: dict) -> dict: - return _request("POST", "/api/flight/save", payload) + try: + return _request("POST", "/api/flight/save", payload) + except AirTrailError as e: + # AirTrail resolves `aircraft` against the `icao` column of its own + # aircraft table and fails the ENTIRE save if the type code isn't + # there (apiError('Invalid aircraft'), HTTP 500). The type is a + # nicety; the flight, its times and its track are not. Drop it and + # retry once rather than losing the import over an unknown variant. + if "Invalid aircraft" in str(e) and payload.get("aircraft"): + retry = dict(payload) + retry["aircraft"] = None + return _request("POST", "/api/flight/save", retry) + raise def approve_flight(flight_id: int) -> tuple[bool, str]: diff --git a/db.py b/db.py index 7a5e551..5db1d96 100644 --- a/db.py +++ b/db.py @@ -41,6 +41,12 @@ CREATE TABLE IF NOT EXISTS pending_flights ( gate_arrival_actual TEXT, departure_utc_offset TEXT, arrival_utc_offset TEXT, + aircraft_reg TEXT, + aircraft_icao TEXT, + departure_terminal TEXT, + departure_gate TEXT, + arrival_terminal TEXT, + arrival_gate TEXT, schedule_error TEXT, airtrail_flight_id INTEGER, parse_warnings TEXT, @@ -90,6 +96,12 @@ MIGRATIONS = [ ("gate_arrival_actual", "TEXT"), ("departure_utc_offset", "TEXT"), ("arrival_utc_offset", "TEXT"), + ("aircraft_reg", "TEXT"), + ("aircraft_icao", "TEXT"), + ("departure_terminal", "TEXT"), + ("departure_gate", "TEXT"), + ("arrival_terminal", "TEXT"), + ("arrival_gate", "TEXT"), ("schedule_error", "TEXT"), ] diff --git a/scheduler.py b/scheduler.py index 4f3cb08..ad309c7 100644 --- a/scheduler.py +++ b/scheduler.py @@ -104,6 +104,22 @@ def _fetch_gate_times(flight_number: str, flight_date: date, updates: dict) -> N # AeroDataBox came up empty. updates["takeoff_actual"] = gate_times.get("takeoffActual") updates["landing_actual"] = gate_times.get("landingActual") + updates["departure_terminal"] = gate_times.get("departureTerminal") + updates["departure_gate"] = gate_times.get("departureGate") + updates["arrival_terminal"] = gate_times.get("arrivalTerminal") + updates["arrival_gate"] = gate_times.get("arrivalGate") + + registration = gate_times.get("aircraftReg") + updates["aircraft_reg"] = registration + updates["aircraft_icao"] = None + if registration: + # One extra API unit, and only worth spending once we actually have a + # registration to look up. A failure here is not worth losing the rest + # of the lookup over - the registration alone still reaches AirTrail. + try: + updates["aircraft_icao"] = aerodatabox.lookup_aircraft_icao(registration) + except aerodatabox.AeroDataBoxError as e: + log.warning("aircraft type lookup failed for %s: %s", registration, e) def process_one_flight(row: dict) -> None: diff --git a/templates/review_detail.html b/templates/review_detail.html index b2d7202..d4dffd7 100644 --- a/templates/review_detail.html +++ b/templates/review_detail.html @@ -73,6 +73,12 @@ {% if flight.landing_actual %}
Landing (actual)
{{ flight.landing_actual }}
{% endif %} + {% if flight.aircraft_reg or flight.aircraft_icao %} +
Aircraft
{{ flight.aircraft_icao or '—' }} · {{ flight.aircraft_reg or '—' }}
+ {% endif %} + {% if flight.departure_terminal or flight.departure_gate or flight.arrival_terminal or flight.arrival_gate %} +
Terminal / gate
{{ flight.departure_terminal or '—' }}/{{ flight.departure_gate or '—' }} → {{ flight.arrival_terminal or '—' }}/{{ flight.arrival_gate or '—' }}
+ {% endif %} {% if flight.departure_utc_offset or flight.arrival_utc_offset %}
Airport UTC offsets
{{ flight.departure_utc_offset or '—' }} → {{ flight.arrival_utc_offset or '—' }}
{% endif %}