fill aircraft type/registration and terminal/gate from AeroDataBox

aerodatabox.py

@@ -43,6 +43,7 @@ wheels-up/touchdown).
 import json
 import re
 import urllib.error
+import urllib.parse
 import urllib.request
 from datetime import date, datetime
 
@@ -90,6 +91,64 @@ def _utc_offset(local: str | None) -> str | None:
     return f"{offset[:3]}:{offset[3:]}" if len(offset) == 5 else None
 
 
+def _get(url: str):
+    """GET an AeroDataBox endpoint and return the decoded JSON, or None for
+    a "no content" answer. Raises AeroDataBoxError for anything retryable.
+
+    A default urllib request (no User-Agent) gets HTTP 403 from AeroDataBox's
+    Cloudflare-fronted gateway even with a perfectly valid, subscribed key -
+    confirmed live, side by side, same key/URL: bare urllib -> 403, same
+    request + this UA -> 200 with real data. See config.BROWSER_USER_AGENT."""
+    req = urllib.request.Request(
+        url,
+        headers={
+            "x-rapidapi-key": config.AERODATABOX_API_KEY,
+            "User-Agent": config.BROWSER_USER_AGENT,
+        },
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=20) as resp:
+            if resp.status == 204:
+                return None
+            raw = resp.read()
+    except urllib.error.HTTPError as e:
+        if e.code == 204:
+            return None
+        raise AeroDataBoxError(f"HTTP {e.code} from AeroDataBox") from e
+    except urllib.error.URLError as e:
+        raise AeroDataBoxError(f"error reaching AeroDataBox: {e}") from e
+
+    if not raw:
+        return None
+    try:
+        return json.loads(raw.decode("utf-8"))
+    except (json.JSONDecodeError, UnicodeDecodeError) as e:
+        raise AeroDataBoxError(f"bad AeroDataBox response: {e}") from e
+
+
+def lookup_aircraft_icao(registration: str) -> str | None:
+    """Registration (e.g. 'VH-8VE') -> ICAO type code (e.g. 'B38M').
+
+    Costs one extra API unit per flight. Needed because AirTrail's
+    `aircraft` field is matched against the `icao` column of its own
+    aircraft table (getAircraftByIcao, confirmed in its source), so the
+    plain model string AeroDataBox puts on the flight record
+    ('Boeing 737 MAX 8') is not something AirTrail can resolve."""
+    if not config.AERODATABOX_API_KEY or not registration:
+        return None
+
+    data = _get(f"{BASE_URL}/aircrafts/reg/{urllib.parse.quote(registration)}")
+    if not isinstance(data, dict):
+        return None
+    # AirTrail tries model before icaoCode against the same column, so both
+    # are worth passing on in that order.
+    for key in ("icaoCode", "model"):
+        value = (data.get(key) or "").strip()
+        if value:
+            return value
+    return None
+
+
 def _first_offset(leg: dict) -> str | None:
     """The UTC offset for one leg, from whichever of its timestamps exists.
     All of a leg's times share the same airport and day, so any of them
@@ -126,35 +185,10 @@ def lookup_gate_times(flight_number: str, flight_date: date) -> dict | None:
         return None
 
     cleaned = _sanitize(flight_number)
-    url = (
+    data = _get(
         f"{BASE_URL}/flights/number/{cleaned}/{flight_date.isoformat()}"
         "?dateLocalRole=Both&withAircraftImage=false&withLocation=false"
     )
-    # A default urllib request (no User-Agent) gets HTTP 403 from AeroDataBox's
-    # Cloudflare-fronted gateway even with a perfectly valid, subscribed key -
-    # confirmed live, side by side, same key/URL: bare urllib -> 403, same
-    # request + this UA -> 200 with real data. See config.BROWSER_USER_AGENT.
-    req = urllib.request.Request(
-        url,
-        headers={
-            "x-rapidapi-key": config.AERODATABOX_API_KEY,
-            "User-Agent": config.BROWSER_USER_AGENT,
-        },
-    )
-    try:
-        with urllib.request.urlopen(req, timeout=20) as resp:
-            raw = resp.read()
-    except urllib.error.HTTPError as e:
-        if e.code == 204:
-            return None
-        raise AeroDataBoxError(f"HTTP {e.code} from AeroDataBox") from e
-    except urllib.error.URLError as e:
-        raise AeroDataBoxError(f"error reaching AeroDataBox: {e}") from e
-
-    try:
-        data = json.loads(raw.decode("utf-8"))
-    except (json.JSONDecodeError, UnicodeDecodeError) as e:
-        raise AeroDataBoxError(f"bad AeroDataBox response: {e}") from e
 
     if not isinstance(data, list) or not data:
         return None
@@ -191,4 +225,12 @@ def lookup_gate_times(flight_number: str, flight_date: date) -> dict | None:
         # into the same airport-local terms as everything else here.
         "departureUtcOffset": _first_offset(dep),
         "arrivalUtcOffset": _first_offset(arr),
+        # Straight off the flight record, no extra call. aircraftReg is a
+        # free-text field in AirTrail; the ICAO *type* code needs a separate
+        # registration lookup (see lookup_aircraft_icao).
+        "aircraftReg": ((flight.get("aircraft") or {}).get("reg") or "").strip() or None,
+        "departureTerminal": (dep.get("terminal") or "").strip() or None,
+        "departureGate": (dep.get("gate") or "").strip() or None,
+        "arrivalTerminal": (arr.get("terminal") or "").strip() or None,
+        "arrivalGate": (arr.get("gate") or "").strip() or None,
     }

airtrail_client.py

@@ -131,8 +131,14 @@ def build_save_payload(row: dict, existing_id: int | None) -> dict:
         "datePrecision": "day",
         "flightNumber": flight_number or None,
         "airline": row.get("operating_carrier_icao"),
-        "aircraft": None,
-        "aircraftReg": None,
+        # ICAO *type* code (e.g. 'B38M'); AirTrail validates it against its
+        # own aircraft table and 500s the whole save if it doesn't know it,
+        # so save_flight() drops it and retries rather than losing the import.
+        "aircraft": row.get("aircraft_icao"),
+        "aircraftReg": row.get("aircraft_reg"),
+        # Seat position and flight reason are deliberately left unset - the
+        # BCBP carries neither and no API knows them, so they stay a manual
+        # choice in AirTrail rather than a guess made here.
         "flightReason": None,
         "note": f"Imported via boarding-pass-pipeline (PNR {row.get('pnr') or 'unknown'})",
         "seats": [
@@ -171,6 +177,15 @@ def build_save_payload(row: dict, existing_id: int | None) -> dict:
         payload["landingActual"] = row["landing_actual"]
         payload["landingActualTime"] = _hhmm(row["landing_actual"])
 
+    for payload_key, row_key in (
+        ("departureTerminal", "departure_terminal"),
+        ("departureGate", "departure_gate"),
+        ("arrivalTerminal", "arrival_terminal"),
+        ("arrivalGate", "arrival_gate"),
+    ):
+        if row.get(row_key):
+            payload[payload_key] = row[row_key]
+
     coordinates = row.get("track_coordinates")
     if coordinates:
         times = row.get("track_times")
@@ -192,7 +207,19 @@ def build_save_payload(row: dict, existing_id: int | None) -> dict:
 
 
 def save_flight(payload: dict) -> dict:
+    try:
         return _request("POST", "/api/flight/save", payload)
+    except AirTrailError as e:
+        # AirTrail resolves `aircraft` against the `icao` column of its own
+        # aircraft table and fails the ENTIRE save if the type code isn't
+        # there (apiError('Invalid aircraft'), HTTP 500). The type is a
+        # nicety; the flight, its times and its track are not. Drop it and
+        # retry once rather than losing the import over an unknown variant.
+        if "Invalid aircraft" in str(e) and payload.get("aircraft"):
+            retry = dict(payload)
+            retry["aircraft"] = None
+            return _request("POST", "/api/flight/save", retry)
+        raise
 
 
 def approve_flight(flight_id: int) -> tuple[bool, str]:

boarding-pass-aircraft-terminal-fix.patch

@@ -0,0 +1,259 @@
+diff --git a/aerodatabox.py b/aerodatabox.py
+index 5764b82..94accf3 100644
+--- a/aerodatabox.py
++++ b/aerodatabox.py
+@@ -43,6 +43,7 @@ wheels-up/touchdown).
+ import json
+ import re
+ import urllib.error
++import urllib.parse
+ import urllib.request
+ from datetime import date, datetime
+ 
+@@ -90,6 +91,64 @@ def _utc_offset(local: str | None) -> str | None:
+     return f"{offset[:3]}:{offset[3:]}" if len(offset) == 5 else None
+ 
+ 
++def _get(url: str):
++    """GET an AeroDataBox endpoint and return the decoded JSON, or None for
++    a "no content" answer. Raises AeroDataBoxError for anything retryable.
++
++    A default urllib request (no User-Agent) gets HTTP 403 from AeroDataBox's
++    Cloudflare-fronted gateway even with a perfectly valid, subscribed key -
++    confirmed live, side by side, same key/URL: bare urllib -> 403, same
++    request + this UA -> 200 with real data. See config.BROWSER_USER_AGENT."""
++    req = urllib.request.Request(
++        url,
++        headers={
++            "x-rapidapi-key": config.AERODATABOX_API_KEY,
++            "User-Agent": config.BROWSER_USER_AGENT,
++        },
++    )
++    try:
++        with urllib.request.urlopen(req, timeout=20) as resp:
++            if resp.status == 204:
++                return None
++            raw = resp.read()
++    except urllib.error.HTTPError as e:
++        if e.code == 204:
++            return None
++        raise AeroDataBoxError(f"HTTP {e.code} from AeroDataBox") from e
++    except urllib.error.URLError as e:
++        raise AeroDataBoxError(f"error reaching AeroDataBox: {e}") from e
++
++    if not raw:
++        return None
++    try:
++        return json.loads(raw.decode("utf-8"))
++    except (json.JSONDecodeError, UnicodeDecodeError) as e:
++        raise AeroDataBoxError(f"bad AeroDataBox response: {e}") from e
++
++
++def lookup_aircraft_icao(registration: str) -> str | None:
++    """Registration (e.g. 'VH-8VE') -> ICAO type code (e.g. 'B38M').
++
++    Costs one extra API unit per flight. Needed because AirTrail's
++    `aircraft` field is matched against the `icao` column of its own
++    aircraft table (getAircraftByIcao, confirmed in its source), so the
++    plain model string AeroDataBox puts on the flight record
++    ('Boeing 737 MAX 8') is not something AirTrail can resolve."""
++    if not config.AERODATABOX_API_KEY or not registration:
++        return None
++
++    data = _get(f"{BASE_URL}/aircrafts/reg/{urllib.parse.quote(registration)}")
++    if not isinstance(data, dict):
++        return None
++    # AirTrail tries model before icaoCode against the same column, so both
++    # are worth passing on in that order.
++    for key in ("icaoCode", "model"):
++        value = (data.get(key) or "").strip()
++        if value:
++            return value
++    return None
++
++
+ def _first_offset(leg: dict) -> str | None:
+     """The UTC offset for one leg, from whichever of its timestamps exists.
+     All of a leg's times share the same airport and day, so any of them
+@@ -126,35 +185,10 @@ def lookup_gate_times(flight_number: str, flight_date: date) -> dict | None:
+         return None
+ 
+     cleaned = _sanitize(flight_number)
+-    url = (
++    data = _get(
+         f"{BASE_URL}/flights/number/{cleaned}/{flight_date.isoformat()}"
+         "?dateLocalRole=Both&withAircraftImage=false&withLocation=false"
+     )
+-    # A default urllib request (no User-Agent) gets HTTP 403 from AeroDataBox's
+-    # Cloudflare-fronted gateway even with a perfectly valid, subscribed key -
+-    # confirmed live, side by side, same key/URL: bare urllib -> 403, same
+-    # request + this UA -> 200 with real data. See config.BROWSER_USER_AGENT.
+-    req = urllib.request.Request(
+-        url,
+-        headers={
+-            "x-rapidapi-key": config.AERODATABOX_API_KEY,
+-            "User-Agent": config.BROWSER_USER_AGENT,
+-        },
+-    )
+-    try:
+-        with urllib.request.urlopen(req, timeout=20) as resp:
+-            raw = resp.read()
+-    except urllib.error.HTTPError as e:
+-        if e.code == 204:
+-            return None
+-        raise AeroDataBoxError(f"HTTP {e.code} from AeroDataBox") from e
+-    except urllib.error.URLError as e:
+-        raise AeroDataBoxError(f"error reaching AeroDataBox: {e}") from e
+-
+-    try:
+-        data = json.loads(raw.decode("utf-8"))
+-    except (json.JSONDecodeError, UnicodeDecodeError) as e:
+-        raise AeroDataBoxError(f"bad AeroDataBox response: {e}") from e
+ 
+     if not isinstance(data, list) or not data:
+         return None
+@@ -191,4 +225,12 @@ def lookup_gate_times(flight_number: str, flight_date: date) -> dict | None:
+         # into the same airport-local terms as everything else here.
+         "departureUtcOffset": _first_offset(dep),
+         "arrivalUtcOffset": _first_offset(arr),
++        # Straight off the flight record, no extra call. aircraftReg is a
++        # free-text field in AirTrail; the ICAO *type* code needs a separate
++        # registration lookup (see lookup_aircraft_icao).
++        "aircraftReg": ((flight.get("aircraft") or {}).get("reg") or "").strip() or None,
++        "departureTerminal": (dep.get("terminal") or "").strip() or None,
++        "departureGate": (dep.get("gate") or "").strip() or None,
++        "arrivalTerminal": (arr.get("terminal") or "").strip() or None,
++        "arrivalGate": (arr.get("gate") or "").strip() or None,
+     }
+diff --git a/airtrail_client.py b/airtrail_client.py
+index afc19b8..aaf138a 100644
+--- a/airtrail_client.py
++++ b/airtrail_client.py
+@@ -131,8 +131,14 @@ def build_save_payload(row: dict, existing_id: int | None) -> dict:
+         "datePrecision": "day",
+         "flightNumber": flight_number or None,
+         "airline": row.get("operating_carrier_icao"),
+-        "aircraft": None,
+-        "aircraftReg": None,
++        # ICAO *type* code (e.g. 'B38M'); AirTrail validates it against its
++        # own aircraft table and 500s the whole save if it doesn't know it,
++        # so save_flight() drops it and retries rather than losing the import.
++        "aircraft": row.get("aircraft_icao"),
++        "aircraftReg": row.get("aircraft_reg"),
++        # Seat position and flight reason are deliberately left unset - the
++        # BCBP carries neither and no API knows them, so they stay a manual
++        # choice in AirTrail rather than a guess made here.
+         "flightReason": None,
+         "note": f"Imported via boarding-pass-pipeline (PNR {row.get('pnr') or 'unknown'})",
+         "seats": [
+@@ -171,6 +177,15 @@ def build_save_payload(row: dict, existing_id: int | None) -> dict:
+         payload["landingActual"] = row["landing_actual"]
+         payload["landingActualTime"] = _hhmm(row["landing_actual"])
+ 
++    for payload_key, row_key in (
++        ("departureTerminal", "departure_terminal"),
++        ("departureGate", "departure_gate"),
++        ("arrivalTerminal", "arrival_terminal"),
++        ("arrivalGate", "arrival_gate"),
++    ):
++        if row.get(row_key):
++            payload[payload_key] = row[row_key]
++
+     coordinates = row.get("track_coordinates")
+     if coordinates:
+         times = row.get("track_times")
+@@ -192,7 +207,19 @@ def build_save_payload(row: dict, existing_id: int | None) -> dict:
+ 
+ 
+ def save_flight(payload: dict) -> dict:
+-    return _request("POST", "/api/flight/save", payload)
++    try:
++        return _request("POST", "/api/flight/save", payload)
++    except AirTrailError as e:
++        # AirTrail resolves `aircraft` against the `icao` column of its own
++        # aircraft table and fails the ENTIRE save if the type code isn't
++        # there (apiError('Invalid aircraft'), HTTP 500). The type is a
++        # nicety; the flight, its times and its track are not. Drop it and
++        # retry once rather than losing the import over an unknown variant.
++        if "Invalid aircraft" in str(e) and payload.get("aircraft"):
++            retry = dict(payload)
++            retry["aircraft"] = None
++            return _request("POST", "/api/flight/save", retry)
++        raise
+ 
+ 
+ def approve_flight(flight_id: int) -> tuple[bool, str]:
+diff --git a/db.py b/db.py
+index 7a5e551..5db1d96 100644
+--- a/db.py
++++ b/db.py
+@@ -41,6 +41,12 @@ CREATE TABLE IF NOT EXISTS pending_flights (
+     gate_arrival_actual     TEXT,
+     departure_utc_offset    TEXT,
+     arrival_utc_offset      TEXT,
++    aircraft_reg            TEXT,
++    aircraft_icao           TEXT,
++    departure_terminal      TEXT,
++    departure_gate          TEXT,
++    arrival_terminal        TEXT,
++    arrival_gate            TEXT,
+     schedule_error          TEXT,
+     airtrail_flight_id      INTEGER,
+     parse_warnings          TEXT,
+@@ -90,6 +96,12 @@ MIGRATIONS = [
+     ("gate_arrival_actual", "TEXT"),
+     ("departure_utc_offset", "TEXT"),
+     ("arrival_utc_offset", "TEXT"),
++    ("aircraft_reg", "TEXT"),
++    ("aircraft_icao", "TEXT"),
++    ("departure_terminal", "TEXT"),
++    ("departure_gate", "TEXT"),
++    ("arrival_terminal", "TEXT"),
++    ("arrival_gate", "TEXT"),
+     ("schedule_error", "TEXT"),
+ ]
+ 
+diff --git a/scheduler.py b/scheduler.py
+index 4f3cb08..ad309c7 100644
+--- a/scheduler.py
++++ b/scheduler.py
+@@ -104,6 +104,22 @@ def _fetch_gate_times(flight_number: str, flight_date: date, updates: dict) -> N
+     # AeroDataBox came up empty.
+     updates["takeoff_actual"] = gate_times.get("takeoffActual")
+     updates["landing_actual"] = gate_times.get("landingActual")
++    updates["departure_terminal"] = gate_times.get("departureTerminal")
++    updates["departure_gate"] = gate_times.get("departureGate")
++    updates["arrival_terminal"] = gate_times.get("arrivalTerminal")
++    updates["arrival_gate"] = gate_times.get("arrivalGate")
++
++    registration = gate_times.get("aircraftReg")
++    updates["aircraft_reg"] = registration
++    updates["aircraft_icao"] = None
++    if registration:
++        # One extra API unit, and only worth spending once we actually have a
++        # registration to look up. A failure here is not worth losing the rest
++        # of the lookup over - the registration alone still reaches AirTrail.
++        try:
++            updates["aircraft_icao"] = aerodatabox.lookup_aircraft_icao(registration)
++        except aerodatabox.AeroDataBoxError as e:
++            log.warning("aircraft type lookup failed for %s: %s", registration, e)
+ 
+ 
+ def process_one_flight(row: dict) -> None:
+diff --git a/templates/review_detail.html b/templates/review_detail.html
+index b2d7202..d4dffd7 100644
+--- a/templates/review_detail.html
++++ b/templates/review_detail.html
+@@ -73,6 +73,12 @@
+       {% if flight.landing_actual %}
+       <dt>Landing (actual)</dt><dd>{{ flight.landing_actual }}</dd>
+       {% endif %}
++      {% if flight.aircraft_reg or flight.aircraft_icao %}
++      <dt>Aircraft</dt><dd>{{ flight.aircraft_icao or '—' }} · {{ flight.aircraft_reg or '—' }}</dd>
++      {% endif %}
++      {% if flight.departure_terminal or flight.departure_gate or flight.arrival_terminal or flight.arrival_gate %}
++      <dt>Terminal / gate</dt><dd>{{ flight.departure_terminal or '—' }}/{{ flight.departure_gate or '—' }} &rarr; {{ flight.arrival_terminal or '—' }}/{{ flight.arrival_gate or '—' }}</dd>
++      {% endif %}
+       {% if flight.departure_utc_offset or flight.arrival_utc_offset %}
+       <dt>Airport UTC offsets</dt><dd>{{ flight.departure_utc_offset or '—' }} &rarr; {{ flight.arrival_utc_offset or '—' }}</dd>
+       {% endif %}

db.py

@@ -41,6 +41,12 @@ CREATE TABLE IF NOT EXISTS pending_flights (
     gate_arrival_actual     TEXT,
     departure_utc_offset    TEXT,
     arrival_utc_offset      TEXT,
+    aircraft_reg            TEXT,
+    aircraft_icao           TEXT,
+    departure_terminal      TEXT,
+    departure_gate          TEXT,
+    arrival_terminal        TEXT,
+    arrival_gate            TEXT,
     schedule_error          TEXT,
     airtrail_flight_id      INTEGER,
     parse_warnings          TEXT,
@@ -90,6 +96,12 @@ MIGRATIONS = [
     ("gate_arrival_actual", "TEXT"),
     ("departure_utc_offset", "TEXT"),
     ("arrival_utc_offset", "TEXT"),
+    ("aircraft_reg", "TEXT"),
+    ("aircraft_icao", "TEXT"),
+    ("departure_terminal", "TEXT"),
+    ("departure_gate", "TEXT"),
+    ("arrival_terminal", "TEXT"),
+    ("arrival_gate", "TEXT"),
     ("schedule_error", "TEXT"),
 ]
 

scheduler.py

@@ -104,6 +104,22 @@ def _fetch_gate_times(flight_number: str, flight_date: date, updates: dict) -> N
     # AeroDataBox came up empty.
     updates["takeoff_actual"] = gate_times.get("takeoffActual")
     updates["landing_actual"] = gate_times.get("landingActual")
+    updates["departure_terminal"] = gate_times.get("departureTerminal")
+    updates["departure_gate"] = gate_times.get("departureGate")
+    updates["arrival_terminal"] = gate_times.get("arrivalTerminal")
+    updates["arrival_gate"] = gate_times.get("arrivalGate")
+
+    registration = gate_times.get("aircraftReg")
+    updates["aircraft_reg"] = registration
+    updates["aircraft_icao"] = None
+    if registration:
+        # One extra API unit, and only worth spending once we actually have a
+        # registration to look up. A failure here is not worth losing the rest
+        # of the lookup over - the registration alone still reaches AirTrail.
+        try:
+            updates["aircraft_icao"] = aerodatabox.lookup_aircraft_icao(registration)
+        except aerodatabox.AeroDataBoxError as e:
+            log.warning("aircraft type lookup failed for %s: %s", registration, e)
 
 
 def process_one_flight(row: dict) -> None:

templates/review_detail.html

@@ -73,6 +73,12 @@
       {% if flight.landing_actual %}
       <dt>Landing (actual)</dt><dd>{{ flight.landing_actual }}</dd>
       {% endif %}
+      {% if flight.aircraft_reg or flight.aircraft_icao %}
+      <dt>Aircraft</dt><dd>{{ flight.aircraft_icao or '—' }} · {{ flight.aircraft_reg or '—' }}</dd>
+      {% endif %}
+      {% if flight.departure_terminal or flight.departure_gate or flight.arrival_terminal or flight.arrival_gate %}
+      <dt>Terminal / gate</dt><dd>{{ flight.departure_terminal or '—' }}/{{ flight.departure_gate or '—' }} &rarr; {{ flight.arrival_terminal or '—' }}/{{ flight.arrival_gate or '—' }}</dd>
+      {% endif %}
       {% if flight.departure_utc_offset or flight.arrival_utc_offset %}
       <dt>Airport UTC offsets</dt><dd>{{ flight.departure_utc_offset or '—' }} &rarr; {{ flight.arrival_utc_offset or '—' }}</dd>
       {% endif %}