bsncubed/boarding-pass
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

actual times via AeroDataBox + FlightAware runway fallback

Browse Source
This commit is contained in:
Ben Nicholson
2026-09-12 12:40:41 +10:00
parent c2e78cb256
commit d524e6d343
8 changed files with 698 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
boarding-pass-actual-times.patch
+409
View File
@@ -0,0 +1,409 @@
diff --git a/.env.example b/.env.example
index 72b0774..abdb94a 100644
--- a/.env.example
+++ b/.env.example
@@ -6,6 +6,12 @@ PUBLIC_BASE_URL=https://boarding-pass.example.com
AIRTRAIL_BASE_URL=https://airtrail.example.com
AIRTRAIL_API_KEY=
+# Optional: RapidAPI AeroDataBox key, used to fill in scheduled/actual GATE
+# times (same provider AirTrail's own "Search" button uses). Leave blank to
+# skip this - gate times just stay blank for AirTrail's manual Search to
+# fill in, same as before this was added.
+AERODATABOX_API_KEY=
+
# Container timezone. Set explicitly - flight_date + 1 day and the daily
# job's run time are date-sensitive.
TZ=Australia/Sydney
diff --git a/aerodatabox.py b/aerodatabox.py
new file mode 100644
index 0000000..8ba4e12
--- /dev/null
+++ b/aerodatabox.py
@@ -0,0 +1,150 @@
+"""Fetch scheduled + actual gate departure/arrival times, plus actual
+takeoff/landing (runway) times, from AeroDataBox (RapidAPI) - the same
+provider AirTrail's own "Search" button uses internally when you fill in
+a flight manually.
+
+Confirmed directly against AirTrail's source (johanohly/AirTrail):
+- src/lib/server/utils/flight-lookup/aerodatabox.ts (endpoint, headers,
+ response shape, and the actualTime -> revisedTime -> scheduledTime
+ fallback order for "best known actual time")
+- src/lib/zod/flight.ts (target field names: departure/departureScheduled/
+ arrival/arrivalScheduled expect full-seconds ISO-8601 datetimes)
+
+Why this exists as well as flightaware.py: FlightAware's public history
+page (scraped there) only gives us a GPS track - no gate/schedule times.
+BCBP boarding passes don't carry a time-of-day at all. AeroDataBox is the
+only piece that can fill in actual scheduled/gate-actual times without
+requiring a logged-in AirTrail session (its own lookup is cookie-authed,
+not reachable with our API key).
+
+VERIFIED (2026-09-12, real key, Free Tier): a flight 4 days in the past
+(VA559, 2026-09-08) returned full data - scheduledTime/revisedTime/
+runwayTime on both legs. Free Tier quota is 400 API units / period, 1
+unit per lookup here, so the daily one-flight job has plenty of headroom.
+A missing/empty response should still fail soft (see lookup_gate_times'
+return of None) rather than block the FlightAware track or the
+review/approve flow - just in case an individual flight falls outside
+whatever window AeroDataBox actually indexed.
+
+Bonus found in that same real response: AeroDataBox also returns
+departure.runwayTime / arrival.runwayTime - actual takeoff/landing off
+the runway, not just gate times. AirTrail's own aerodatabox.ts never
+reads this field at all (confirmed against its source above - it types
+departure/arrival with only scheduledTime/revisedTime/actualTime, no
+runwayTime), which is why AirTrail's own Search button leaves
+Takeoff/Landing blank even after a successful search. We use it here for
+takeoffActual/landingActual - preferred in scheduler.py over the
+FlightAware-track-derived guess when present, since it's an authoritative
+field rather than an inferred one (the track only tells us when the
+transponder started/stopped squawking, which is close to but not exactly
+wheels-up/touchdown).
+"""
+
+import json
+import re
+import urllib.error
+import urllib.request
+from datetime import date, datetime
+
+import config
+
+BASE_URL = "https://aerodatabox.p.rapidapi.com"
+
+# Fields this module can fill in build_save_payload(), keyed the same way
+# AirTrail's /api/flight/save expects them.
+GATE_FIELDS = (
+ "departureScheduled",
+ "departure",
+ "arrivalScheduled",
+ "arrival",
+ "takeoffActual",
+ "landingActual",
+)
+
+
+class AeroDataBoxError(Exception):
+ """Network/config/HTTP error - safe to retry on a later run, same
+ treatment as FlightAwareTransientError."""
+
+
+def _sanitize(flight_number: str) -> str:
+ return re.sub(r"[\s-]", "", flight_number).upper()
+
+
+def _parse_local(local: str | None) -> str | None:
+ """AeroDataBox 'local' strings look like '2026-06-22 13:45+10:00' -
+ already carrying their own UTC offset, no separate airport-timezone
+ lookup needed. Reformats to the seconds-precision ISO-8601 string
+ AirTrail's zod schema (z.string().datetime({offset: true})) requires,
+ e.g. '2026-06-22T13:45:00+10:00'."""
+ if not local:
+ return None
+ try:
+ dt = datetime.strptime(local, "%Y-%m-%d %H:%M%z")
+ except ValueError:
+ return None
+ return dt.isoformat(timespec="seconds")
+
+
+def lookup_gate_times(flight_number: str, flight_date: date) -> dict | None:
+ """Returns a dict with some/all of GATE_FIELDS set (None for anything
+ AeroDataBox didn't have), or None if the lookup failed outright or
+ matched nothing. Never raises for "no data" - only for genuine
+ network/config failures (AeroDataBoxError), mirroring flightaware.py's
+ distinction between "definitively not found" and "try again later"."""
+ if not config.AERODATABOX_API_KEY:
+ return None
+
+ cleaned = _sanitize(flight_number)
+ url = (
+ f"{BASE_URL}/flights/number/{cleaned}/{flight_date.isoformat()}"
+ "?dateLocalRole=Both&withAircraftImage=false&withLocation=false"
+ )
+ req = urllib.request.Request(
+ url, headers={"x-rapidapi-key": config.AERODATABOX_API_KEY}
+ )
+ try:
+ with urllib.request.urlopen(req, timeout=20) as resp:
+ raw = resp.read()
+ except urllib.error.HTTPError as e:
+ if e.code == 204:
+ return None
+ raise AeroDataBoxError(f"HTTP {e.code} from AeroDataBox") from e
+ except urllib.error.URLError as e:
+ raise AeroDataBoxError(f"error reaching AeroDataBox: {e}") from e
+
+ try:
+ data = json.loads(raw.decode("utf-8"))
+ except (json.JSONDecodeError, UnicodeDecodeError) as e:
+ raise AeroDataBoxError(f"bad AeroDataBox response: {e}") from e
+
+ if not isinstance(data, list) or not data:
+ return None
+
+ # BCBP gives no time-of-day, so - same limitation as the FlightAware
+ # step - we can't disambiguate multiple same-day legs of the same
+ # flight number from the boarding pass alone. Take the first result,
+ # same as AirTrail's own lookup falls back to when nothing else to
+ # filter on (from/to aren't sent here, unlike AirTrail's UI search,
+ # since AeroDataBox's API doesn't support filtering by them server-side
+ # anyway - see the PR that added client-side from/to filtering upstream).
+ flight = data[0]
+ dep = flight.get("departure") or {}
+ arr = flight.get("arrival") or {}
+
+ return {
+ "departureScheduled": _parse_local((dep.get("scheduledTime") or {}).get("local")),
+ "departure": _parse_local(
+ (dep.get("actualTime") or {}).get("local")
+ or (dep.get("revisedTime") or {}).get("local")
+ or (dep.get("scheduledTime") or {}).get("local")
+ ),
+ "arrivalScheduled": _parse_local((arr.get("scheduledTime") or {}).get("local")),
+ "arrival": _parse_local(
+ (arr.get("actualTime") or {}).get("local")
+ or (arr.get("revisedTime") or {}).get("local")
+ or (arr.get("scheduledTime") or {}).get("local")
+ ),
+ "takeoffActual": _parse_local((dep.get("runwayTime") or {}).get("local")),
+ "landingActual": _parse_local((arr.get("runwayTime") or {}).get("local")),
+ }
diff --git a/airtrail_client.py b/airtrail_client.py
index a51b702..962f989 100644
--- a/airtrail_client.py
+++ b/airtrail_client.py
@@ -98,7 +98,10 @@ def build_save_payload(row: dict, existing_id: int | None) -> dict:
payload = {
"from": row["from_iata"],
"to": row["to_iata"],
- "departure": f"{flight_date_str}T00:00:00.000+00:00",
+ # Falls back to the bare date (day precision, no time) exactly like
+ # before if AeroDataBox never returned a gate-actual time - e.g. no
+ # AERODATABOX_API_KEY configured, or the lookup came up empty.
+ "departure": row.get("gate_departure_actual") or f"{flight_date_str}T00:00:00.000+00:00",
"datePrecision": "day",
"flightNumber": flight_number or None,
"airline": row.get("operating_carrier_icao"),
@@ -117,6 +120,23 @@ def build_save_payload(row: dict, existing_id: int | None) -> dict:
],
}
+ # Scheduled/actual gate times from AeroDataBox (scheduler._fetch_gate_times),
+ # and actual takeoff/landing derived from the FlightAware track's first/last
+ # timestamp (scheduler.process_one_flight). Any of these can be missing -
+ # e.g. no AERODATABOX_API_KEY set, the lookup came up empty, or no track was
+ # found - in which case the field is simply omitted and AirTrail's own
+ # manual Search remains the fallback, same as before this change.
+ if row.get("gate_departure_scheduled"):
+ payload["departureScheduled"] = row["gate_departure_scheduled"]
+ if row.get("gate_arrival_scheduled"):
+ payload["arrivalScheduled"] = row["gate_arrival_scheduled"]
+ if row.get("gate_arrival_actual"):
+ payload["arrival"] = row["gate_arrival_actual"]
+ if row.get("takeoff_actual"):
+ payload["takeoffActual"] = row["takeoff_actual"]
+ if row.get("landing_actual"):
+ payload["landingActual"] = row["landing_actual"]
+
coordinates = row.get("track_coordinates")
if coordinates:
times = row.get("track_times")
diff --git a/config.py b/config.py
index e497ae4..25f93bc 100644
--- a/config.py
+++ b/config.py
@@ -13,6 +13,12 @@ PUBLIC_BASE_URL = os.environ.get("PUBLIC_BASE_URL", "").rstrip("/")
AIRTRAIL_BASE_URL = os.environ.get("AIRTRAIL_BASE_URL", "").rstrip("/")
AIRTRAIL_API_KEY = os.environ.get("AIRTRAIL_API_KEY", "")
+# RapidAPI AeroDataBox key - same provider/plan AirTrail's own "Search"
+# button uses internally to fill in schedule/actual gate times. Optional:
+# if unset, gate scheduled/actual times are simply left for AirTrail's own
+# manual Search to fill in, same as today.
+AERODATABOX_API_KEY = os.environ.get("AERODATABOX_API_KEY", "")
+
FLIGHTAWARE_USER_AGENT = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/124.0 Safari/537.36"
diff --git a/db.py b/db.py
index 3cd82ff..5e4a2cc 100644
--- a/db.py
+++ b/db.py
@@ -33,6 +33,13 @@ CREATE TABLE IF NOT EXISTS pending_flights (
track_times TEXT,
track_point_count INTEGER,
flightaware_url TEXT,
+ takeoff_actual TEXT,
+ landing_actual TEXT,
+ gate_departure_scheduled TEXT,
+ gate_departure_actual TEXT,
+ gate_arrival_scheduled TEXT,
+ gate_arrival_actual TEXT,
+ schedule_error TEXT,
airtrail_flight_id INTEGER,
parse_warnings TEXT,
error_message TEXT,
@@ -68,6 +75,29 @@ def connect():
conn.close()
+# Columns added after the original schema. CREATE TABLE IF NOT EXISTS above
+# is a no-op against an already-existing table, so anything added there
+# needs a matching entry here too or it silently never appears on an
+# upgraded (vs. freshly created) database.
+MIGRATIONS = [
+ ("takeoff_actual", "TEXT"),
+ ("landing_actual", "TEXT"),
+ ("gate_departure_scheduled", "TEXT"),
+ ("gate_departure_actual", "TEXT"),
+ ("gate_arrival_scheduled", "TEXT"),
+ ("gate_arrival_actual", "TEXT"),
+ ("schedule_error", "TEXT"),
+]
+
+
+def _run_migrations(conn: sqlite3.Connection) -> None:
+ existing = {row["name"] for row in conn.execute("PRAGMA table_info(pending_flights)")}
+ for column, coltype in MIGRATIONS:
+ if column not in existing:
+ conn.execute(f"ALTER TABLE pending_flights ADD COLUMN {column} {coltype}")
+
+
def init_db() -> None:
with connect() as conn:
conn.executescript(SCHEMA)
+ _run_migrations(conn)
diff --git a/scheduler.py b/scheduler.py
index ab2e70b..cb77644 100644
--- a/scheduler.py
+++ b/scheduler.py
@@ -11,8 +11,9 @@ review page's "re-query" button - same function, so a corrected field
import logging
import threading
import time
-from datetime import date, datetime, timedelta
+from datetime import date, datetime, timedelta, timezone
+import aerodatabox
import config
import flightaware
import models
@@ -34,6 +35,41 @@ def next_run_datetime(hhmm: str, now: datetime | None = None) -> datetime:
return candidate
+def _iso_from_unix(ts: int) -> str:
+ return datetime.fromtimestamp(ts, tz=timezone.utc).isoformat(timespec="seconds")
+
+
+def _fetch_gate_times(flight_number: str, flight_date: date, updates: dict) -> None:
+ """Best-effort AeroDataBox lookup for scheduled/actual GATE times
+ (departure/arrival) - the data BCBP and the FlightAware track can't
+ provide. Failures here are recorded (schedule_error) but never block
+ the FlightAware track or push status to track_unavailable: worst case
+ the gate fields stay blank exactly like they do today, and the
+ existing fallback (AirTrail's own manual Search) still works."""
+ try:
+ gate_times = aerodatabox.lookup_gate_times(flight_number, flight_date)
+ except aerodatabox.AeroDataBoxError as e:
+ updates["schedule_error"] = str(e)
+ return
+
+ updates["schedule_error"] = None
+ if not gate_times:
+ return
+
+ updates["gate_departure_scheduled"] = gate_times.get("departureScheduled")
+ updates["gate_departure_actual"] = gate_times.get("departure")
+ updates["gate_arrival_scheduled"] = gate_times.get("arrivalScheduled")
+ updates["gate_arrival_actual"] = gate_times.get("arrival")
+ # AeroDataBox's runwayTime is an authoritative actual takeoff/landing
+ # time - prefer it. Only set here if present; the FlightAware-track
+ # fallback below fills in behind it (checks "not in updates") rather
+ # than overwriting it unconditionally.
+ if gate_times.get("takeoffActual"):
+ updates["takeoff_actual"] = gate_times["takeoffActual"]
+ if gate_times.get("landingActual"):
+ updates["landing_actual"] = gate_times["landingActual"]
+
+
def process_one_flight(row: dict) -> None:
flight_id = row["id"]
@@ -45,6 +81,19 @@ def process_one_flight(row: dict) -> None:
updates = {"from_icao": from_icao, "to_icao": to_icao, "operating_carrier_icao": carrier_icao}
+ try:
+ flight_date = date.fromisoformat(row["flight_date"])
+ except (TypeError, ValueError):
+ flight_date = None
+
+ # Gate times only need a flight number + date (no ICAO resolution), so
+ # attempt this regardless of whether the airport/carrier lookups below
+ # succeed - an unresolved airport shouldn't cost us data we could
+ # otherwise have gotten.
+ iata_flight_number = f"{row.get('operating_carrier_iata') or ''}{row.get('flight_number') or ''}"
+ if iata_flight_number.strip() and flight_date:
+ _fetch_gate_times(iata_flight_number, flight_date, updates)
+
if not from_icao or not to_icao or not carrier_icao:
missing = [
name
@@ -57,9 +106,7 @@ def process_one_flight(row: dict) -> None:
notifier.notify_flight_staged(models.get_flight(flight_id))
return
- try:
- flight_date = date.fromisoformat(row["flight_date"])
- except (TypeError, ValueError):
+ if flight_date is None:
updates["status"] = "track_unavailable"
updates["error_message"] = f"invalid flight_date: {row.get('flight_date')!r}"
models.update_flight(flight_id, **updates)
@@ -98,6 +145,16 @@ def process_one_flight(row: dict) -> None:
updates["track_times"] = times
updates["track_point_count"] = len(coordinates)
updates["error_message"] = None
+ # The ADS-B track only exists while the aircraft is squawking -
+ # roughly wheels-up to touchdown - so its first/last timestamps are
+ # a decent proxy for actual takeoff/landing when AeroDataBox didn't
+ # already give us its authoritative runwayTime above (no key
+ # configured, lookup failed, or that particular flight had no
+ # runwayTime in its response). Never overwrite a value AeroDataBox
+ # already set - that one's more precise.
+ if times:
+ updates.setdefault("takeoff_actual", _iso_from_unix(times[0]))
+ updates.setdefault("landing_actual", _iso_from_unix(times[-1]))
models.update_flight(flight_id, **updates)
notifier.notify_flight_staged(models.get_flight(flight_id))
diff --git a/templates/review_detail.html b/templates/review_detail.html
index f6ac0cc..b21e042 100644
--- a/templates/review_detail.html
+++ b/templates/review_detail.html
@@ -61,6 +61,21 @@
{% if flight.track_point_count %}
<dt>Track points</dt><dd>{{ flight.track_point_count }}</dd>
{% endif %}
+ {% if flight.gate_departure_scheduled or flight.gate_departure_actual %}
+ <dt>Gate departure</dt><dd>sched {{ flight.gate_departure_scheduled or '—' }} / actual {{ flight.gate_departure_actual or '—' }}</dd>
+ {% endif %}
+ {% if flight.gate_arrival_scheduled or flight.gate_arrival_actual %}
+ <dt>Gate arrival</dt><dd>sched {{ flight.gate_arrival_scheduled or '—' }} / actual {{ flight.gate_arrival_actual or '—' }}</dd>
+ {% endif %}
+ {% if flight.takeoff_actual %}
+ <dt>Takeoff (actual)</dt><dd>{{ flight.takeoff_actual }}</dd>
+ {% endif %}
+ {% if flight.landing_actual %}
+ <dt>Landing (actual)</dt><dd>{{ flight.landing_actual }}</dd>
+ {% endif %}
+ {% if flight.schedule_error %}
+ <dt>Schedule lookup</dt><dd class="muted">{{ flight.schedule_error }}</dd>
+ {% endif %}
{% if flight.airtrail_flight_id %}
<dt>AirTrail flight</dt><dd>{{ flight.airtrail_flight_id }}</dd>
{% endif %}