From 8a6cb53e3b429678965ea2dea0b7a354263df26e Mon Sep 17 00:00:00 2001 From: Ben Nicholson Date: Fri, 25 Sep 2026 14:48:12 +1000 Subject: [PATCH] Step 7.3: decode HLS TS segments to PCM (own TS demux + AAC decoder) - main/decoder: MPEG-TS demux (packets reassembled across HTTP chunks, PAT -> PMT -> first ADTS-AAC PID, PES headers stripped) feeding the esp_audio_codec simple AAC decoder (ADTS, AAC-Plus enabled for HE-AAC v1/v2 variants). 7.3 only counts and logs PCM per segment. - esp_audio_codec pinned to ~2.5.0: 2.6+ needs P4 rev >= 3 (this board is rev 1.3). Noted in CLAUDE.md, also for esp_audio_effects < 1.4. - The library's combined TS decoder lost ~8% of the frames (segments decoded to 7.9-9.6 s, "decode error -1"); with the own demux every segment is sample exact: 441344 / 440320 frames = 10.008 / 9.985 s, matching EXTINF 10.0078 / 9.9846 (431 / 430 AAC frames), no errors. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 1 + dependencies.lock | 13 +++- main/CMakeLists.txt | 2 +- main/decoder.c | 172 +++++++++++++++++++++++++++++++++++++++++ main/decoder.h | 12 +++ main/idf_component.yml | 3 + main/player.c | 7 +- 7 files changed, 207 insertions(+), 3 deletions(-) create mode 100644 main/decoder.c create mode 100644 main/decoder.h create mode 100644 main/idf_component.yml diff --git a/CLAUDE.md b/CLAUDE.md index 61bb5ce..6a3f5be 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -25,6 +25,7 @@ Repo: https://gitea.apointless.space/bsncubed/aes67-ESP32-P4 - Chip revision < v3.0 (engineering silicon, seen on some of these boards) needs `CONFIG_ESP32P4_SELECTS_REV_LESS_V3=y`. Our board: **v1.3** (set in sdkconfig.defaults, min rev v1.0). - Our board: **32 MB** flash (GigaDevice c8/4019). App slots must stay below 16 MB (cache mapping above 16 MB is experimental in IDF). + - Rev < 3 also limits Espressif's prebuilt audio libraries: `esp_audio_codec` must stay < 2.6 and `esp_audio_effects` < 1.4 (newer versions use P4 assembly that needs rev >= 3; the build fails with a message saying so). Check this for any new Espressif binary component. - Embed `web/index.html` via `EMBED_TXTFILES` in `aes67_web`. - Flash over the network (normal way since step 2b; keep USB for recovery): `curl -f --data-binary @build/aes67_p4.bin -H 'Content-Type: application/octet-stream' http://p4-aes67/api/ota` diff --git a/dependencies.lock b/dependencies.lock index d432dcc..b00d459 100644 --- a/dependencies.lock +++ b/dependencies.lock @@ -1,4 +1,14 @@ dependencies: + espressif/esp_audio_codec: + component_hash: 16e2880dbdd5a72264051f750f33a6e5a38fd25621c83e3a32a85a152d2d2643 + dependencies: + - name: idf + require: private + version: '>=4.4' + source: + registry_url: https://components.espressif.com/ + type: service + version: 2.5.0 espressif/mdns: component_hash: b679eafd0acae2066e2645bd91d073e33ca5be515e2545cd3c4e55a3e3bce3cb dependencies: @@ -14,7 +24,8 @@ dependencies: type: idf version: 5.5.5 direct_dependencies: +- espressif/esp_audio_codec - espressif/mdns -manifest_hash: e5eba11c864076c14a58153c7f761bec2b064244e66ca3f76c0dc170db907329 +manifest_hash: e85f2bff1cc890def1a35e7da5210473c09ec845364a2bc2e54f6d2a7de54cbc target: esp32p4 version: 2.0.0 diff --git a/main/CMakeLists.txt b/main/CMakeLists.txt index 535cf9a..1f2cce8 100644 --- a/main/CMakeLists.txt +++ b/main/CMakeLists.txt @@ -1,4 +1,4 @@ -idf_component_register(SRCS "main.c" "project_cfg.c" "player.c" "audio_ring.c" "hls.c" +idf_component_register(SRCS "main.c" "project_cfg.c" "player.c" "audio_ring.c" "hls.c" "decoder.c" INCLUDE_DIRS "." REQUIRES esp_app_format esp_hw_support heap esp_http_client mbedtls esp_timer aes67_board aes67_health aes67_net aes67_ota aes67_ptp aes67_sdp_sap diff --git a/main/decoder.c b/main/decoder.c new file mode 100644 index 0000000..2571fa6 --- /dev/null +++ b/main/decoder.c @@ -0,0 +1,172 @@ +#include "decoder.h" + +#include +#include + +#include "esp_audio_dec_default.h" +#include "esp_audio_simple_dec.h" +#include "esp_audio_simple_dec_default.h" +#include "esp_heap_caps.h" +#include "esp_log.h" + +static const char *TAG = "decoder"; + +static esp_audio_simple_dec_handle_t s_dec; +static uint8_t *s_pcm; +static uint32_t s_pcm_size = 8192; +static esp_audio_simple_dec_info_t s_info; +static uint64_t s_seg_frames, s_seg_count, s_seg_es; + +// MPEG-TS demux state: packets reassembled across HTTP chunks, PAT -> PMT -> ADTS-AAC PID. +static uint8_t s_pkt[188]; +static size_t s_pkt_len; +static int s_pmt_pid = -1, s_audio_pid = -1; + +static void report_segment(void) +{ + if (s_seg_count && s_info.sample_rate) { + ESP_LOGI(TAG, "segment: %llu ES bytes -> %llu PCM frames = %.3f s (%lu Hz, %u ch, %u bit)", + s_seg_es, s_seg_frames, (double)s_seg_frames / s_info.sample_rate, + (unsigned long)s_info.sample_rate, s_info.channel, s_info.bits_per_sample); + } + s_seg_frames = 0; + s_seg_es = 0; + s_seg_count++; +} + +// Feed ADTS-AAC elementary stream bytes to the decoder. +static void decode_es(const uint8_t *data, size_t len) +{ + s_seg_es += len; + esp_audio_simple_dec_raw_t raw = { .buffer = (uint8_t *)data, .len = len }; + while (raw.len) { + esp_audio_simple_dec_out_t out = { .buffer = s_pcm, .len = s_pcm_size }; + esp_audio_err_t err = esp_audio_simple_dec_process(s_dec, &raw, &out); + if (err == ESP_AUDIO_ERR_BUFF_NOT_ENOUGH) { + uint8_t *p = heap_caps_realloc(s_pcm, out.needed_size, MALLOC_CAP_SPIRAM); + if (!p) { + return; + } + s_pcm = p; + s_pcm_size = out.needed_size; + continue; + } + if (err != ESP_AUDIO_ERR_OK) { + ESP_LOGW(TAG, "decode error %d, resetting decoder", err); + esp_audio_simple_dec_reset(s_dec); + return; + } + if (out.decoded_size) { + if (!s_info.sample_rate) { + esp_audio_simple_dec_get_info(s_dec, &s_info); + ESP_LOGI(TAG, "stream: %lu Hz, %u ch, %u bit, %lu bit/s", (unsigned long)s_info.sample_rate, + s_info.channel, s_info.bits_per_sample, (unsigned long)s_info.bitrate); + } + s_seg_frames += out.decoded_size / (s_info.channel * s_info.bits_per_sample / 8); + } + raw.buffer += raw.consumed; + raw.len -= raw.consumed; + } +} + +static void ts_packet(const uint8_t *p) +{ + if (p[0] != 0x47) { + return; + } + bool pusi = p[1] & 0x40; + int pid = ((p[1] & 0x1f) << 8) | p[2]; + int afc = (p[3] >> 4) & 3; + if (!(afc & 1)) { + return; // no payload + } + size_t off = 4 + ((afc & 2) ? 1 + p[4] : 0); + if (off >= 188) { + return; + } + const uint8_t *pl = p + off; + size_t len = 188 - off; + + if (pid == 0 && pusi) { // PAT: first program's PMT PID + const uint8_t *sec = pl + 1 + pl[0]; + s_pmt_pid = ((sec[10] & 0x1f) << 8) | sec[11]; + } else if (pid == s_pmt_pid && pusi && s_audio_pid < 0) { // PMT: first ADTS-AAC stream + const uint8_t *sec = pl + 1 + pl[0]; + int slen = ((sec[1] & 0x0f) << 8) | sec[2]; + int i = 12 + (((sec[10] & 0x0f) << 8) | sec[11]); + while (i + 5 <= 3 + slen - 4 && sec + i + 5 <= p + 188) { + int type = sec[i], epid = ((sec[i + 1] & 0x1f) << 8) | sec[i + 2]; + if (type == 0x0F) { + s_audio_pid = epid; + ESP_LOGI(TAG, "TS: PMT PID 0x%x, ADTS-AAC on PID 0x%x", s_pmt_pid, epid); + break; + } + i += 5 + (((sec[i + 3] & 0x0f) << 8) | sec[i + 4]); + } + } else if (pid == s_audio_pid) { + if (pusi) { // strip the PES header + if (len < 9 || pl[0] || pl[1] || pl[2] != 1 || 9u + pl[8] > len) { + return; + } + size_t h = 9 + pl[8]; + pl += h; + len -= h; + } + decode_es(pl, len); + } +} + +// hls_sink_t: MPEG-TS bytes in any chunking. +bool decoder_feed(const uint8_t *data, size_t len, bool segment_start) +{ + if (segment_start) { + report_segment(); + s_pkt_len = 0; // segments start on a packet boundary + } + while (len) { + if (!s_pkt_len) { + // resync on 0x47 if needed, then take whole packets straight from the input + while (len && data[0] != 0x47) { + data++; + len--; + } + while (len >= 188 && data[0] == 0x47) { + ts_packet(data); + data += 188; + len -= 188; + } + if (len && data[0] != 0x47) { + continue; + } + } + size_t n = 188 - s_pkt_len < len ? 188 - s_pkt_len : len; + memcpy(s_pkt + s_pkt_len, data, n); + s_pkt_len += n; + data += n; + len -= n; + if (s_pkt_len == 188) { + ts_packet(s_pkt); + s_pkt_len = 0; + } + } + return true; +} + +esp_err_t decoder_init(void) +{ + esp_audio_dec_register_default(); + esp_audio_simple_dec_register_default(); + // Own TS demux (the library's TS decoder lost ~8% of the frames); AAC with ADTS headers, + // AAC-Plus on so HE-AAC v1/v2 variants work too. + esp_aac_dec_cfg_t aac_cfg = ESP_AAC_DEC_CONFIG_DEFAULT(); + aac_cfg.aac_plus_enable = true; + esp_audio_simple_dec_cfg_t cfg = { + .dec_type = ESP_AUDIO_SIMPLE_DEC_TYPE_AAC, .dec_cfg = &aac_cfg, .cfg_size = sizeof(aac_cfg), + }; + if (esp_audio_simple_dec_open(&cfg, &s_dec) != ESP_AUDIO_ERR_OK) { + ESP_LOGE(TAG, "AAC decoder open failed"); + return ESP_FAIL; + } + s_pcm = heap_caps_malloc(s_pcm_size, MALLOC_CAP_SPIRAM); + return s_pcm ? ESP_OK : ESP_ERR_NO_MEM; +} diff --git a/main/decoder.h b/main/decoder.h new file mode 100644 index 0000000..562b9de --- /dev/null +++ b/main/decoder.h @@ -0,0 +1,12 @@ +// Decoder for HLS segments (MPEG-TS with AAC-LC / HE-AAC), fed by the HLS client's sink. +#pragma once + +#include +#include +#include + +#include "esp_err.h" + +esp_err_t decoder_init(void); +// hls_sink_t: segment bytes in; PCM out (7.3: counted and logged). +bool decoder_feed(const uint8_t *data, size_t len, bool segment_start); diff --git a/main/idf_component.yml b/main/idf_component.yml new file mode 100644 index 0000000..7c0a874 --- /dev/null +++ b/main/idf_component.yml @@ -0,0 +1,3 @@ +dependencies: + # 2.6+ uses P4 assembly that needs chip rev >= 3.0; this board is rev 1.3 (see CLAUDE.md). + espressif/esp_audio_codec: "~2.5.0" diff --git a/main/player.c b/main/player.c index 84ea666..268c676 100644 --- a/main/player.c +++ b/main/player.c @@ -7,6 +7,7 @@ #include "aes67_tx.h" #include "aes67_web.h" #include "audio_ring.h" +#include "decoder.h" #include "hls.h" #include "esp_log.h" #include "freertos/FreeRTOS.h" @@ -113,7 +114,11 @@ esp_err_t player_init(void) player_apply(src); cJSON_Delete(src); xTaskCreate(test_producer, "test_prod", 3072, NULL, 5, NULL); - err = hls_start(NULL); // 7.2: fetch and log only; decoding in 7.3 + err = decoder_init(); + if (err != ESP_OK) { + return err; + } + err = hls_start(decoder_feed); // 7.3: decode and log; resampling + ring in 7.4 if (err != ESP_OK) { return err; }