RTP: resume at access-unit boundary after packet loss, not NAL start
After a sequence gap the H.264/H.265 depacketizers resumed reassembly at
the next NAL/FU start. On multi-slice streams every slice starts a NAL,
so reassembly resumed MID-FRAME and delivered access units missing their
leading slices — manufacturing "First slice in a frame missing" /
"Ignoring POC change between slices" / "Could not find ref" decoder
errors and green/blocky concealment output far beyond the actual loss.
Resume only at a true AU boundary instead:
H.265 - non-VCL NAL (type >= 32) or VCL slice with
first_slice_segment_in_pic_flag set (first bit after the
2-byte NAL header)
H.264 - non-VCL NAL (type >= 6) or VCL slice with
first_mb_in_slice == 0 (ue(v): MSB set after 1-byte header)
Also: stderr visibility for RTP sequence gaps (rtp_rx logs went only to
the sys_log file) and for av_log-bridge latches in video_decoder — the
global av_log level filters decoder errors from the console, which made
keyframe-gate arms look causeless during diagnosis.
Live verification, HEVC camera on lossy Wi-Fi (NVDEC decode):
before: 271 decoder errors / 3 min, delivery collapsed to ~2 fps
after: 3 errors / 2 min, full ~21 fps, zero corrupt frames displayed
Remaining rare errors show POC ref misses with ZERO RTP sequence gaps,
i.e. pre-decode queue eviction under TCP burst clumps - a separate,
much smaller issue absorbed invisibly by the keyframe gate.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -43,10 +43,24 @@ namespace {
|
|||||||
void ansAvLogBridge(void* ptr, int level, const char* fmt, va_list vl)
|
void ansAvLogBridge(void* ptr, int level, const char* fmt, va_list vl)
|
||||||
{
|
{
|
||||||
if (ptr && level <= AV_LOG_ERROR) {
|
if (ptr && level <= AV_LOG_ERROR) {
|
||||||
|
bool latched = false;
|
||||||
|
{
|
||||||
std::lock_guard<std::mutex> lk(s_logBridgeMutex);
|
std::lock_guard<std::mutex> lk(s_logBridgeMutex);
|
||||||
auto it = s_logBridgeMap.find(ptr);
|
auto it = s_logBridgeMap.find(ptr);
|
||||||
if (it != s_logBridgeMap.end()) {
|
if (it != s_logBridgeMap.end()) {
|
||||||
it->second->store(true, std::memory_order_relaxed);
|
it->second->store(true, std::memory_order_relaxed);
|
||||||
|
latched = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (latched) {
|
||||||
|
// Show exactly which decoder-context error latched the corrupt
|
||||||
|
// flag — av_log_default_callback may filter it out (global log
|
||||||
|
// level), which would otherwise make gate arms look causeless.
|
||||||
|
va_list vc;
|
||||||
|
va_copy(vc, vl);
|
||||||
|
fprintf(stderr, "[FFmpeg-latch] ");
|
||||||
|
vfprintf(stderr, fmt, vc);
|
||||||
|
va_end(vc);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
av_log_default_callback(ptr, level, fmt, vl);
|
av_log_default_callback(ptr, level, fmt, vl);
|
||||||
@@ -1176,8 +1190,9 @@ BOOL CVideoDecoder::readFrame()
|
|||||||
// • m_bCorruptSeen — latched by the av_log bridge; the ONLY signal
|
// • m_bCorruptSeen — latched by the av_log bridge; the ONLY signal
|
||||||
// on the hwaccel (NVDEC) path, which logs the error but leaves
|
// on the hwaccel (NVDEC) path, which logs the error but leaves
|
||||||
// decode_error_flags at 0 (verified empirically).
|
// decode_error_flags at 0 (verified empirically).
|
||||||
const bool frameCorrupt = (m_pFrame->decode_error_flags != 0)
|
const bool corruptByFlags = (m_pFrame->decode_error_flags != 0);
|
||||||
|| m_bCorruptSeen.exchange(false, std::memory_order_relaxed);
|
const bool corruptByBridge = m_bCorruptSeen.exchange(false, std::memory_order_relaxed);
|
||||||
|
const bool frameCorrupt = corruptByFlags || corruptByBridge;
|
||||||
if (frameCorrupt) {
|
if (frameCorrupt) {
|
||||||
if (!m_bWaitKeyframe.load(std::memory_order_relaxed)) {
|
if (!m_bWaitKeyframe.load(std::memory_order_relaxed)) {
|
||||||
m_bWaitKeyframe.store(true, std::memory_order_relaxed);
|
m_bWaitKeyframe.store(true, std::memory_order_relaxed);
|
||||||
@@ -1188,7 +1203,8 @@ BOOL CVideoDecoder::readFrame()
|
|||||||
// Also to stderr so it appears next to FFmpeg's own [hevc @ ...]
|
// Also to stderr so it appears next to FFmpeg's own [hevc @ ...]
|
||||||
// error lines (log_print goes to the log file, which unit tests
|
// error lines (log_print goes to the log file, which unit tests
|
||||||
// and field debugging sessions usually don't have open).
|
// and field debugging sessions usually don't have open).
|
||||||
fprintf(stderr, "[MediaClient] corrupt frame detected -- gating until next keyframe\n");
|
fprintf(stderr, "[MediaClient] corrupt frame detected (flags=0x%x bridge=%d) -- gating until next keyframe\n",
|
||||||
|
m_pFrame->decode_error_flags, (int)corruptByBridge);
|
||||||
ANS_DBG("MEDIA_Decode",
|
ANS_DBG("MEDIA_Decode",
|
||||||
"corrupt frame (decode_error_flags=0x%x) — re-arming keyframe gate",
|
"corrupt frame (decode_error_flags=0x%x) — re-arming keyframe gate",
|
||||||
m_pFrame->decode_error_flags);
|
m_pFrame->decode_error_flags);
|
||||||
|
|||||||
@@ -197,17 +197,32 @@ BOOL h264_data_rx(H264RXI * p_rxi, uint8 * p_data, int len)
|
|||||||
|
|
||||||
if (p_rxi->rtprxi.rxf_loss)
|
if (p_rxi->rtprxi.rxf_loss)
|
||||||
{
|
{
|
||||||
// Packet loss, until the next start packet
|
// Packet loss — resume ONLY at an access-unit boundary (see the
|
||||||
|
// H.265 counterpart in h265_rtp_rx.cpp for the full rationale).
|
||||||
if (!beginPacket)
|
// Resuming at a mere NAL/FU start delivers frames missing their
|
||||||
|
// leading slices on multi-slice streams. An AU boundary is a
|
||||||
|
// non-VCL NAL (SEI/SPS/PPS/AUD, type >= 6) or a VCL slice whose
|
||||||
|
// first_mb_in_slice == 0 — ue(v)-coded, so value 0 is a single
|
||||||
|
// '1' bit: MSB of the byte after the 1-byte NAL header.
|
||||||
|
BOOL auStart = FALSE;
|
||||||
|
if (beginPacket)
|
||||||
{
|
{
|
||||||
|
uint8* payload = headerStart + numBytesToSkip;
|
||||||
|
int payloadLen = (int)packetSize - (int)numBytesToSkip;
|
||||||
|
if (payloadLen >= 2)
|
||||||
|
{
|
||||||
|
uint8 nt = payload[0] & 0x1F;
|
||||||
|
if (nt >= 6) auStart = TRUE; // non-VCL
|
||||||
|
else if (payload[1] & 0x80) auStart = TRUE; // first_mb_in_slice == 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!auStart)
|
||||||
|
{
|
||||||
|
p_rxi->d_offset = 0;
|
||||||
return FALSE;
|
return FALSE;
|
||||||
}
|
}
|
||||||
else
|
|
||||||
{
|
|
||||||
p_rxi->rtprxi.rxf_loss = 0;
|
p_rxi->rtprxi.rxf_loss = 0;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// save the H264 parameter sets
|
// save the H264 parameter sets
|
||||||
|
|
||||||
|
|||||||
@@ -265,17 +265,36 @@ BOOL h265_data_rx(H265RXI * p_rxi, uint8 * p_data, int len)
|
|||||||
|
|
||||||
if (p_rxi->rtprxi.rxf_loss)
|
if (p_rxi->rtprxi.rxf_loss)
|
||||||
{
|
{
|
||||||
// Packet loss, until the next start packet
|
// Packet loss — resume ONLY at an access-unit boundary.
|
||||||
|
//
|
||||||
if (!beginPacket)
|
// The old behaviour resumed at the next NAL/FU start, but with
|
||||||
|
// multi-slice frames every slice starts a NAL, so reassembly resumed
|
||||||
|
// MID-FRAME and delivered access units missing their leading slices.
|
||||||
|
// The decoder then logs "First slice in a frame missing" / "Ignoring
|
||||||
|
// POC change between slices" and conceals the damage as green/blocky
|
||||||
|
// garbage. An AU boundary is either a non-VCL NAL (VPS/SPS/PPS/SEI,
|
||||||
|
// type >= 32, which precede an IDR) or a VCL slice with
|
||||||
|
// first_slice_segment_in_pic_flag set — the first bit after the
|
||||||
|
// 2-byte NAL header (H.265 spec 7.3.6.1).
|
||||||
|
BOOL auStart = FALSE;
|
||||||
|
if (beginPacket)
|
||||||
{
|
{
|
||||||
|
uint8* payload = headerStart + numBytesToSkip;
|
||||||
|
int payloadLen = (int)packetSize - (int)numBytesToSkip;
|
||||||
|
if (payloadLen >= 3)
|
||||||
|
{
|
||||||
|
uint8 nt = (payload[0] >> 1) & 0x3F;
|
||||||
|
if (nt >= 32) auStart = TRUE; // non-VCL (param sets/SEI)
|
||||||
|
else if (payload[2] & 0x80) auStart = TRUE; // first slice in picture
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!auStart)
|
||||||
|
{
|
||||||
|
p_rxi->d_offset = 0;
|
||||||
return FALSE;
|
return FALSE;
|
||||||
}
|
}
|
||||||
else
|
|
||||||
{
|
|
||||||
p_rxi->rtprxi.rxf_loss = 0;
|
p_rxi->rtprxi.rxf_loss = 0;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// save the H265 parameter sets
|
// save the H265 parameter sets
|
||||||
|
|
||||||
|
|||||||
@@ -106,6 +106,11 @@ BOOL rtp_data_rx(RTPRXI * p_rxi, uint8 * p_data, int len)
|
|||||||
{
|
{
|
||||||
p_rxi->rxf_loss = 1;
|
p_rxi->rxf_loss = 1;
|
||||||
log_print(HT_LOG_WARN, "%s, prev seq[%u], cur seq[%u]!!!\r\n", __FUNCTION__, p_rxi->seq, rtpSeq);
|
log_print(HT_LOG_WARN, "%s, prev seq[%u], cur seq[%u]!!!\r\n", __FUNCTION__, p_rxi->seq, rtpSeq);
|
||||||
|
// stderr too (log_print goes to the log file): over TCP transport a
|
||||||
|
// sequence gap means the SENDER (camera) skipped packets — usually
|
||||||
|
// backpressure from a slow receiver. Gap size shows how much vanished.
|
||||||
|
fprintf(stderr, "[RTP] seq gap: prev=%u cur=%u (lost ~%u pkt)\n",
|
||||||
|
p_rxi->seq, rtpSeq, (uint16)(rtpSeq - p_rxi->seq - 1));
|
||||||
}
|
}
|
||||||
p_rxi->seq = rtpSeq;
|
p_rxi->seq = rtpSeq;
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user