From 63440621791ae99ca19575537562a12ff35935bb Mon Sep 17 00:00:00 2001 From: Tuan Nghia Nguyen Date: Mon, 6 Jul 2026 21:49:22 +1000 Subject: [PATCH] MediaClient: keyframe gating (startup + corruption re-arm), libyuv NV12 path Restores the previously shelved decoder/player hardening and extends it: 1. Startup keyframe gating (video_decoder): after (re)start the decoder joins the stream mid-GOP; frames referencing unseen pictures render as green/blocky concealment garbage. Drop everything until the first IDR (300-frame safety valve so unmarked streams can't blank forever). 2. NEW - corruption re-arm: on lossy links (Wi-Fi cameras) dropped slices surface as "First slice in a frame missing" / "Could not find ref with POC n" and FFmpeg marks the output via AVFrame::decode_error_flags. Re-arm the keyframe gate and drop the rest of the GOP so the display holds the last clean frame instead of showing concealment artifacts; corrupt drops count toward the safety valve. 3. libyuv NV12->BGR conversion (video_player): honors NVDEC/D3D11VA padded strides (1080->1088 rows) that cv::cvtColorTwoPlane mishandled (green cast + striped noise), picks BT.601/709/2020 coefficients from the frame's signalled colorspace, and is ~3-5x faster than sws_scale. Verified live against an RTSP HEVC camera (NVDEC path): steady frame flow, no regressions on the clean-stream path. Co-Authored-By: Claude Fable 5 --- MediaClient/media/video_decoder.cpp | 76 ++++++++++++++++++++++++++ MediaClient/media/video_decoder.h | 8 +++ MediaClient/media/video_player.cpp | 85 ++++++++++++++++++++++++++++- MediaClient/media/video_player.h | 7 +++ 4 files changed, 173 insertions(+), 3 deletions(-) diff --git a/MediaClient/media/video_decoder.cpp b/MediaClient/media/video_decoder.cpp index d84b3e9..394fed0 100644 --- a/MediaClient/media/video_decoder.cpp +++ b/MediaClient/media/video_decoder.cpp @@ -761,6 +761,10 @@ void CVideoDecoder::Start() { if (m_pContext) { avcodec_flush_buffers(m_pContext); } + // Re-arm keyframe gating: after a flush the DPB is empty, so wait for the + // next IDR before delivering frames again (avoids post-restart green/blocky). + m_bWaitKeyframe = true; + m_nKeyframeWaitCount = 0; m_bRunning = TRUE; log_print(HT_LOG_INFO, "%s, Video decoder started\r\n", __FUNCTION__); } @@ -1093,6 +1097,78 @@ BOOL CVideoDecoder::readFrame() continue; } + // Mid-stream corruption gating — re-arm the keyframe gate on decode errors. + // + // Packet loss on lossy links (Wi-Fi cameras) drops slices mid-GOP: + // FFmpeg logs "First slice in a frame missing" / "Could not find ref + // with POC n", conceals the damage, and marks the output frame via + // AVFrame::decode_error_flags (missing refs, concealment active, ...). + // Every subsequent frame in the GOP references the concealed one, so + // rather than display green/blocky garbage until the next IDR, drop + // the frame and re-arm the keyframe gate — the display holds the last + // clean frame and resumes on the next clean keyframe. + if (m_pFrame->decode_error_flags != 0) { + if (!m_bWaitKeyframe.load(std::memory_order_relaxed)) { + m_bWaitKeyframe.store(true, std::memory_order_relaxed); + m_nKeyframeWaitCount = 0; + ANS_DBG("MEDIA_Decode", + "corrupt frame (decode_error_flags=0x%x) — re-arming keyframe gate", + m_pFrame->decode_error_flags); + } else { + // Count corrupt drops toward the safety valve so a stream that + // is corrupt for a long stretch eventually shows *something* + // (concealed frames) instead of freezing indefinitely. + ++m_nKeyframeWaitCount; + } + if (m_nKeyframeWaitCount < 300) { + av_frame_unref(m_pFrame); + continue; + } + // Safety valve tripped: fall through and let the frame pass + // (concealment artifacts beat a permanently frozen display). + m_bWaitKeyframe.store(false, std::memory_order_relaxed); + } + + // Startup keyframe gating — discard everything until the first IDR. + // + // On (re)connect we join the stream mid-GOP. Until a keyframe arrives, + // every frame references pictures we never received; the decoder (and + // NVDEC especially) conceals them into green / blocky garbage and reports + // success, so there is no error flag to test. Waiting for the first + // keyframe gives a clean intra reference, after which inter frames decode + // correctly. This is the standard RTSP "wait for IDR" behaviour. + // + // Keyframe flag is set from the bitstream parse and is reliable for both + // SW and HW decode (it does not depend on the HW surface). pict_type is a + // fallback. The counter is a safety valve: if a stream never advertises a + // keyframe within ~300 frames (~10 s @30fps) we stop gating rather than + // blank the display forever. + if (m_bWaitKeyframe.load(std::memory_order_relaxed)) { + // pict_type==I is version-independent and set for HEVC IDR/I frames + // on both SW and HW decode. AV_FRAME_FLAG_KEY (FFmpeg 5.1+) is added + // when available for robustness; guarded so it also builds against + // the older vendored headers that predate that flag. + bool isKey = (m_pFrame->pict_type == AV_PICTURE_TYPE_I); +#if defined(AV_FRAME_FLAG_KEY) + isKey = isKey || ((m_pFrame->flags & AV_FRAME_FLAG_KEY) != 0); +#endif + if (isKey) { + m_bWaitKeyframe.store(false, std::memory_order_relaxed); + ANS_DBG("MEDIA_Decode", + "keyframe acquired after %d dropped frame(s) — resuming output", + m_nKeyframeWaitCount); + } else if (++m_nKeyframeWaitCount >= 300) { + m_bWaitKeyframe.store(false, std::memory_order_relaxed); + ANS_DBG("MEDIA_Decode", + "no keyframe after %d frames — releasing gate (safety valve)", + m_nKeyframeWaitCount); + } else { + // Pre-IDR frame: drop it (would render as green/blocky garbage). + av_frame_unref(m_pFrame); + continue; + } + } + if (m_pFrame->format == m_hwPixFmt) { // CUDA HW accel: clone the HW frame BEFORE transfer so inference diff --git a/MediaClient/media/video_decoder.h b/MediaClient/media/video_decoder.h index 2d964d5..24d3c14 100644 --- a/MediaClient/media/video_decoder.h +++ b/MediaClient/media/video_decoder.h @@ -165,6 +165,14 @@ private: std::atomic m_bRunning; BOOL m_bHardwareDecoderEnabled; // Track if hardware decoder is enabled bool m_bCudaHWAccel; // true when using AV_HWDEVICE_TYPE_CUDA + // Startup keyframe gating. On connect/reconnect we join the stream mid-GOP, + // before the first IDR, so early frames reference data we never received and + // the decoder conceals them (green / blocky macroblocks). We drop everything + // until the first keyframe (clean reference point), then deliver normally. + // The frame counter is a safety valve so a stream that never advertises a + // keyframe can't blank the display forever. + std::atomic m_bWaitKeyframe{ true }; // true until first IDR after (re)start + int m_nKeyframeWaitCount{ 0 }; // frames dropped while waiting int m_hwGpuIndex; // GPU index assigned by HWDecoderPool (-1 = legacy) AVFrame* m_pCudaHWFrame; // Cloned CUDA HW frame (device ptrs) for inference const AVCodec* m_pCodec; diff --git a/MediaClient/media/video_player.cpp b/MediaClient/media/video_player.cpp index 8534016..7a01250 100644 --- a/MediaClient/media/video_player.cpp +++ b/MediaClient/media/video_player.cpp @@ -1405,6 +1405,80 @@ cv::Mat CVideoPlayer::avframeNV12ToCvMat(const AVFrame* frame) m_nv12OrigWidth = width; m_nv12OrigHeight = height; + // Preferred path: libyuv NV12→RGB24 (BGR in memory, matches cv::Mat). + // SIMD (SSSE3/AVX2) and ~3-5x faster than sws_scale's quality config. + // Crucially it honors src_stride_uv, so the padded strides/height that + // NVDEC and D3D11VA emit (e.g. 1080→1088 rows, aligned strides) convert + // correctly — that padding is what cv::cvtColorTwoPlane mishandled, + // reading the chroma plane at the wrong offset and rendering a green + // cast with striped noise toward the bottom of the frame. + // Coefficients are chosen from the frame's signalled colorspace/range, + // mirroring initNV12SwsContext() (BT.709 for HD/4K, BT.601 for SD, + // BT.2020 if tagged). The constants map studio-range YUV to full-range + // BGR, so the result is display-ready — no manual range expansion. +#if defined(ANSCORE_HAS_LIBYUV) && ANSCORE_HAS_LIBYUV + { + const bool fullRange = (frame->color_range == AVCOL_RANGE_JPEG); + const libyuv::YuvConstants* yuvc; + switch (frame->colorspace) { + case AVCOL_SPC_BT470BG: + case AVCOL_SPC_SMPTE170M: + yuvc = fullRange ? &libyuv::kYuvJPEGConstants : &libyuv::kYuvI601Constants; + break; + case AVCOL_SPC_BT2020_NCL: + case AVCOL_SPC_BT2020_CL: + yuvc = fullRange ? &libyuv::kYuvV2020Constants : &libyuv::kYuv2020Constants; + break; + case AVCOL_SPC_BT709: + yuvc = fullRange ? &libyuv::kYuvF709Constants : &libyuv::kYuvH709Constants; + break; + default: + // Auto-detect by resolution (matches initNV12SwsContext()). + if (width >= 1280 || height >= 720) + yuvc = fullRange ? &libyuv::kYuvF709Constants : &libyuv::kYuvH709Constants; + else + yuvc = fullRange ? &libyuv::kYuvJPEGConstants : &libyuv::kYuvI601Constants; + break; + } + cv::Mat bgrImage(height, width, CV_8UC3); + int ret = libyuv::NV12ToRGB24Matrix( + frame->data[0], frame->linesize[0], + frame->data[1], frame->linesize[1], + bgrImage.data, static_cast(bgrImage.step), + yuvc, width, height); + if (ret == 0) { + return bgrImage; + } + ANS_DBG("MEDIA_Convert", + "libyuv NV12ToRGB24 failed (ret=%d) — falling back to sws_scale", ret); + } +#endif + + // Fallback path: sws_scale via the dedicated NV12→BGR24 context. Also + // stride-safe (honors frame->linesize[]) and colorspace/range-correct; + // used on non-libyuv builds or if libyuv ever reports an error. + { + // Dedicated lock (not _mutex) so close()/dtor cannot free the context + // mid-scale, while still keeping onVideoFrame off the hot _mutex. + std::lock_guard nv12Lock(m_nv12SwsMutex); + initNV12SwsContext(frame); + if (m_nv12SwsCtx) { + cv::Mat bgrImage(height, width, CV_8UC3); + uint8_t* dst[1] = { bgrImage.data }; + int dstStride[1] = { static_cast(bgrImage.step[0]) }; + int ret = sws_scale(m_nv12SwsCtx, frame->data, frame->linesize, 0, + height, dst, dstStride); + if (ret > 0) { + return bgrImage; + } + ANS_DBG("MEDIA_Convert", + "NV12 sws_scale failed (ret=%d) — falling back to cvtColorTwoPlane", ret); + } + } + + // Fallback: OpenCV two-plane conversion. Correct only when the planes + // are unpadded (linesize == width); kept for builds/inputs where the + // sws context could not be created. cv::Mat yPlane(height, width, CV_8UC1, frame->data[0], frame->linesize[0]); cv::Mat uvPlane(height / 2, width / 2, CV_8UC2, frame->data[1], frame->linesize[1]); @@ -2200,9 +2274,14 @@ CVideoPlayer::~CVideoPlayer() sws_freeContext(swsCtx); swsCtx = nullptr; } - if (m_nv12SwsCtx != nullptr) { - sws_freeContext(m_nv12SwsCtx); - m_nv12SwsCtx = nullptr; + { + // Lock order: _mutex (held) → m_nv12SwsMutex. Protects against a + // consumer thread still inside avframeNV12ToCvMat's sws_scale. + std::lock_guard nv12Lock(m_nv12SwsMutex); + if (m_nv12SwsCtx != nullptr) { + sws_freeContext(m_nv12SwsCtx); + m_nv12SwsCtx = nullptr; + } } if (this->_tjInstance) { tjDestroy(this->_tjInstance); diff --git a/MediaClient/media/video_player.h b/MediaClient/media/video_player.h index b9d465b..bd3a7d9 100644 --- a/MediaClient/media/video_player.h +++ b/MediaClient/media/video_player.h @@ -428,6 +428,13 @@ protected: int m_nv12LastWidth = 0, m_nv12LastHeight = 0; int m_nv12LastColorspace = -1; int m_nv12LastRange = -1; + // Guards m_nv12SwsCtx create/use/free. The NV12→BGR conversion deliberately + // runs WITHOUT _mutex (see avframeToCvMat, commit "Fix RTSP stall by + // releasing _mutex during BGR conversion"), so this dedicated lock protects + // the context against close()/dtor freeing it mid-sws_scale without + // reintroducing contention with onVideoFrame. Lock order: _mutex → this + // (close()/dtor hold _mutex first); avframeNV12ToCvMat takes only this lock. + std::mutex m_nv12SwsMutex; void initSwsContext(int width, int height, AVPixelFormat pixFmt, AVPixelFormat outputPixFmt = AV_PIX_FMT_YUVJ420P); void initNV12SwsContext(const AVFrame* frame);