MediaClient: keyframe gating (startup + corruption re-arm), libyuv NV12 path
Restores the previously shelved decoder/player hardening and extends it: 1. Startup keyframe gating (video_decoder): after (re)start the decoder joins the stream mid-GOP; frames referencing unseen pictures render as green/blocky concealment garbage. Drop everything until the first IDR (300-frame safety valve so unmarked streams can't blank forever). 2. NEW - corruption re-arm: on lossy links (Wi-Fi cameras) dropped slices surface as "First slice in a frame missing" / "Could not find ref with POC n" and FFmpeg marks the output via AVFrame::decode_error_flags. Re-arm the keyframe gate and drop the rest of the GOP so the display holds the last clean frame instead of showing concealment artifacts; corrupt drops count toward the safety valve. 3. libyuv NV12->BGR conversion (video_player): honors NVDEC/D3D11VA padded strides (1080->1088 rows) that cv::cvtColorTwoPlane mishandled (green cast + striped noise), picks BT.601/709/2020 coefficients from the frame's signalled colorspace, and is ~3-5x faster than sws_scale. Verified live against an RTSP HEVC camera (NVDEC path): steady frame flow, no regressions on the clean-stream path. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -1405,6 +1405,80 @@ cv::Mat CVideoPlayer::avframeNV12ToCvMat(const AVFrame* frame)
|
||||
m_nv12OrigWidth = width;
|
||||
m_nv12OrigHeight = height;
|
||||
|
||||
// Preferred path: libyuv NV12→RGB24 (BGR in memory, matches cv::Mat).
|
||||
// SIMD (SSSE3/AVX2) and ~3-5x faster than sws_scale's quality config.
|
||||
// Crucially it honors src_stride_uv, so the padded strides/height that
|
||||
// NVDEC and D3D11VA emit (e.g. 1080→1088 rows, aligned strides) convert
|
||||
// correctly — that padding is what cv::cvtColorTwoPlane mishandled,
|
||||
// reading the chroma plane at the wrong offset and rendering a green
|
||||
// cast with striped noise toward the bottom of the frame.
|
||||
// Coefficients are chosen from the frame's signalled colorspace/range,
|
||||
// mirroring initNV12SwsContext() (BT.709 for HD/4K, BT.601 for SD,
|
||||
// BT.2020 if tagged). The constants map studio-range YUV to full-range
|
||||
// BGR, so the result is display-ready — no manual range expansion.
|
||||
#if defined(ANSCORE_HAS_LIBYUV) && ANSCORE_HAS_LIBYUV
|
||||
{
|
||||
const bool fullRange = (frame->color_range == AVCOL_RANGE_JPEG);
|
||||
const libyuv::YuvConstants* yuvc;
|
||||
switch (frame->colorspace) {
|
||||
case AVCOL_SPC_BT470BG:
|
||||
case AVCOL_SPC_SMPTE170M:
|
||||
yuvc = fullRange ? &libyuv::kYuvJPEGConstants : &libyuv::kYuvI601Constants;
|
||||
break;
|
||||
case AVCOL_SPC_BT2020_NCL:
|
||||
case AVCOL_SPC_BT2020_CL:
|
||||
yuvc = fullRange ? &libyuv::kYuvV2020Constants : &libyuv::kYuv2020Constants;
|
||||
break;
|
||||
case AVCOL_SPC_BT709:
|
||||
yuvc = fullRange ? &libyuv::kYuvF709Constants : &libyuv::kYuvH709Constants;
|
||||
break;
|
||||
default:
|
||||
// Auto-detect by resolution (matches initNV12SwsContext()).
|
||||
if (width >= 1280 || height >= 720)
|
||||
yuvc = fullRange ? &libyuv::kYuvF709Constants : &libyuv::kYuvH709Constants;
|
||||
else
|
||||
yuvc = fullRange ? &libyuv::kYuvJPEGConstants : &libyuv::kYuvI601Constants;
|
||||
break;
|
||||
}
|
||||
cv::Mat bgrImage(height, width, CV_8UC3);
|
||||
int ret = libyuv::NV12ToRGB24Matrix(
|
||||
frame->data[0], frame->linesize[0],
|
||||
frame->data[1], frame->linesize[1],
|
||||
bgrImage.data, static_cast<int>(bgrImage.step),
|
||||
yuvc, width, height);
|
||||
if (ret == 0) {
|
||||
return bgrImage;
|
||||
}
|
||||
ANS_DBG("MEDIA_Convert",
|
||||
"libyuv NV12ToRGB24 failed (ret=%d) — falling back to sws_scale", ret);
|
||||
}
|
||||
#endif
|
||||
|
||||
// Fallback path: sws_scale via the dedicated NV12→BGR24 context. Also
|
||||
// stride-safe (honors frame->linesize[]) and colorspace/range-correct;
|
||||
// used on non-libyuv builds or if libyuv ever reports an error.
|
||||
{
|
||||
// Dedicated lock (not _mutex) so close()/dtor cannot free the context
|
||||
// mid-scale, while still keeping onVideoFrame off the hot _mutex.
|
||||
std::lock_guard<std::mutex> nv12Lock(m_nv12SwsMutex);
|
||||
initNV12SwsContext(frame);
|
||||
if (m_nv12SwsCtx) {
|
||||
cv::Mat bgrImage(height, width, CV_8UC3);
|
||||
uint8_t* dst[1] = { bgrImage.data };
|
||||
int dstStride[1] = { static_cast<int>(bgrImage.step[0]) };
|
||||
int ret = sws_scale(m_nv12SwsCtx, frame->data, frame->linesize, 0,
|
||||
height, dst, dstStride);
|
||||
if (ret > 0) {
|
||||
return bgrImage;
|
||||
}
|
||||
ANS_DBG("MEDIA_Convert",
|
||||
"NV12 sws_scale failed (ret=%d) — falling back to cvtColorTwoPlane", ret);
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: OpenCV two-plane conversion. Correct only when the planes
|
||||
// are unpadded (linesize == width); kept for builds/inputs where the
|
||||
// sws context could not be created.
|
||||
cv::Mat yPlane(height, width, CV_8UC1, frame->data[0], frame->linesize[0]);
|
||||
cv::Mat uvPlane(height / 2, width / 2, CV_8UC2, frame->data[1], frame->linesize[1]);
|
||||
|
||||
@@ -2200,9 +2274,14 @@ CVideoPlayer::~CVideoPlayer()
|
||||
sws_freeContext(swsCtx);
|
||||
swsCtx = nullptr;
|
||||
}
|
||||
if (m_nv12SwsCtx != nullptr) {
|
||||
sws_freeContext(m_nv12SwsCtx);
|
||||
m_nv12SwsCtx = nullptr;
|
||||
{
|
||||
// Lock order: _mutex (held) → m_nv12SwsMutex. Protects against a
|
||||
// consumer thread still inside avframeNV12ToCvMat's sws_scale.
|
||||
std::lock_guard<std::mutex> nv12Lock(m_nv12SwsMutex);
|
||||
if (m_nv12SwsCtx != nullptr) {
|
||||
sws_freeContext(m_nv12SwsCtx);
|
||||
m_nv12SwsCtx = nullptr;
|
||||
}
|
||||
}
|
||||
if (this->_tjInstance) {
|
||||
tjDestroy(this->_tjInstance);
|
||||
|
||||
Reference in New Issue
Block a user