#include "sys_inc.h" #include "media_util.h" #include "media_parse.h" #include "media_codec.h" #include "h264.h" #include "h265.h" #include "video_player.h" extern "C" { #include #include #include #include #include #include #include #include } #if __WINDOWS_OS__ #include "audio_play_win.h" #elif defined(IOS) #include "video_render_sdl.h" #include "audio_play_mac.h" #elif __LINUX_OS__ #include "video_render_sdl.h" #include "audio_play_qt.h" #endif #include #include #include #include #include #include #include #if defined(__SSE2__) || defined(_M_X64) || defined(_M_AMD64) #include #define HAS_SSE2 1 #endif #include "ANSLicense.h" // ANS_DBG macro (gated by ANSCORE_DEBUGVIEW) // --------------------------------------------------------------------------- // Leak diagnostics — definitions for counters declared extern in header. // Also references counters defined in video_decoder.cpp so the heartbeat // below can report media allocator balance in a single line. // --------------------------------------------------------------------------- std::atomic g_queueClones{0}; std::atomic g_queueFrees{0}; std::atomic g_nv12Clones{0}; std::atomic g_nv12Frees{0}; std::atomic g_nv12Escapes{0}; std::atomic g_cudaHWClones{0}; std::atomic g_cudaHWFrees{0}; std::atomic g_cudaHWEscapes{0}; std::atomic g_avframePendingReturns{0}; extern std::atomic g_contiguousAllocs; extern std::atomic g_contiguousFrees; extern std::atomic g_contiguousBytesInFlight; // --------------------------------------------------------------------------- // Freeze diagnostics — once-per-second pulse showing both producer // (onVideoFrame) and consumer (getImage) accounting in a single line so a // 1-2 second freeze can be localized in DebugView. Filter on "MEDIA_Pulse". // // Reading the line during a freeze: // arrived high, pushed=0 ............ all frames dropped at producer // drop(err=N) UDP packet loss → SW decoder set decode_error_flags // drop(keyWait=N) waiting for first IDR after start/restart // drop(rate=N) SetTargetFPS throttle skipped them // arrived=0 .................. decoder silent (network/rx stall) // arrived≈pushed, gets high, new=0 ... consumer not seeing the new push // (mutex contention, seq stuck, ...) // arrived≈pushed, new high, smooth ..... pipeline healthy // --------------------------------------------------------------------------- namespace { std::atomic g_pulseArrived{0}; // onVideoFrame entries std::atomic g_pulsePushed{0}; // pushFrame() calls (frame survived all drop checks) std::atomic g_pulseDropDecodeErr{0}; // decode_error_flags != 0 (now also pushed; counted for visibility) std::atomic g_pulseDropKeyframeWait{0}; // m_bWaitingForKeyframe and not IDR std::atomic g_pulseDropRateLimit{0}; // SetTargetFPS interval skip std::atomic g_pulseGetCalls{0}; // getImage entries std::atomic g_pulseGetNew{0}; // returned a brand-new frame std::atomic g_pulseGetCachedFast{0}; // currentSeq == m_lastFrameSeq (no new push) std::atomic g_pulseGetCachedEmpty{0}; // queue empty std::atomic g_pulseGetCachedNotPlaying{0};// !m_bPlaying std::atomic g_pulseGetCachedSettle{0}; // waiting for keyframe / settle period std::atomic g_pulseSeqJump{0}; // total frames the consumer flew past via getLatestFrame // (sum of (currentSeq - m_lastFrameSeq - 1) across all reads) std::atomic g_pulseNextEmitTick{0}; // monotonic deadline (steady_clock::now ticks) // Inter-arrival timing — exposes "21 frames bunched then 900ms silence" patterns // that aggregate counters hide. Both stored as steady_clock raw ticks. std::atomic g_pulseLastArrivalTick{0}; // tick of previous onVideoFrame entry (0 = none) std::atomic g_pulseMaxGapTicks{0}; // max gap seen in current 1s window // Called from onVideoFrame on every arrival. Updates "max gap" if the gap // since previous arrival exceeds the current window's running max. inline void note_arrival_gap() { using clk = std::chrono::steady_clock; const long long now = clk::now().time_since_epoch().count(); const long long prev = g_pulseLastArrivalTick.exchange(now, std::memory_order_relaxed); if (prev == 0) return; // first arrival ever — no gap const long long gap = now - prev; // Atomic max: only update if larger long long cur = g_pulseMaxGapTicks.load(std::memory_order_relaxed); while (gap > cur && !g_pulseMaxGapTicks.compare_exchange_weak(cur, gap, std::memory_order_relaxed)) { // CAS retry until we win or someone else wrote a larger value } } // Emit if 1s has elapsed since the previous emission. CAS makes only one // thread per second do the print; the rest just no-op. inline void emit_media_pulse_if_due(size_t queueDepth) { using clk = std::chrono::steady_clock; const long long tick = clk::now().time_since_epoch().count(); long long expected = g_pulseNextEmitTick.load(std::memory_order_relaxed); if (tick < expected) return; const long long deadline = tick + std::chrono::duration_cast( std::chrono::milliseconds(1000)).count(); if (!g_pulseNextEmitTick.compare_exchange_strong( expected, deadline, std::memory_order_relaxed)) { return; // another thread won the window } // Snapshot + reset (delta semantics — every line is "events in last ~1s") const uint64_t arr = g_pulseArrived.exchange(0, std::memory_order_relaxed); const uint64_t psh = g_pulsePushed.exchange(0, std::memory_order_relaxed); const uint64_t dErr = g_pulseDropDecodeErr.exchange(0, std::memory_order_relaxed); const uint64_t dKey = g_pulseDropKeyframeWait.exchange(0, std::memory_order_relaxed); const uint64_t dRat = g_pulseDropRateLimit.exchange(0, std::memory_order_relaxed); const uint64_t gtl = g_pulseGetCalls.exchange(0, std::memory_order_relaxed); const uint64_t gNew = g_pulseGetNew.exchange(0, std::memory_order_relaxed); const uint64_t gFst = g_pulseGetCachedFast.exchange(0, std::memory_order_relaxed); const uint64_t gEmt = g_pulseGetCachedEmpty.exchange(0, std::memory_order_relaxed); const uint64_t gNp = g_pulseGetCachedNotPlaying.exchange(0, std::memory_order_relaxed); const uint64_t gStl = g_pulseGetCachedSettle.exchange(0, std::memory_order_relaxed); const uint64_t evct = FrameQueue::evictedCounter().exchange(0, std::memory_order_relaxed); const uint64_t skip = g_pulseSeqJump.exchange(0, std::memory_order_relaxed); const long long maxGapTicks = g_pulseMaxGapTicks.exchange(0, std::memory_order_relaxed); const double maxGapMs = std::chrono::duration( clk::duration(maxGapTicks)).count(); // Production: route through ANS_DBG so the pulse is gated by the // ANSCORE_DEBUGVIEW runtime toggle (env var ANSCENTER_DBGVIEW=1 or // sentinel file C:\ProgramData\ANSCENTER\ansvisdebugview.txt). // Silent by default; one-line snapshot per second when enabled. // // maxGap is the largest interval between two consecutive arrivals in // this second. Healthy ~25fps streams should sit at maxGap≈40-60ms. // A spike to 1000+ ms is the smoking gun for a producer-side stall: // RTP rx hung, decoder waiting on references, etc. ANS_DBG("MEDIA_Pulse", "[1s] producer arrived=%llu pushed=%llu drop(err=%llu keyWait=%llu rate=%llu) " "maxGap=%.0fms evicted=%llu | " "consumer gets=%llu new=%llu skipped=%llu cached(fast=%llu empty=%llu notPlay=%llu settle=%llu) " "| queueDepth=%zu", (unsigned long long)arr, (unsigned long long)psh, (unsigned long long)dErr, (unsigned long long)dKey, (unsigned long long)dRat, maxGapMs, (unsigned long long)evct, (unsigned long long)gtl, (unsigned long long)gNew, (unsigned long long)skip, (unsigned long long)gFst, (unsigned long long)gEmt, (unsigned long long)gNp, (unsigned long long)gStl, queueDepth); } } // anonymous namespace // libyuv: SIMD-accelerated YUV↔RGB conversion with native strided-plane input. // Replaces the memcpy-into-staging + cv::cvtColor(COLOR_YUV2BGR_I420) chain // in avframeYUV420PToCvMat with a direct I420→RGB24 (== OpenCV BGR memory // order) call. When the submodule isn't checked out, ANSCORE_HAS_LIBYUV is // not defined and we fall back to the pre-libyuv path. // // Extended coverage: I420/J420/I422/J422/I444/J444/I010/I210 → BGR. // 4:2:2 and 4:4:4 8-bit formats have direct ToRGB24 converters. // 10-bit formats (I010/I210) require a two-step chain: // I010/I210ToARGB → ARGBToRGB24 (drops alpha byte, no channel swap). #if defined(ANSCORE_HAS_LIBYUV) && ANSCORE_HAS_LIBYUV #include "libyuv/convert.h" // I212ToI422, I412ToI444 (12→8-bit downconvert) #include "libyuv/convert_argb.h" // I420/J420/I422/I444/I010/I210/I410/I012 → ARGB/RGB24 #include "libyuv/convert_from_argb.h" // ARGBToRGB24 (for 10/12-bit paths) #endif void VideoDecoderCallback(AVFrame* frame, void* userdata) { CVideoPlayer* pPlayer = (CVideoPlayer*)userdata; pPlayer->onVideoFrame(frame); } void AudioDecoderCallback(AVFrame* frame, void* userdata) { CVideoPlayer* pPlayer = (CVideoPlayer*)userdata; pPlayer->onAudioFrame(frame); } void CVideoPlayer::setBbox(cv::Rect bbox) { std::lock_guard lock(_mutex); m_Bbox = bbox; } void CVideoPlayer::setCrop(bool crop) { std::lock_guard lock(_mutex); m_bCrop = crop; } AVFrame* CVideoPlayer::cropFrame(const AVFrame* srcFrame, cv::Rect bBox, bool cropFlag) { std::lock_guard lock(_mutex); try { // Validate prerequisites if (!cropFlag || !srcFrame || !m_bPlaying) { return nullptr; } // Ensure the bounding box is within the source frame's boundaries bBox.x = std::clamp(bBox.x, 0, srcFrame->width); bBox.y = std::clamp(bBox.y, 0, srcFrame->height); bBox.width = std::clamp(bBox.width, 0, srcFrame->width - bBox.x); bBox.height = std::clamp(bBox.height, 0, srcFrame->height - bBox.y); // Validate the bounding box dimensions if (bBox.width <= 10 || bBox.height <= 10) { ANS_DBG("MEDIA_Crop", "invalid bounding box dimensions"); return nullptr; } // Allocate memory for the cropped frame AVFrame* croppedFrame = av_frame_alloc(); if (!croppedFrame) { ANS_DBG("MEDIA_Crop", "failed to allocate cropped frame"); return nullptr; } // Set cropped frame attributes croppedFrame->format = srcFrame->format; croppedFrame->width = bBox.width; croppedFrame->height = bBox.height; // *** REMOVED: Don't allocate buffer since we're only setting pointers *** // The cropFrameData() function will set pointers to the original frame's data // Crop the frame based on its format if (!cropFrameData(srcFrame, croppedFrame, bBox)) { av_frame_free(&croppedFrame); return nullptr; } return croppedFrame; } catch (const std::exception& e) { ANS_DBG("MEDIA_Crop", "cropFrame exception: %s", e.what()); return nullptr; } } // Helper function to crop frame data bool CVideoPlayer::cropFrameData(const AVFrame* srcFrame, AVFrame* croppedFrame, const cv::Rect& bBox) { std::lock_guard lock(_mutex); try { switch (srcFrame->format) { case AV_PIX_FMT_YUVJ444P: case AV_PIX_FMT_YUV444P: // Full chroma resolution (No subsampling) cropPlane(srcFrame, croppedFrame, 0, bBox.x, bBox.y, 1, 1); // Y plane cropPlane(srcFrame, croppedFrame, 1, bBox.x, bBox.y, 1, 1); // U plane cropPlane(srcFrame, croppedFrame, 2, bBox.x, bBox.y, 1, 1); // V plane break; case AV_PIX_FMT_YUVJ422P: case AV_PIX_FMT_YUV422P: // Horizontal chroma subsampling (chroma resolution is half in X direction) cropPlane(srcFrame, croppedFrame, 0, bBox.x, bBox.y, 1, 1); // Y plane cropPlane(srcFrame, croppedFrame, 1, bBox.x / 2, bBox.y, 1, 1); // U plane cropPlane(srcFrame, croppedFrame, 2, bBox.x / 2, bBox.y, 1, 1); // V plane break; case AV_PIX_FMT_YUVJ420P: case AV_PIX_FMT_YUV420P: // Both horizontal and vertical chroma subsampling (chroma is 1/4 of Y resolution) cropPlane(srcFrame, croppedFrame, 0, bBox.x, bBox.y, 1, 1); // Y plane cropPlane(srcFrame, croppedFrame, 1, bBox.x / 2, bBox.y / 2, 1, 1); // U plane cropPlane(srcFrame, croppedFrame, 2, bBox.x / 2, bBox.y / 2, 1, 1); // V plane break; case AV_PIX_FMT_NV12: // NV12 has a **single interleaved UV plane** cropPlane(srcFrame, croppedFrame, 0, bBox.x, bBox.y, 1, 1); // Y plane cropPlane(srcFrame, croppedFrame, 1, bBox.x / 2, bBox.y / 2, 2, 1); // UV plane (interleaved, stepX=2) break; default: ANS_DBG("MEDIA_Crop", "unsupported pixel format: %s", av_get_pix_fmt_name((AVPixelFormat)srcFrame->format)); return false; } return true; } catch (const std::exception& e) { ANS_DBG("MEDIA_Crop", "cropFrameData exception: %s", e.what()); return false; } } // Helper function to crop individual planes void CVideoPlayer::cropPlane(const AVFrame* srcFrame, AVFrame* croppedFrame, int planeIndex, int offsetX, int offsetY, int subsampleX, int subsampleY) { std::lock_guard lock(_mutex); try { croppedFrame->data[planeIndex] = srcFrame->data[planeIndex] + offsetY * srcFrame->linesize[planeIndex] + offsetX * subsampleX; croppedFrame->linesize[planeIndex] = srcFrame->linesize[planeIndex]; } catch (const std::exception& e) { ANS_DBG("MEDIA_Crop", "cropPlane exception: %s", e.what()); } } // Convert NV12 AVFrame to YUVJ420P AVFrame* CVideoPlayer::convertNV12ToYUVJ420P(const AVFrame* nv12Frame) { std::lock_guard lock(_mutex); AVFrame* yuvjFrame = nullptr; try { if (!nv12Frame || !nv12Frame->data[0] || nv12Frame->width <= 10 || nv12Frame->height <= 10) { ANS_DBG("MEDIA_NV12", "invalid or empty NV12 frame data"); return nullptr; } int width = nv12Frame->width; int height = nv12Frame->height; // ✅ Allocate new YUVJ420P frame yuvjFrame = av_frame_alloc(); if (!yuvjFrame) { ANS_DBG("MEDIA_NV12", "failed to allocate YUVJ420P frame"); return nullptr; } yuvjFrame->format = AV_PIX_FMT_YUVJ420P; yuvjFrame->width = width; yuvjFrame->height = height; // ✅ Allocate buffer for YUVJ420P frame if (av_frame_get_buffer(yuvjFrame, 32) < 0) { ANS_DBG("MEDIA_NV12", "failed to allocate YUVJ420P buffer"); av_frame_free(&yuvjFrame); return nullptr; } // ✅ Copy Y plane (Luma) row by row (prevents memory corruption) for (int j = 0; j < height; ++j) { memcpy(yuvjFrame->data[0] + j * yuvjFrame->linesize[0], nv12Frame->data[0] + j * nv12Frame->linesize[0], width); } // ✅ Correctly extract UV planes from interleaved NV12 uint8_t* nv12_uv = nv12Frame->data[1]; uint8_t* yuvj_u = yuvjFrame->data[1]; uint8_t* yuvj_v = yuvjFrame->data[2]; int uvWidth = width / 2; int uvHeight = height / 2; for (int j = 0; j < uvHeight; ++j) { uint8_t* nv12Row = nv12_uv + j * nv12Frame->linesize[1]; uint8_t* uRow = yuvj_u + j * yuvjFrame->linesize[1]; uint8_t* vRow = yuvj_v + j * yuvjFrame->linesize[2]; for (int i = 0; i < uvWidth; ++i) { uRow[i] = nv12Row[i * 2]; // Extract U vRow[i] = nv12Row[i * 2 + 1]; // Extract V } } return yuvjFrame; } catch (const std::exception& e) { ANS_DBG("MEDIA_NV12", "convertNV12ToYUVJ420P exception: %s", e.what()); // ✅ Prevent Memory Leak by Freeing the Allocated Frame if (yuvjFrame) { av_frame_free(&yuvjFrame); } return nullptr; } } std::string CVideoPlayer::avframeYUVJ420PToJpegStringUsingFFMpeg(const AVFrame* pFrame) { std::lock_guard lock(_mutex); try { if (!m_bPlaying) { return ""; } if (!pFrame || !pFrame->data[0] || pFrame->width <= 10 || pFrame->height <= 10) { ANS_DBG("MEDIA_Convert", "invalid or empty frame data / dimensions"); return ""; } AVCodec* jpegCodec = avcodec_find_encoder(AV_CODEC_ID_MJPEG); if (!jpegCodec) { ANS_DBG("MEDIA_Jpeg", "failed to find MJPEG encoder"); return ""; } AVCodecContext* jpegContext = avcodec_alloc_context3(jpegCodec); if (!jpegContext) { ANS_DBG("MEDIA_Jpeg", "failed to allocate codec context"); return ""; } int imageSize = std::max(pFrame->width, pFrame->height); AVPixelFormat pixFmt = AV_PIX_FMT_YUVJ420P;// Fix to use YUVJ420P for all resolutions jpegContext->pix_fmt = pixFmt; jpegContext->time_base.num = 1; jpegContext->time_base.den = 30; jpegContext->compression_level = 10; jpegContext->flags |= AV_CODEC_FLAG_QSCALE; // Enable quality scale jpegContext->global_quality = 90 * FF_QP2LAMBDA; // Adjust quality (90 is near lossless) AVFrame* convertedFrame = nullptr; AVPacket packet; av_init_packet(&packet); packet.data = nullptr; packet.size = 0; bool isSuccess = false; std::string jpegData; // Determine if conversion is needed based on the pixel format if ((pFrame->format == AV_PIX_FMT_YUVJ420P) || (pFrame->format == AV_PIX_FMT_YUV420P)) { jpegContext->width = pFrame->width; jpegContext->height = pFrame->height; if (avcodec_open2(jpegContext, jpegCodec, NULL) >= 0) { if (avcodec_send_frame(jpegContext, pFrame) >= 0) { if (avcodec_receive_packet(jpegContext, &packet) >= 0) { jpegData.assign(reinterpret_cast(packet.data), packet.size); m_Width = pFrame->width; m_Height = pFrame->height; m_pts = m_pts + 1; isSuccess = true; } } } } else { // Conversion is needed to AV_PIX_FMT_YUVJ420P initSwsContext(pFrame->width, pFrame->height, static_cast(pFrame->format)); convertedFrame = av_frame_alloc(); if (convertedFrame) { convertedFrame->format = pixFmt; convertedFrame->width = pFrame->width; convertedFrame->height = pFrame->height; convertedFrame->color_range = AVCOL_RANGE_JPEG; if (av_frame_get_buffer(convertedFrame, 32) >= 0) { sws_scale(swsCtx, pFrame->data, pFrame->linesize, 0, pFrame->height, convertedFrame->data, convertedFrame->linesize); jpegContext->width = convertedFrame->width; jpegContext->height = convertedFrame->height; if (avcodec_open2(jpegContext, jpegCodec, NULL) >= 0) { if (avcodec_send_frame(jpegContext, convertedFrame) >= 0) { if (avcodec_receive_packet(jpegContext, &packet) >= 0) { // Successfully encoded to JPEG jpegData.assign(reinterpret_cast(packet.data), packet.size); m_Width = convertedFrame->width; m_Height = convertedFrame->height; m_pts = m_pts + 1; isSuccess = true; } } } } } av_frame_free(&convertedFrame); // Free the converted frame if allocated } // Cleanup av_packet_unref(&packet); // Free the packet data avcodec_free_context(&jpegContext); // Free the codec context // Return the JPEG data as a string if successful, otherwise an empty string return isSuccess ? jpegData : ""; } catch (const std::exception& e) { ANS_DBG("MEDIA_Jpeg", "avframeToJpegString exception: %s", e.what()); return ""; // Return empty string on error } } std::string CVideoPlayer::avframeYUVJ420PToJpegStringUsingTurboJPEG(const AVFrame* pFrame) { std::lock_guard lock(_mutex); try { if (!m_bPlaying || !pFrame || !pFrame->data[0] || pFrame->width <= 10 || pFrame->height <= 10) { return ""; } // Ensure TurboJPEG instance is valid if (!_tjInstance) { return ""; } unsigned char* yuvPlanes[3] = { pFrame->data[0], pFrame->data[1], pFrame->data[2] }; int strides[3] = { pFrame->linesize[0], pFrame->linesize[1], pFrame->linesize[2] }; int width = pFrame->width; int height = pFrame->height; constexpr int subsampling = TJSAMP_420; constexpr int quality = 85; // Use thread-local buffers to avoid malloc/free overhead static thread_local std::vector jpegBuffer; static thread_local std::vector yuvBuffer; // Estimate required buffer sizes unsigned long jpegBufferSize = tjBufSize(width, height, subsampling); unsigned long yuvBufferSize = tjBufSizeYUV(width, height, subsampling); // Resize buffers only if necessary if (jpegBuffer.size() < jpegBufferSize) { jpegBuffer.resize(jpegBufferSize); } if (yuvBuffer.size() < yuvBufferSize) { yuvBuffer.resize(yuvBufferSize); } // Pointers for JPEG output unsigned char* jpegDataPtr = jpegBuffer.data(); unsigned long jpegSize = 0; // Convert YUV to JPEG using TurboJPEG int ret = tjCompressFromYUVPlanes( _tjInstance, (const unsigned char**)yuvPlanes, width, strides, height, subsampling, &jpegDataPtr, // Using preallocated buffer &jpegSize, quality, TJFLAG_FASTDCT | TJFLAG_FASTUPSAMPLE ); // Check if TurboJPEG reallocated the buffer if (ret < 0) { return ""; } // If TurboJPEG allocated a new buffer, we must free it if (jpegDataPtr != jpegBuffer.data()) { std::string jpegString(reinterpret_cast(jpegDataPtr), jpegSize); tjFree(jpegDataPtr); // Free the buffer allocated by TurboJPEG return jpegString; } // Convert to std::string (without extra allocations) return std::string(reinterpret_cast(jpegDataPtr), jpegSize); } catch (const std::exception& e) { ANS_DBG("MEDIA_Jpeg", "avframeToJpegString exception: %s", e.what()); return ""; // Return empty string on error } } std::string CVideoPlayer::encodeYUVJ420PToJPEG(AVFrame* frame, int quality) { std::lock_guard lock(_mutex); try { if (!frame || frame->format != AV_PIX_FMT_YUVJ420P) { ANS_DBG("MEDIA_Convert", "invalid frame format (must be YUVJ420P)"); return ""; } // Find MJPEG encoder AVCodec* codec = avcodec_find_encoder(AV_CODEC_ID_MJPEG); if (!codec) { ANS_DBG("MEDIA_Jpeg", "JPEG encoder not found"); return ""; } // Allocate codec context AVCodecContext* codecCtx = avcodec_alloc_context3(codec); if (!codecCtx) { ANS_DBG("MEDIA_Jpeg", "failed to allocate codec context (yuvj420p)"); return ""; } // Set encoding parameters codecCtx->pix_fmt = AV_PIX_FMT_YUVJ420P; // Use full-range YUV for better quality codecCtx->width = frame->width; codecCtx->height = frame->height; codecCtx->time_base.num = 1; codecCtx->time_base.den = 30; codecCtx->gop_size = 1; codecCtx->max_b_frames = 0; codecCtx->compression_level = 10; // Increase quality codecCtx->flags |= AV_CODEC_FLAG_QSCALE; // Enable quality scale codecCtx->global_quality = quality * FF_QP2LAMBDA; // Adjust quality (90 is near lossless) // Enable optimal Huffman tables AVDictionary* opts = nullptr; av_dict_set(&opts, "huffman", "optimal", 0); // Open codec if (avcodec_open2(codecCtx, codec, &opts) < 0) { ANS_DBG("MEDIA_Jpeg", "failed to open JPEG encoder"); avcodec_free_context(&codecCtx); return ""; } AVPacket pkt; av_init_packet(&pkt); pkt.data = nullptr; pkt.size = 0; // Send frame to encoder if (avcodec_send_frame(codecCtx, frame) < 0) { ANS_DBG("MEDIA_Jpeg", "failed to send frame for encoding"); avcodec_free_context(&codecCtx); return ""; } // Receive encoded packet if (avcodec_receive_packet(codecCtx, &pkt) < 0) { ANS_DBG("MEDIA_Jpeg", "failed to receive encoded packet"); avcodec_free_context(&codecCtx); return ""; } // Convert to string and clean up std::string jpegString(reinterpret_cast(pkt.data), pkt.size); av_packet_unref(&pkt); avcodec_free_context(&codecCtx); av_dict_free(&opts); return jpegString; } catch (const std::exception& e) { ANS_DBG("MEDIA_Jpeg", "encodeYUVJ420PToJPEG exception: %s", e.what()); return ""; // Return empty string on error } } std::string CVideoPlayer::avframeYUVJ420PToJpegString(const AVFrame* spFrame) { std::lock_guard lock(_mutex); AVFrame* croppedFrame = nullptr; AVFrame* convertedFrame = nullptr; AVFrame* convertedNV12Frame = nullptr; AVFrame* pFrame = const_cast(spFrame); // Default to original frame bool isSuccess = false; std::string jpegData; try { if (!m_bPlaying) { return ""; } if (!spFrame || !spFrame->data[0] || spFrame->width <= 10 || spFrame->height <= 10) { ANS_DBG("MEDIA_Convert", "invalid or empty frame data / dimensions"); return ""; } // ✅ Convert NV12 to YUVJ420P if needed if (pFrame->format == AV_PIX_FMT_NV12) { convertedNV12Frame = convertNV12ToYUVJ420P(spFrame); if (convertedNV12Frame) { pFrame = convertedNV12Frame; // Use the converted frame } } // ✅ Process the frame if it's already in YUVJ420P or YUV420P if ((pFrame->format == AV_PIX_FMT_YUVJ420P) || (pFrame->format == AV_PIX_FMT_YUV420P)) { croppedFrame = cropFrame(pFrame, m_Bbox, m_bCrop); if (!croppedFrame) { croppedFrame = pFrame; // Use original frame if cropping failed } // TurboJPEG handles all resolutions efficiently jpegData = avframeYUVJ420PToJpegStringUsingTurboJPEG(croppedFrame); if (!jpegData.empty()) { m_Width = croppedFrame->width; m_Height = croppedFrame->height; m_pts = m_pts + 1; isSuccess = true; } // ✅ Free cropped frame if allocated if (croppedFrame != pFrame) { av_frame_free(&croppedFrame); croppedFrame = nullptr; } } else { // ✅ Convert non-YUVJ420P frames initSwsContext(pFrame->width, pFrame->height, static_cast(pFrame->format)); convertedFrame = av_frame_alloc(); if (convertedFrame) { convertedFrame->format = AV_PIX_FMT_YUVJ420P; convertedFrame->width = pFrame->width; convertedFrame->height = pFrame->height; convertedFrame->color_range = AVCOL_RANGE_JPEG; if (av_frame_get_buffer(convertedFrame, 32) >= 0) { sws_scale(swsCtx, pFrame->data, pFrame->linesize, 0, pFrame->height, convertedFrame->data, convertedFrame->linesize); croppedFrame = cropFrame(convertedFrame, m_Bbox, m_bCrop); if (!croppedFrame) { croppedFrame = convertedFrame; // Use converted frame if cropping failed } // TurboJPEG handles all resolutions efficiently jpegData = avframeYUVJ420PToJpegStringUsingTurboJPEG(croppedFrame); if (!jpegData.empty()) { m_Width = croppedFrame->width; m_Height = croppedFrame->height; m_pts = m_pts + 1; isSuccess = true; } // ✅ Free cropped frame if allocated if (croppedFrame != convertedFrame) { av_frame_free(&croppedFrame); croppedFrame = nullptr; } } } // ✅ Free converted frame if allocated if (convertedFrame) { av_frame_free(&convertedFrame); convertedFrame = nullptr; } } // ✅ Free the NV12 converted frame if used if (convertedNV12Frame) { av_frame_free(&convertedNV12Frame); convertedNV12Frame = nullptr; } return isSuccess ? jpegData : ""; } catch (const std::exception& e) { ANS_DBG("MEDIA_Jpeg", "avframeYUVJ420PToJpegString exception: %s", e.what()); // ✅ Ensure all allocated frames are freed in case of an exception if (croppedFrame && croppedFrame != pFrame && croppedFrame != convertedFrame) { av_frame_free(&croppedFrame); } if (convertedFrame) { av_frame_free(&convertedFrame); } if (convertedNV12Frame) { av_frame_free(&convertedNV12Frame); } return ""; } } // Direct conversion of AVFrame to JPEG using TurboJPEG std::string CVideoPlayer::encodeNV12ToJPEG_TurboJPEG(const AVFrame* pFrame, int quality) { // NOTE: caller (avframeToJpegString) already holds _mutex — no lock needed here try { if (!m_bPlaying || !pFrame || !pFrame->data[0] || pFrame->width <= 10 || pFrame->height <= 10) { return ""; } // Ensure TurboJPEG instance is valid if (!_tjInstance) { ANS_DBG("MEDIA_Jpeg", "TurboJPEG instance not initialized"); return ""; } // Ensure the frame format is NV12 if (pFrame->format != AV_PIX_FMT_NV12) { ANS_DBG("MEDIA_Jpeg", "TurboJPEG: unsupported format (expected NV12, got %s)", av_get_pix_fmt_name((AVPixelFormat)pFrame->format)); return ""; } int width = pFrame->width; int height = pFrame->height; // Use caller's quality parameter (default 90 from function signature) // NV12 has interleaved UV, but TurboJPEG requires separate U and V planes unsigned char* yuvPlanes[3]; int strides[3]; yuvPlanes[0] = pFrame->data[0]; // Y plane (full resolution) strides[0] = pFrame->linesize[0]; // **Convert NV12 interleaved UV to separate U and V planes** int uvWidth = width / 2; int uvHeight = height / 2; int uvSize = uvWidth * uvHeight; static thread_local std::vector uPlane(uvSize); static thread_local std::vector vPlane(uvSize); // Deinterleave NV12 UV plane into separate U and V planes unsigned char* uvData = pFrame->data[1]; int uvStride = pFrame->linesize[1]; for (int j = 0; j < uvHeight; j++) { const unsigned char* uvRow = uvData + j * uvStride; unsigned char* uRow = uPlane.data() + j * uvWidth; unsigned char* vRow = vPlane.data() + j * uvWidth; int i = 0; #ifdef HAS_SSE2 // SSE2: process 16 UV pairs (32 bytes) at a time for (; i + 15 < uvWidth; i += 16) { __m128i uv0 = _mm_loadu_si128((__m128i*)(uvRow + i * 2)); __m128i uv1 = _mm_loadu_si128((__m128i*)(uvRow + i * 2 + 16)); // Deinterleave: even bytes = U, odd bytes = V __m128i mask = _mm_set1_epi16(0x00FF); __m128i u0 = _mm_and_si128(uv0, mask); __m128i u1 = _mm_and_si128(uv1, mask); __m128i v0 = _mm_srli_epi16(uv0, 8); __m128i v1 = _mm_srli_epi16(uv1, 8); __m128i uPacked = _mm_packus_epi16(u0, u1); __m128i vPacked = _mm_packus_epi16(v0, v1); _mm_storeu_si128((__m128i*)(uRow + i), uPacked); _mm_storeu_si128((__m128i*)(vRow + i), vPacked); } #endif // Scalar fallback for remaining pixels for (; i < uvWidth; i++) { uRow[i] = uvRow[i * 2]; vRow[i] = uvRow[i * 2 + 1]; } } // Assign separate planes to TurboJPEG input yuvPlanes[1] = uPlane.data(); yuvPlanes[2] = vPlane.data(); strides[1] = uvWidth; strides[2] = uvWidth; // Use thread-local buffers to avoid malloc/free overhead static thread_local std::vector jpegBuffer; // Estimate required buffer size for JPEG unsigned long jpegBufferSize = tjBufSize(width, height, TJSAMP_420); // Resize JPEG buffer only if necessary if (jpegBuffer.size() < jpegBufferSize) { jpegBuffer.resize(jpegBufferSize); } // Pointer for JPEG output unsigned char* jpegDataPtr = jpegBuffer.data(); unsigned long jpegSize = 0; // Convert NV12 (separated into YUV420P) to JPEG using TurboJPEG int ret = tjCompressFromYUVPlanes( _tjInstance, (const unsigned char**)yuvPlanes, width, strides, height, TJSAMP_420, // Explicitly define subsampling format for NV12 &jpegDataPtr, // Preallocated buffer &jpegSize, quality, TJFLAG_FASTDCT | TJFLAG_FASTUPSAMPLE ); if (ret < 0) { ANS_DBG("MEDIA_Jpeg", "TurboJPEG compression failed: %s", tjGetErrorStr()); return ""; } // If TurboJPEG allocated a new buffer, free it after copying if (jpegDataPtr != jpegBuffer.data()) { std::string jpegString(reinterpret_cast(jpegDataPtr), jpegSize); tjFree(jpegDataPtr); return jpegString; } // Convert to std::string without extra allocations return std::string(reinterpret_cast(jpegDataPtr), jpegSize); } catch (const std::exception& e) { ANS_DBG("MEDIA_Jpeg", "avframeNV12ToJpegStringUsingTurboJPEG exception: %s", e.what()); return ""; // Return empty string on error } } std::string CVideoPlayer::encodeNV12ToJPEG_FFmpeg(const AVFrame* nv12Frame, int quality) { // NOTE: caller (avframeToJpegString) already holds _mutex — no lock needed here AVCodecContext* codecCtx = nullptr; AVFrame* yuvjFrame = nullptr; AVPacket pkt; try { if (!m_bPlaying || !nv12Frame || !nv12Frame->data[0] || nv12Frame->width <= 10 || nv12Frame->height <= 10) { return ""; } if (nv12Frame->format != AV_PIX_FMT_NV12) { ANS_DBG("MEDIA_Convert", "invalid frame format (expected NV12)"); return ""; } int width = nv12Frame->width; int height = nv12Frame->height; // ✅ Find and allocate MJPEG encoder AVCodec* jpegCodec = avcodec_find_encoder(AV_CODEC_ID_MJPEG); if (!jpegCodec) { ANS_DBG("MEDIA_Jpeg", "MJPEG encoder not found"); return ""; } codecCtx = avcodec_alloc_context3(jpegCodec); if (!codecCtx) { ANS_DBG("MEDIA_Jpeg", "failed to allocate codec context (NV12)"); return ""; } // ✅ Set encoding parameters codecCtx->pix_fmt = AV_PIX_FMT_YUVJ420P; codecCtx->width = width; codecCtx->height = height; codecCtx->time_base = { 1, 25 }; codecCtx->gop_size = 1; codecCtx->max_b_frames = 0; codecCtx->compression_level = 10; codecCtx->flags |= AV_CODEC_FLAG_QSCALE; codecCtx->global_quality = quality * FF_QP2LAMBDA; if (avcodec_open2(codecCtx, jpegCodec, nullptr) < 0) { ANS_DBG("MEDIA_Jpeg", "failed to open MJPEG encoder"); avcodec_free_context(&codecCtx); return ""; } // ✅ Allocate YUVJ420P frame yuvjFrame = av_frame_alloc(); if (!yuvjFrame) { ANS_DBG("MEDIA_Jpeg", "failed to allocate YUVJ420P frame"); avcodec_free_context(&codecCtx); return ""; } yuvjFrame->format = AV_PIX_FMT_YUVJ420P; yuvjFrame->width = width; yuvjFrame->height = height; if (av_frame_get_buffer(yuvjFrame, 32) < 0) { ANS_DBG("MEDIA_Jpeg", "failed to allocate buffer for YUVJ420P frame"); av_frame_free(&yuvjFrame); avcodec_free_context(&codecCtx); return ""; } // ✅ Copy Y plane row by row (Prevents memory corruption) for (int j = 0; j < height; ++j) { memcpy(yuvjFrame->data[0] + j * yuvjFrame->linesize[0], nv12Frame->data[0] + j * nv12Frame->linesize[0], width); } // ✅ Correctly extract UV planes from NV12 uint8_t* nv12_uv = nv12Frame->data[1]; uint8_t* yuvj_u = yuvjFrame->data[1]; uint8_t* yuvj_v = yuvjFrame->data[2]; int uvWidth = width / 2; int uvHeight = height / 2; for (int j = 0; j < uvHeight; ++j) { uint8_t* nv12Row = nv12_uv + j * nv12Frame->linesize[1]; uint8_t* uRow = yuvj_u + j * yuvjFrame->linesize[1]; uint8_t* vRow = yuvj_v + j * yuvjFrame->linesize[2]; for (int i = 0; i < uvWidth; ++i) { uRow[i] = nv12Row[i * 2]; // Extract U vRow[i] = nv12Row[i * 2 + 1]; // Extract V } } // ✅ Encode frame to JPEG av_init_packet(&pkt); pkt.data = nullptr; pkt.size = 0; bool isSuccess = false; std::string jpegData; if (avcodec_send_frame(codecCtx, yuvjFrame) >= 0) { if (avcodec_receive_packet(codecCtx, &pkt) >= 0) { jpegData.assign(reinterpret_cast(pkt.data), pkt.size); isSuccess = true; } } // ✅ Cleanup av_packet_unref(&pkt); av_frame_free(&yuvjFrame); avcodec_free_context(&codecCtx); return isSuccess ? jpegData : ""; } catch (const std::exception& e) { ANS_DBG("MEDIA_Jpeg", "encodeNV12ToJPEG_FFmpeg exception: %s", e.what()); } // ✅ Ensure memory cleanup in case of exceptions if (yuvjFrame) av_frame_free(&yuvjFrame); if (codecCtx) avcodec_free_context(&codecCtx); av_packet_unref(&pkt); return ""; // Return empty string on error } std::string CVideoPlayer::avframeToJpegString(const AVFrame* spFrame) { std::lock_guard lock(_mutex); AVFrame* croppedFrame = nullptr; AVFrame* convertedFrame = nullptr; AVFrame* pFrame = const_cast(spFrame); bool isSuccess = false; std::string jpegData; try { if (!m_bPlaying) { return ""; } if (!spFrame || !spFrame->data[0] || spFrame->width <= 10 || spFrame->height <= 10) { ANS_DBG("MEDIA_Convert", "invalid or empty frame data / dimensions"); return ""; } // ✅ Process NV12 frames directly if (pFrame->format == AV_PIX_FMT_NV12) { croppedFrame = cropFrame(pFrame, m_Bbox, m_bCrop); if (!croppedFrame) { croppedFrame = pFrame; // Use original frame if cropping failed } // TurboJPEG handles all resolutions — no need for slow FFmpeg MJPEG path jpegData = encodeNV12ToJPEG_TurboJPEG(croppedFrame); if (!jpegData.empty()) { m_Width = croppedFrame->width; m_Height = croppedFrame->height; m_pts = m_pts + 1; isSuccess = true; } // ✅ Free cropped frame if allocated if (croppedFrame != pFrame) { av_frame_free(&croppedFrame); croppedFrame = nullptr; } } else { // ✅ Convert other formats to NV12 before processing initSwsContext(pFrame->width, pFrame->height, static_cast(pFrame->format), AV_PIX_FMT_NV12); convertedFrame = av_frame_alloc(); if (convertedFrame) { convertedFrame->format = AV_PIX_FMT_NV12; convertedFrame->width = pFrame->width; convertedFrame->height = pFrame->height; convertedFrame->color_range = AVCOL_RANGE_JPEG; if (av_frame_get_buffer(convertedFrame, 32) >= 0) { sws_scale(swsCtx, pFrame->data, pFrame->linesize, 0, pFrame->height, convertedFrame->data, convertedFrame->linesize); croppedFrame = cropFrame(convertedFrame, m_Bbox, m_bCrop); if (!croppedFrame) { croppedFrame = convertedFrame; // Use converted frame if cropping failed } // TurboJPEG handles all resolutions jpegData = encodeNV12ToJPEG_TurboJPEG(croppedFrame); if (!jpegData.empty()) { m_Width = croppedFrame->width; m_Height = croppedFrame->height; m_pts = m_pts + 1; isSuccess = true; } // ✅ Free cropped frame if allocated if (croppedFrame != convertedFrame) { av_frame_free(&croppedFrame); croppedFrame = nullptr; } } } // ✅ Free converted frame if allocated if (convertedFrame) { av_frame_free(&convertedFrame); convertedFrame = nullptr; } } return isSuccess ? jpegData : ""; } catch (const std::exception& e) { ANS_DBG("MEDIA_Jpeg", "avframeToJpegString exception: %s", e.what()); // ✅ Cleanup memory in case of exceptions if (croppedFrame && croppedFrame != pFrame && croppedFrame != convertedFrame) { av_frame_free(&croppedFrame); } if (convertedFrame) { av_frame_free(&convertedFrame); } return ""; } } bool CVideoPlayer::areFramesIdentical(AVFrame* frame1, AVFrame* frame2) { std::lock_guard lock(_mutex); try { if (!frame1 || !frame2) return false; // Ensure the frames have the same width, height, and format if (frame1->width != frame2->width || frame1->height != frame2->height || frame1->format != frame2->format) { return false; } int height = frame1->height; int width = frame1->width; // Compare Y plane (Luma) for (int y = 0; y < height; y++) { if (std::memcmp(frame1->data[0] + y * frame1->linesize[0], frame2->data[0] + y * frame2->linesize[0], width) != 0) { return false; } } if (frame1->format == AV_PIX_FMT_NV12) { // Compare UV plane (Interleaved) int chromaHeight = height / 2; for (int y = 0; y < chromaHeight; y++) { if (std::memcmp(frame1->data[1] + y * frame1->linesize[1], frame2->data[1] + y * frame2->linesize[1], width) != 0) { return false; } } } else if (frame1->format == AV_PIX_FMT_YUVJ420P) { // Compare U and V planes separately int chromaWidth = width / 2; int chromaHeight = height / 2; for (int y = 0; y < chromaHeight; y++) { if (std::memcmp(frame1->data[1] + y * frame1->linesize[1], // U frame2->data[1] + y * frame2->linesize[1], chromaWidth) != 0) { return false; } if (std::memcmp(frame1->data[2] + y * frame1->linesize[2], // V frame2->data[2] + y * frame2->linesize[2], chromaWidth) != 0) { return false; } } } return true; // If all planes match } catch (const std::exception& e) { ANS_DBG("MEDIA_Convert", "areFramesIdentical exception: %s", e.what()); return false; } } void CVideoPlayer::initSwsContext(int width, int height, AVPixelFormat pixFmt, AVPixelFormat outputPixFmt) { std::lock_guard lock(_mutex); try { // Validate input dimensions and pixel format if (width <= 0 || height <= 0 || pixFmt == AV_PIX_FMT_NONE) { ANS_DBG("MEDIA_Convert", "initSwsContext invalid params: w=%d h=%d pixFmt=%d", width, height, (int)pixFmt); return; } // Check if reinitialization is required bool needsReinit = (swsCtx == nullptr) || (width != lastWidth || height != lastHeight || pixFmt != lastPixFmt || outputPixFmt != lastOutPixFmt); if (!needsReinit) { // SwsContext is already up-to-date return; } // Free the existing SwsContext if it exists if (swsCtx) { sws_freeContext(swsCtx); swsCtx = nullptr; } // Determine output pixel format and scaling options based on resolution int scalingFlags = SWS_BILINEAR; // Fast scaling — LANCZOS is too slow for real-time // Create a new SwsContext swsCtx = sws_getContext(width, height, pixFmt, width, height, outputPixFmt, scalingFlags, nullptr, nullptr, nullptr); // Check for errors in SwsContext creation if (!swsCtx) { ANS_DBG("MEDIA_Convert", "sws_getContext failed: w=%d h=%d in=%d out=%d", width, height, (int)pixFmt, (int)outputPixFmt); return; } // Update last known parameters lastWidth = width; lastHeight = height; lastPixFmt = pixFmt; lastOutPixFmt = outputPixFmt; } catch (const std::exception& e) { ANS_DBG("MEDIA_Convert", "initSwsContext exception: %s", e.what()); } catch (...) { ANS_DBG("MEDIA_Convert", "initSwsContext unknown exception"); } } cv::Mat CVideoPlayer::avframeAnyToCvmat(const AVFrame* frame) { std::lock_guard lock(_mutex); // Protect against concurrent access try { if (!frame || !frame->data[0] || frame->width <= 10 || frame->height <= 10) { ANS_DBG("MEDIA_Convert", "invalid or empty frame data / dimensions"); return cv::Mat(); // Return an empty matrix if the frame is invalid } initSwsContext(frame->width, frame->height, static_cast(frame->format), AV_PIX_FMT_BGR24); // Create OpenCV Mat to store the resulting image cv::Mat image(frame->height, frame->width, CV_8UC3); uint8_t* dst[1] = { image.data }; int dstStride[1] = { static_cast(image.step[0]) }; // OpenCV's stride // Perform the conversion using sws_scale int result = sws_scale(swsCtx, frame->data, frame->linesize, 0, frame->height, dst, dstStride); if (result < 0) { ANS_DBG("MEDIA_Convert", "sws_scale failed"); return cv::Mat(); // Return an empty matrix if scaling fails } return image; // Return the successfully converted OpenCV Mat } catch (const std::exception& e) { ANS_DBG("MEDIA_Convert", "avframeToCvmat exception: %s", e.what()); return cv::Mat(); // Return an empty matrix on error } } cv::Mat CVideoPlayer::avframeYUVJ420PToCvmat(const AVFrame* frame) { std::lock_guard lock(_mutex); try { if (!frame || !frame->data[0] || frame->width <= 10 || frame->height <= 10) { ANS_DBG("MEDIA_Convert", "invalid or empty frame data / dimensions"); return cv::Mat(); } // Create OpenCV Mat for the output image (RGB) cv::Mat image(frame->height, frame->width, CV_8UC3); // 8-bit 3 channels for RGB image // Pointer to Y, U, V data from AVFrame uint8_t* yPlane = frame->data[0]; // Y plane (luminance) uint8_t* uPlane = frame->data[1]; // U plane (chrominance) uint8_t* vPlane = frame->data[2]; // V plane (chrominance) int yStride = frame->linesize[0]; // Stride of Y plane int uStride = frame->linesize[1]; // Stride of U plane int vStride = frame->linesize[2]; // Stride of V plane // Precompute offsets for U and V channels int uvWidth = frame->width / 2; // U and V are subsampled (half resolution) int uvHeight = frame->height / 2; // Loop through each pixel and convert YUV to RGB for (int y = 0; y < frame->height; ++y) { for (int x = 0; x < frame->width; ++x) { // Y, U, V values for each pixel int yVal = yPlane[y * yStride + x]; int uVal = uPlane[(y / 2) * uStride + (x / 2)]; int vVal = vPlane[(y / 2) * vStride + (x / 2)]; // Precompute differences for speed int uDiff = uVal - 128; int vDiff = vVal - 128; // Convert YUV to RGB (clamping values inline) int r = yVal + (1.402 * vDiff); int g = yVal - (0.344136 * uDiff) - (0.714136 * vDiff); int b = yVal + (1.772 * uDiff); // Clamp the values to the valid range for RGB (0-255) r = std::clamp(r, 0, 255); g = std::clamp(g, 0, 255); b = std::clamp(b, 0, 255); // Store the result in the OpenCV Mat (BGR format) image.at(y, x) = cv::Vec3b(b, g, r); // OpenCV uses BGR by default } } return image; // Return the converted OpenCV Mat (BGR) } catch (const std::exception& e) { ANS_DBG("MEDIA_Convert", "avframeToCvmatYUVJ420P exception: %s", e.what()); return cv::Mat(); // Return an empty matrix on error } } // Initialize a dedicated SwsContext for NV12→BGR with correct color space void CVideoPlayer::initNV12SwsContext(const AVFrame* frame) { int width = frame->width; int height = frame->height; // Detect color space from frame metadata (BT.709 for HD/4K, BT.601 for SD) int colorspace = SWS_CS_ITU709; // Default to BT.709 for HD/4K if (frame->colorspace == AVCOL_SPC_BT470BG || frame->colorspace == AVCOL_SPC_SMPTE170M) { colorspace = SWS_CS_ITU601; } else if (frame->colorspace == AVCOL_SPC_BT2020_NCL || frame->colorspace == AVCOL_SPC_BT2020_CL) { colorspace = SWS_CS_BT2020; } else if (frame->colorspace == AVCOL_SPC_BT709) { colorspace = SWS_CS_ITU709; } else if (width >= 1280 || height >= 720) { // Auto-detect: HD and above → BT.709 (most common for IP cameras) colorspace = SWS_CS_ITU709; } else { colorspace = SWS_CS_ITU601; // SD content } // Detect color range: limited (16-235) vs full (0-255) int srcRange = (frame->color_range == AVCOL_RANGE_JPEG) ? 1 : 0; // 0=limited, 1=full int dstRange = 1; // Output always full range (0-255) for display/AI processing // Check if reinit needed if (m_nv12SwsCtx && width == m_nv12LastWidth && height == m_nv12LastHeight && colorspace == m_nv12LastColorspace && srcRange == m_nv12LastRange) { return; // Already configured } // Free old context if (m_nv12SwsCtx) { sws_freeContext(m_nv12SwsCtx); m_nv12SwsCtx = nullptr; } // Create context: NV12 → BGR24, same dimensions (no scaling) // SWS_BILINEAR + SWS_FULL_CHR_H_INT: good quality chroma upsampling (~12ms for 4K) // SWS_ACCURATE_RND: better rounding for color precision // Note: SWS_LANCZOS gives VLC-matching quality but costs 50-80ms — too slow. // VLC achieves its quality via GPU shaders, not CPU processing. m_nv12SwsCtx = sws_getContext(width, height, AV_PIX_FMT_NV12, width, height, AV_PIX_FMT_BGR24, SWS_BILINEAR | SWS_ACCURATE_RND | SWS_FULL_CHR_H_INT, nullptr, nullptr, nullptr); if (!m_nv12SwsCtx) { ANS_DBG("MEDIA_Convert", "failed to create NV12 SwsContext"); return; } // Configure correct color space and range const int* coefficients = sws_getCoefficients(colorspace); int* inv_table; int* table; int curSrcRange, curDstRange, brightness, contrast, saturation; sws_getColorspaceDetails(m_nv12SwsCtx, &inv_table, &curSrcRange, &table, &curDstRange, &brightness, &contrast, &saturation); sws_setColorspaceDetails(m_nv12SwsCtx, coefficients, srcRange, coefficients, dstRange, brightness, contrast, saturation); m_nv12LastWidth = width; m_nv12LastHeight = height; m_nv12LastColorspace = colorspace; m_nv12LastRange = srcRange; } cv::Mat CVideoPlayer::avframeNV12ToCvMat(const AVFrame* frame) { try { if (!frame || frame->width <= 0 || frame->height <= 0) { ANS_DBG("MEDIA_Convert", "invalid frame: null/wrong-format/zero-dims"); return cv::Mat(); } // Software decode handler if (frame->format != AV_PIX_FMT_NV12) return avframeAnyToCvmat(frame); int width = frame->width; int height = frame->height; // Store original NV12 dimensions for inference coordinate mapping m_nv12OrigWidth = width; m_nv12OrigHeight = height; // Return full-resolution BGR image. // No forced downscale — LabVIEW manages display resolution via SetDisplayResolution(). // If the caller needs a specific display size, SetDisplayResolution(w, h) applies // resizing in GetImage() at the ANSRTSP/ANS*Client level after this returns. // Store original NV12 dimensions for inference coordinate mapping m_nv12OrigWidth = width; m_nv12OrigHeight = height; // Preferred path: libyuv NV12→RGB24 (BGR in memory, matches cv::Mat). // SIMD (SSSE3/AVX2) and ~3-5x faster than sws_scale's quality config. // Crucially it honors src_stride_uv, so the padded strides/height that // NVDEC and D3D11VA emit (e.g. 1080→1088 rows, aligned strides) convert // correctly — that padding is what cv::cvtColorTwoPlane mishandled, // reading the chroma plane at the wrong offset and rendering a green // cast with striped noise toward the bottom of the frame. // Coefficients are chosen from the frame's signalled colorspace/range, // mirroring initNV12SwsContext() (BT.709 for HD/4K, BT.601 for SD, // BT.2020 if tagged). The constants map studio-range YUV to full-range // BGR, so the result is display-ready — no manual range expansion. #if defined(ANSCORE_HAS_LIBYUV) && ANSCORE_HAS_LIBYUV { const bool fullRange = (frame->color_range == AVCOL_RANGE_JPEG); const libyuv::YuvConstants* yuvc; switch (frame->colorspace) { case AVCOL_SPC_BT470BG: case AVCOL_SPC_SMPTE170M: yuvc = fullRange ? &libyuv::kYuvJPEGConstants : &libyuv::kYuvI601Constants; break; case AVCOL_SPC_BT2020_NCL: case AVCOL_SPC_BT2020_CL: yuvc = fullRange ? &libyuv::kYuvV2020Constants : &libyuv::kYuv2020Constants; break; case AVCOL_SPC_BT709: yuvc = fullRange ? &libyuv::kYuvF709Constants : &libyuv::kYuvH709Constants; break; default: // Auto-detect by resolution (matches initNV12SwsContext()). if (width >= 1280 || height >= 720) yuvc = fullRange ? &libyuv::kYuvF709Constants : &libyuv::kYuvH709Constants; else yuvc = fullRange ? &libyuv::kYuvJPEGConstants : &libyuv::kYuvI601Constants; break; } cv::Mat bgrImage(height, width, CV_8UC3); int ret = libyuv::NV12ToRGB24Matrix( frame->data[0], frame->linesize[0], frame->data[1], frame->linesize[1], bgrImage.data, static_cast(bgrImage.step), yuvc, width, height); if (ret == 0) { return bgrImage; } ANS_DBG("MEDIA_Convert", "libyuv NV12ToRGB24 failed (ret=%d) — falling back to sws_scale", ret); } #endif // Fallback path: sws_scale via the dedicated NV12→BGR24 context. Also // stride-safe (honors frame->linesize[]) and colorspace/range-correct; // used on non-libyuv builds or if libyuv ever reports an error. { // Dedicated lock (not _mutex) so close()/dtor cannot free the context // mid-scale, while still keeping onVideoFrame off the hot _mutex. std::lock_guard nv12Lock(m_nv12SwsMutex); initNV12SwsContext(frame); if (m_nv12SwsCtx) { cv::Mat bgrImage(height, width, CV_8UC3); uint8_t* dst[1] = { bgrImage.data }; int dstStride[1] = { static_cast(bgrImage.step[0]) }; int ret = sws_scale(m_nv12SwsCtx, frame->data, frame->linesize, 0, height, dst, dstStride); if (ret > 0) { return bgrImage; } ANS_DBG("MEDIA_Convert", "NV12 sws_scale failed (ret=%d) — falling back to cvtColorTwoPlane", ret); } } // Fallback: OpenCV two-plane conversion. Correct only when the planes // are unpadded (linesize == width); kept for builds/inputs where the // sws context could not be created. cv::Mat yPlane(height, width, CV_8UC1, frame->data[0], frame->linesize[0]); cv::Mat uvPlane(height / 2, width / 2, CV_8UC2, frame->data[1], frame->linesize[1]); cv::Mat bgrImage; cv::cvtColorTwoPlane(yPlane, uvPlane, bgrImage, cv::COLOR_YUV2BGR_NV12); if (m_nImageQuality == 1) { bgrImage.convertTo(bgrImage, -1, 255.0 / 219.0, -16.0 * 255.0 / 219.0); } return bgrImage; } catch (const std::exception& e) { ANS_DBG("MEDIA_Convert", "avframeNV12ToCvMat exception: %s", e.what()); return cv::Mat(); } } cv::Mat CVideoPlayer::avframeYUV420PToCvMat(const AVFrame* frame) { try { if (!frame || frame->width <= 0 || frame->height <= 0) { return cv::Mat(); } const int width = frame->width; const int height = frame->height; // Debug: confirm this SW-decode conversion is actually hit. // Throttled to ~1 log/sec at 30 fps to keep DebugView readable. // Gated by ANSCORE_DEBUGVIEW — compiles to nothing in production. { static std::atomic s_swCallCount{0}; uint64_t n = s_swCallCount.fetch_add(1, std::memory_order_relaxed); if ((n % 30) == 0) { const char* fmtName = av_get_pix_fmt_name((AVPixelFormat)frame->format); const bool contig = (frame->linesize[0] == width && frame->linesize[1] == width / 2 && frame->linesize[2] == width / 2 && frame->data[1] == frame->data[0] + width * height && frame->data[2] == frame->data[1] + (width / 2) * (height / 2)); // Report the codec's allocated Y-plane height (inferred from // the Y/U pointer spacing and Y stride). Lets us see whether // our custom get_buffer2 achieved alloc_h == visible_h. const int yStrideDbg = frame->linesize[0] > 0 ? frame->linesize[0] : 1; const int alloc_h_y = (int)((frame->data[1] - frame->data[0]) / yStrideDbg); #if defined(ANSCORE_HAS_LIBYUV) && ANSCORE_HAS_LIBYUV const char* pathLabel = "LIBYUV/I420ToRGB24"; #else const char* pathLabel = contig ? "FAST/zero-copy" : (frame->linesize[0] == width) ? "SLOW/bulk-memcpy" : "SLOW/per-row-copy"; #endif (void)contig; // silence unused warning when libyuv is on ANS_DBG("MEDIA_SWDec", "avframeYUV420PToCvMat ENTRY call#%llu fmt=%s visible=%dx%d alloc_h_y=%d " "linesize=[%d,%d,%d] path=%s (this=%p)", (unsigned long long)n, fmtName ? fmtName : "?", width, height, alloc_h_y, frame->linesize[0], frame->linesize[1], frame->linesize[2], pathLabel, (void*)this); } } #if defined(ANSCORE_HAS_LIBYUV) && ANSCORE_HAS_LIBYUV // libyuv path: direct I420/J420 (3 strided planes) → RGB24 (== BGR in // memory order for libyuv, matches cv::Mat CV_8UC3 default). No staging // buffer, no memcpy, no cv::cvtColor — one SIMD-optimized sweep. // // libyuv's "RGB24" is B,G,R per pixel in memory (see RGB24ToARGBRow_C // in libyuv/source/row_common.cc where src[0]=b, src[1]=g, src[2]=r). // That matches OpenCV's BGR layout — safe to wrap in CV_8UC3. // // YUVJ420P signals full-range ("JPEG-style"); use J420ToRGB24 so the // luma/chroma offsets are not applied. Otherwise studio-range I420. cv::Mat bgrImage(height, width, CV_8UC3); int ret = (frame->format == AV_PIX_FMT_YUVJ420P) ? libyuv::J420ToRGB24( frame->data[0], frame->linesize[0], frame->data[1], frame->linesize[1], frame->data[2], frame->linesize[2], bgrImage.data, static_cast(bgrImage.step), width, height) : libyuv::I420ToRGB24( frame->data[0], frame->linesize[0], frame->data[1], frame->linesize[1], frame->data[2], frame->linesize[2], bgrImage.data, static_cast(bgrImage.step), width, height); if (ret != 0) { ANS_DBG("MEDIA_Convert", "libyuv I420/J420ToRGB24 failed ret=%d %dx%d", ret, width, height); return cv::Mat(); } if (m_nImageQuality == 1) { bgrImage.convertTo(bgrImage, -1, 255.0 / 219.0, -16.0 * 255.0 / 219.0); } return bgrImage; #else // YUV420P has 3 separate planes: Y (full res), U (half), V (half). // OpenCV's cvtColor(COLOR_YUV2BGR_I420) expects a single contiguous buffer // with Y on top (H rows) and U,V stacked below (H/2 rows total). // Layout: [Y: W×H] [U: W/2 × H/2] [V: W/2 × H/2] // Total height = H * 3/2, width = W, single channel. // If all planes are contiguous with matching strides, wrap directly const int yStride = frame->linesize[0]; const int uStride = frame->linesize[1]; const int vStride = frame->linesize[2]; // Fast path: planes are packed contiguously with stride == width if (yStride == width && uStride == width / 2 && vStride == width / 2 && frame->data[1] == frame->data[0] + width * height && frame->data[2] == frame->data[1] + (width / 2) * (height / 2)) { // Contiguous I420 — wrap directly, zero copy cv::Mat yuv(height * 3 / 2, width, CV_8UC1, frame->data[0]); cv::Mat bgrImage; cv::cvtColor(yuv, bgrImage, cv::COLOR_YUV2BGR_I420); if (m_nImageQuality == 1) { bgrImage.convertTo(bgrImage, -1, 255.0 / 219.0, -16.0 * 255.0 / 219.0); } return bgrImage; } // Slow path: planes have padding (linesize > width) OR Y/U/V live in // non-adjacent buffers. Copy into a single I420-layout staging buffer // so cvtColor(COLOR_YUV2BGR_I420) can process it in one SIMD sweep. const int uvWidth = width / 2; const int uvHeight = height / 2; const int totalSize = width * height + uvWidth * uvHeight * 2; // Thread-local staging Mat — reused across calls to avoid a 12 MB malloc // on every 4K frame. Each decoder runs on its own worker thread, so // thread_local is the right granularity (no cross-thread sharing, no // locking). The Mat reallocates only when dimensions change. static thread_local cv::Mat s_yuvStaging; if (s_yuvStaging.rows != height * 3 / 2 || s_yuvStaging.cols != width || s_yuvStaging.type() != CV_8UC1 || !s_yuvStaging.isContinuous()) { s_yuvStaging.create(height * 3 / 2, width, CV_8UC1); } cv::Mat& yuv = s_yuvStaging; uint8_t* dst = yuv.data; // Copy Y plane (line by line if stride != width) if (yStride == width) { std::memcpy(dst, frame->data[0], width * height); } else { for (int row = 0; row < height; ++row) { std::memcpy(dst + row * width, frame->data[0] + row * yStride, width); } } dst += width * height; // Copy U plane if (uStride == uvWidth) { std::memcpy(dst, frame->data[1], uvWidth * uvHeight); } else { for (int row = 0; row < uvHeight; ++row) { std::memcpy(dst + row * uvWidth, frame->data[1] + row * uStride, uvWidth); } } dst += uvWidth * uvHeight; // Copy V plane if (vStride == uvWidth) { std::memcpy(dst, frame->data[2], uvWidth * uvHeight); } else { for (int row = 0; row < uvHeight; ++row) { std::memcpy(dst + row * uvWidth, frame->data[2] + row * vStride, uvWidth); } } cv::Mat bgrImage; cv::cvtColor(yuv, bgrImage, cv::COLOR_YUV2BGR_I420); if (m_nImageQuality == 1) { bgrImage.convertTo(bgrImage, -1, 255.0 / 219.0, -16.0 * 255.0 / 219.0); } return bgrImage; #endif // ANSCORE_HAS_LIBYUV } catch (const std::exception& e) { ANS_DBG("MEDIA_Convert", "avframeYUV420PToCvMat exception: %s", e.what()); return cv::Mat(); } } // YUV422P / YUVJ422P → BGR via libyuv I422ToRGB24. // libyuv provides only studio-range I422ToRGB24 (no J422ToRGB24), so for // full-range YUVJ422P we route through J422ToARGB + ARGBToRGB24 to keep // the correct full-range luma coefficients. cv::Mat CVideoPlayer::avframeYUV422PToCvMat(const AVFrame* frame) { try { if (!frame || frame->width <= 0 || frame->height <= 0) return cv::Mat(); #if defined(ANSCORE_HAS_LIBYUV) && ANSCORE_HAS_LIBYUV const int width = frame->width; const int height = frame->height; cv::Mat bgrImage(height, width, CV_8UC3); int ret; if (frame->format == AV_PIX_FMT_YUVJ422P) { // Full-range: J422ToARGB (BGRA in memory) → ARGBToRGB24 (BGR) static thread_local std::vector s_argbBuf; const size_t argbBytes = static_cast(width) * height * 4; if (s_argbBuf.size() < argbBytes) s_argbBuf.resize(argbBytes); const int argbStride = width * 4; int r1 = libyuv::J422ToARGB( frame->data[0], frame->linesize[0], frame->data[1], frame->linesize[1], frame->data[2], frame->linesize[2], s_argbBuf.data(), argbStride, width, height); if (r1 != 0) { ANS_DBG("MEDIA_Convert", "libyuv J422ToARGB failed ret=%d", r1); return cv::Mat(); } ret = libyuv::ARGBToRGB24( s_argbBuf.data(), argbStride, bgrImage.data, static_cast(bgrImage.step), width, height); } else { ret = libyuv::I422ToRGB24( frame->data[0], frame->linesize[0], frame->data[1], frame->linesize[1], frame->data[2], frame->linesize[2], bgrImage.data, static_cast(bgrImage.step), width, height); } if (ret != 0) { ANS_DBG("MEDIA_Convert", "libyuv I422/J422ToRGB24 failed ret=%d", ret); return cv::Mat(); } if (m_nImageQuality == 1) { bgrImage.convertTo(bgrImage, -1, 255.0 / 219.0, -16.0 * 255.0 / 219.0); } return bgrImage; #else return avframeAnyToCvmat(frame); #endif } catch (const std::exception& e) { ANS_DBG("MEDIA_Convert", "avframeYUV422PToCvMat exception: %s", e.what()); return cv::Mat(); } } // YUV444P / YUVJ444P → BGR via libyuv I444ToRGB24 (studio) or J444ToARGB + // ARGBToRGB24 (full-range — no direct J444ToRGB24 in libyuv). cv::Mat CVideoPlayer::avframeYUV444PToCvMat(const AVFrame* frame) { try { if (!frame || frame->width <= 0 || frame->height <= 0) return cv::Mat(); #if defined(ANSCORE_HAS_LIBYUV) && ANSCORE_HAS_LIBYUV const int width = frame->width; const int height = frame->height; cv::Mat bgrImage(height, width, CV_8UC3); int ret; if (frame->format == AV_PIX_FMT_YUVJ444P) { static thread_local std::vector s_argbBuf; const size_t argbBytes = static_cast(width) * height * 4; if (s_argbBuf.size() < argbBytes) s_argbBuf.resize(argbBytes); const int argbStride = width * 4; int r1 = libyuv::J444ToARGB( frame->data[0], frame->linesize[0], frame->data[1], frame->linesize[1], frame->data[2], frame->linesize[2], s_argbBuf.data(), argbStride, width, height); if (r1 != 0) { ANS_DBG("MEDIA_Convert", "libyuv J444ToARGB failed ret=%d", r1); return cv::Mat(); } ret = libyuv::ARGBToRGB24( s_argbBuf.data(), argbStride, bgrImage.data, static_cast(bgrImage.step), width, height); } else { ret = libyuv::I444ToRGB24( frame->data[0], frame->linesize[0], frame->data[1], frame->linesize[1], frame->data[2], frame->linesize[2], bgrImage.data, static_cast(bgrImage.step), width, height); } if (ret != 0) { ANS_DBG("MEDIA_Convert", "libyuv I444/J444ToRGB24 failed ret=%d", ret); return cv::Mat(); } if (m_nImageQuality == 1) { bgrImage.convertTo(bgrImage, -1, 255.0 / 219.0, -16.0 * 255.0 / 219.0); } return bgrImage; #else return avframeAnyToCvmat(frame); #endif } catch (const std::exception& e) { ANS_DBG("MEDIA_Convert", "avframeYUV444PToCvMat exception: %s", e.what()); return cv::Mat(); } } // YUV420P10LE → BGR via libyuv I010ToARGB (BGRA in memory) → ARGBToRGB24 (BGR). // 10-bit planar: Y/U/V are uint16 little-endian with 10 valid bits; FFmpeg's // linesize[] is in bytes, libyuv expects it in uint16 elements → divide by 2. cv::Mat CVideoPlayer::avframeYUV420P10LEToCvMat(const AVFrame* frame) { try { if (!frame || frame->width <= 0 || frame->height <= 0) return cv::Mat(); #if defined(ANSCORE_HAS_LIBYUV) && ANSCORE_HAS_LIBYUV const int width = frame->width; const int height = frame->height; // Thread-local ARGB staging — ~8MB @ 1080p, ~32MB @ 4K. Reused across // calls to avoid malloc+free on the hot path. static thread_local std::vector s_argbBuf; const size_t argbBytes = static_cast(width) * height * 4; if (s_argbBuf.size() < argbBytes) s_argbBuf.resize(argbBytes); const int argbStride = width * 4; int r1 = libyuv::I010ToARGB( reinterpret_cast(frame->data[0]), frame->linesize[0] / 2, reinterpret_cast(frame->data[1]), frame->linesize[1] / 2, reinterpret_cast(frame->data[2]), frame->linesize[2] / 2, s_argbBuf.data(), argbStride, width, height); if (r1 != 0) { ANS_DBG("MEDIA_Convert", "libyuv I010ToARGB failed ret=%d", r1); return cv::Mat(); } cv::Mat bgrImage(height, width, CV_8UC3); int r2 = libyuv::ARGBToRGB24( s_argbBuf.data(), argbStride, bgrImage.data, static_cast(bgrImage.step), width, height); if (r2 != 0) { ANS_DBG("MEDIA_Convert", "libyuv ARGBToRGB24 failed ret=%d", r2); return cv::Mat(); } if (m_nImageQuality == 1) { bgrImage.convertTo(bgrImage, -1, 255.0 / 219.0, -16.0 * 255.0 / 219.0); } return bgrImage; #else return avframeAnyToCvmat(frame); #endif } catch (const std::exception& e) { ANS_DBG("MEDIA_Convert", "avframeYUV420P10LEToCvMat exception: %s", e.what()); return cv::Mat(); } } // YUV422P10LE → BGR via libyuv I210ToARGB → ARGBToRGB24. cv::Mat CVideoPlayer::avframeYUV422P10LEToCvMat(const AVFrame* frame) { try { if (!frame || frame->width <= 0 || frame->height <= 0) return cv::Mat(); #if defined(ANSCORE_HAS_LIBYUV) && ANSCORE_HAS_LIBYUV const int width = frame->width; const int height = frame->height; static thread_local std::vector s_argbBuf; const size_t argbBytes = static_cast(width) * height * 4; if (s_argbBuf.size() < argbBytes) s_argbBuf.resize(argbBytes); const int argbStride = width * 4; int r1 = libyuv::I210ToARGB( reinterpret_cast(frame->data[0]), frame->linesize[0] / 2, reinterpret_cast(frame->data[1]), frame->linesize[1] / 2, reinterpret_cast(frame->data[2]), frame->linesize[2] / 2, s_argbBuf.data(), argbStride, width, height); if (r1 != 0) { ANS_DBG("MEDIA_Convert", "libyuv I210ToARGB failed ret=%d", r1); return cv::Mat(); } cv::Mat bgrImage(height, width, CV_8UC3); int r2 = libyuv::ARGBToRGB24( s_argbBuf.data(), argbStride, bgrImage.data, static_cast(bgrImage.step), width, height); if (r2 != 0) { ANS_DBG("MEDIA_Convert", "libyuv ARGBToRGB24 failed ret=%d", r2); return cv::Mat(); } if (m_nImageQuality == 1) { bgrImage.convertTo(bgrImage, -1, 255.0 / 219.0, -16.0 * 255.0 / 219.0); } return bgrImage; #else return avframeAnyToCvmat(frame); #endif } catch (const std::exception& e) { ANS_DBG("MEDIA_Convert", "avframeYUV422P10LEToCvMat exception: %s", e.what()); return cv::Mat(); } } // YUV444P10LE → BGR via libyuv I410ToARGBMatrix + ARGBToRGB24. // Only the Matrix variant is provided by libyuv for 10-bit 4:4:4, so pass // an explicit BT.601 constants block (matches the implicit default used by // the non-Matrix ToARGB calls above). cv::Mat CVideoPlayer::avframeYUV444P10LEToCvMat(const AVFrame* frame) { try { if (!frame || frame->width <= 0 || frame->height <= 0) return cv::Mat(); #if defined(ANSCORE_HAS_LIBYUV) && ANSCORE_HAS_LIBYUV const int width = frame->width; const int height = frame->height; static thread_local std::vector s_argbBuf; const size_t argbBytes = static_cast(width) * height * 4; if (s_argbBuf.size() < argbBytes) s_argbBuf.resize(argbBytes); const int argbStride = width * 4; int r1 = libyuv::I410ToARGBMatrix( reinterpret_cast(frame->data[0]), frame->linesize[0] / 2, reinterpret_cast(frame->data[1]), frame->linesize[1] / 2, reinterpret_cast(frame->data[2]), frame->linesize[2] / 2, s_argbBuf.data(), argbStride, &libyuv::kYuvI601Constants, width, height); if (r1 != 0) { ANS_DBG("MEDIA_Convert", "libyuv I410ToARGBMatrix failed ret=%d", r1); return cv::Mat(); } cv::Mat bgrImage(height, width, CV_8UC3); int r2 = libyuv::ARGBToRGB24( s_argbBuf.data(), argbStride, bgrImage.data, static_cast(bgrImage.step), width, height); if (r2 != 0) { ANS_DBG("MEDIA_Convert", "libyuv ARGBToRGB24 failed ret=%d", r2); return cv::Mat(); } if (m_nImageQuality == 1) { bgrImage.convertTo(bgrImage, -1, 255.0 / 219.0, -16.0 * 255.0 / 219.0); } return bgrImage; #else return avframeAnyToCvmat(frame); #endif } catch (const std::exception& e) { ANS_DBG("MEDIA_Convert", "avframeYUV444P10LEToCvMat exception: %s", e.what()); return cv::Mat(); } } // YUV420P12LE → BGR via libyuv I012ToARGBMatrix + ARGBToRGB24. // libyuv's 12-bit path truncates the low bits to 8-bit ARGB internally — fine // for inference which is 8-bit anyway. cv::Mat CVideoPlayer::avframeYUV420P12LEToCvMat(const AVFrame* frame) { try { if (!frame || frame->width <= 0 || frame->height <= 0) return cv::Mat(); #if defined(ANSCORE_HAS_LIBYUV) && ANSCORE_HAS_LIBYUV const int width = frame->width; const int height = frame->height; static thread_local std::vector s_argbBuf; const size_t argbBytes = static_cast(width) * height * 4; if (s_argbBuf.size() < argbBytes) s_argbBuf.resize(argbBytes); const int argbStride = width * 4; int r1 = libyuv::I012ToARGBMatrix( reinterpret_cast(frame->data[0]), frame->linesize[0] / 2, reinterpret_cast(frame->data[1]), frame->linesize[1] / 2, reinterpret_cast(frame->data[2]), frame->linesize[2] / 2, s_argbBuf.data(), argbStride, &libyuv::kYuvI601Constants, width, height); if (r1 != 0) { ANS_DBG("MEDIA_Convert", "libyuv I012ToARGBMatrix failed ret=%d", r1); return cv::Mat(); } cv::Mat bgrImage(height, width, CV_8UC3); int r2 = libyuv::ARGBToRGB24( s_argbBuf.data(), argbStride, bgrImage.data, static_cast(bgrImage.step), width, height); if (r2 != 0) { ANS_DBG("MEDIA_Convert", "libyuv ARGBToRGB24 failed ret=%d", r2); return cv::Mat(); } if (m_nImageQuality == 1) { bgrImage.convertTo(bgrImage, -1, 255.0 / 219.0, -16.0 * 255.0 / 219.0); } return bgrImage; #else return avframeAnyToCvmat(frame); #endif } catch (const std::exception& e) { ANS_DBG("MEDIA_Convert", "avframeYUV420P12LEToCvMat exception: %s", e.what()); return cv::Mat(); } } // YUV422P12LE → BGR via libyuv I212ToI422 (12→8-bit 4:2:2 downconvert) then // I422ToRGB24. libyuv has no direct I212ToARGB, so the two-step SIMD chain // is the fast path. Intermediate I422 staging = 2*W*H bytes (≈16 MB @ 4K). cv::Mat CVideoPlayer::avframeYUV422P12LEToCvMat(const AVFrame* frame) { try { if (!frame || frame->width <= 0 || frame->height <= 0) return cv::Mat(); #if defined(ANSCORE_HAS_LIBYUV) && ANSCORE_HAS_LIBYUV const int width = frame->width; const int height = frame->height; const int uvW = width / 2; // Thread-local 8-bit I422 staging: [Y: W×H][U: W/2×H][V: W/2×H] static thread_local std::vector s_i422Buf; const size_t i422Bytes = static_cast(width) * height + static_cast(uvW) * height * 2; if (s_i422Buf.size() < i422Bytes) s_i422Buf.resize(i422Bytes); uint8_t* y8 = s_i422Buf.data(); uint8_t* u8 = y8 + static_cast(width) * height; uint8_t* v8 = u8 + static_cast(uvW) * height; int r1 = libyuv::I212ToI422( reinterpret_cast(frame->data[0]), frame->linesize[0] / 2, reinterpret_cast(frame->data[1]), frame->linesize[1] / 2, reinterpret_cast(frame->data[2]), frame->linesize[2] / 2, y8, width, u8, uvW, v8, uvW, width, height); if (r1 != 0) { ANS_DBG("MEDIA_Convert", "libyuv I212ToI422 failed ret=%d", r1); return cv::Mat(); } cv::Mat bgrImage(height, width, CV_8UC3); int r2 = libyuv::I422ToRGB24( y8, width, u8, uvW, v8, uvW, bgrImage.data, static_cast(bgrImage.step), width, height); if (r2 != 0) { ANS_DBG("MEDIA_Convert", "libyuv I422ToRGB24 failed ret=%d", r2); return cv::Mat(); } if (m_nImageQuality == 1) { bgrImage.convertTo(bgrImage, -1, 255.0 / 219.0, -16.0 * 255.0 / 219.0); } return bgrImage; #else return avframeAnyToCvmat(frame); #endif } catch (const std::exception& e) { ANS_DBG("MEDIA_Convert", "avframeYUV422P12LEToCvMat exception: %s", e.what()); return cv::Mat(); } } // YUV444P12LE → BGR via libyuv I412ToI444 (12→8-bit 4:4:4 downconvert) then // I444ToRGB24. Intermediate I444 staging = 3*W*H bytes (≈24 MB @ 4K). cv::Mat CVideoPlayer::avframeYUV444P12LEToCvMat(const AVFrame* frame) { try { if (!frame || frame->width <= 0 || frame->height <= 0) return cv::Mat(); #if defined(ANSCORE_HAS_LIBYUV) && ANSCORE_HAS_LIBYUV const int width = frame->width; const int height = frame->height; // Thread-local 8-bit I444 staging: [Y: W×H][U: W×H][V: W×H] static thread_local std::vector s_i444Buf; const size_t planeBytes = static_cast(width) * height; const size_t i444Bytes = planeBytes * 3; if (s_i444Buf.size() < i444Bytes) s_i444Buf.resize(i444Bytes); uint8_t* y8 = s_i444Buf.data(); uint8_t* u8 = y8 + planeBytes; uint8_t* v8 = u8 + planeBytes; int r1 = libyuv::I412ToI444( reinterpret_cast(frame->data[0]), frame->linesize[0] / 2, reinterpret_cast(frame->data[1]), frame->linesize[1] / 2, reinterpret_cast(frame->data[2]), frame->linesize[2] / 2, y8, width, u8, width, v8, width, width, height); if (r1 != 0) { ANS_DBG("MEDIA_Convert", "libyuv I412ToI444 failed ret=%d", r1); return cv::Mat(); } cv::Mat bgrImage(height, width, CV_8UC3); int r2 = libyuv::I444ToRGB24( y8, width, u8, width, v8, width, bgrImage.data, static_cast(bgrImage.step), width, height); if (r2 != 0) { ANS_DBG("MEDIA_Convert", "libyuv I444ToRGB24 failed ret=%d", r2); return cv::Mat(); } if (m_nImageQuality == 1) { bgrImage.convertTo(bgrImage, -1, 255.0 / 219.0, -16.0 * 255.0 / 219.0); } return bgrImage; #else return avframeAnyToCvmat(frame); #endif } catch (const std::exception& e) { ANS_DBG("MEDIA_Convert", "avframeYUV444P12LEToCvMat exception: %s", e.what()); return cv::Mat(); } } cv::Mat CVideoPlayer::avframeToCVMat(const AVFrame* pFrame) { // No _mutex here: caller (getImage) releases the mutex before invoking this // so the expensive NV12/YUV420P→BGR conversion does not block onVideoFrame. // NV12/YUV420P paths touch only the caller-owned AVFrame clone and benign // member reads. avframeAnyToCvmat() takes its own lock for swsCtx. try { // 1. Validate input frame if (!pFrame || !pFrame->data[0] || pFrame->width <= 10 || pFrame->height <= 10) { ANS_DBG("MEDIA_Convert", "invalid or empty frame data / dimensions"); return cv::Mat(); } // One-shot diagnostic: log the pixel format the first time through so // we can see which branch of the switch below is taken. Production: only // emitted when ANSCORE_DEBUGVIEW is enabled. static bool s_loggedFmt = false; if (!s_loggedFmt) { s_loggedFmt = true; const char* name = av_get_pix_fmt_name((AVPixelFormat)pFrame->format); ANS_DBG("MEDIA_Convert", "avframeToCVMat FIRST-FRAME fmt=%d(%s) %dx%d HWDecoding=%d (this=%p)", pFrame->format, name ? name : "?", pFrame->width, pFrame->height, m_nHWDecoding, (void*)this); } // Per-branch throttled trace so we can see the dispatch at runtime. // Gated by ANSCORE_DEBUGVIEW — zero overhead in production. static std::atomic s_dispatchCount{0}; const uint64_t dispN = s_dispatchCount.fetch_add(1, std::memory_order_relaxed); const bool logThis = ((dispN % 30) == 0); switch (pFrame->format) { case AV_PIX_FMT_NV12: if (logThis) { ANS_DBG("MEDIA_Convert", "DISPATCH call#%llu fmt=NV12 %dx%d -> avframeNV12ToCvMat (HW-decode path)", (unsigned long long)dispN, pFrame->width, pFrame->height); } return avframeNV12ToCvMat(pFrame); case AV_PIX_FMT_YUV420P: case AV_PIX_FMT_YUVJ420P: if (logThis) { ANS_DBG("MEDIA_Convert", "DISPATCH call#%llu fmt=%s %dx%d -> avframeYUV420PToCvMat (libyuv SW path)", (unsigned long long)dispN, (pFrame->format == AV_PIX_FMT_YUVJ420P) ? "YUVJ420P" : "YUV420P", pFrame->width, pFrame->height); } return avframeYUV420PToCvMat(pFrame); case AV_PIX_FMT_YUV422P: case AV_PIX_FMT_YUVJ422P: if (logThis) { ANS_DBG("MEDIA_Convert", "DISPATCH call#%llu fmt=%s %dx%d -> avframeYUV422PToCvMat (libyuv SW path)", (unsigned long long)dispN, (pFrame->format == AV_PIX_FMT_YUVJ422P) ? "YUVJ422P" : "YUV422P", pFrame->width, pFrame->height); } return avframeYUV422PToCvMat(pFrame); case AV_PIX_FMT_YUV444P: case AV_PIX_FMT_YUVJ444P: if (logThis) { ANS_DBG("MEDIA_Convert", "DISPATCH call#%llu fmt=%s %dx%d -> avframeYUV444PToCvMat (libyuv SW path)", (unsigned long long)dispN, (pFrame->format == AV_PIX_FMT_YUVJ444P) ? "YUVJ444P" : "YUV444P", pFrame->width, pFrame->height); } return avframeYUV444PToCvMat(pFrame); case AV_PIX_FMT_YUV420P10LE: if (logThis) { ANS_DBG("MEDIA_Convert", "DISPATCH call#%llu fmt=YUV420P10LE %dx%d -> avframeYUV420P10LEToCvMat (libyuv SW path)", (unsigned long long)dispN, pFrame->width, pFrame->height); } return avframeYUV420P10LEToCvMat(pFrame); case AV_PIX_FMT_YUV422P10LE: if (logThis) { ANS_DBG("MEDIA_Convert", "DISPATCH call#%llu fmt=YUV422P10LE %dx%d -> avframeYUV422P10LEToCvMat (libyuv SW path)", (unsigned long long)dispN, pFrame->width, pFrame->height); } return avframeYUV422P10LEToCvMat(pFrame); case AV_PIX_FMT_YUV444P10LE: if (logThis) { ANS_DBG("MEDIA_Convert", "DISPATCH call#%llu fmt=YUV444P10LE %dx%d -> avframeYUV444P10LEToCvMat (libyuv SW path)", (unsigned long long)dispN, pFrame->width, pFrame->height); } return avframeYUV444P10LEToCvMat(pFrame); case AV_PIX_FMT_YUV420P12LE: if (logThis) { ANS_DBG("MEDIA_Convert", "DISPATCH call#%llu fmt=YUV420P12LE %dx%d -> avframeYUV420P12LEToCvMat (libyuv SW path)", (unsigned long long)dispN, pFrame->width, pFrame->height); } return avframeYUV420P12LEToCvMat(pFrame); case AV_PIX_FMT_YUV422P12LE: if (logThis) { ANS_DBG("MEDIA_Convert", "DISPATCH call#%llu fmt=YUV422P12LE %dx%d -> avframeYUV422P12LEToCvMat (libyuv SW path, 12→8→BGR)", (unsigned long long)dispN, pFrame->width, pFrame->height); } return avframeYUV422P12LEToCvMat(pFrame); case AV_PIX_FMT_YUV444P12LE: if (logThis) { ANS_DBG("MEDIA_Convert", "DISPATCH call#%llu fmt=YUV444P12LE %dx%d -> avframeYUV444P12LEToCvMat (libyuv SW path, 12→8→BGR)", (unsigned long long)dispN, pFrame->width, pFrame->height); } return avframeYUV444P12LEToCvMat(pFrame); default: if (logThis) { const char* name = av_get_pix_fmt_name((AVPixelFormat)pFrame->format); ANS_DBG("MEDIA_Convert", "DISPATCH call#%llu fmt=%d(%s) %dx%d -> avframeAnyToCvmat (sws_scale fallback)", (unsigned long long)dispN, pFrame->format, name ? name : "?", pFrame->width, pFrame->height); } return avframeAnyToCvmat(pFrame); } } catch (const std::exception& e) { ANS_DBG("MEDIA_Convert", "avframeToCvMat dispatcher exception: %s", e.what()); return cv::Mat(); // Return an empty matrix on error } } CVideoPlayer::CVideoPlayer() : m_bVideoInited(FALSE) , m_bAudioInited(FALSE) , m_bPlaying(FALSE) , m_bPaused(FALSE) , m_nHWDecoding(HW_DECODING_DISABLE)// Software decode by default — saves VRAM (no NVDEC DPB surfaces) , m_bUpdown(FALSE) , m_bSnapshot(FALSE) , m_nSnapVideoFmt(AV_PIX_FMT_YUVJ420P) , m_nVideoCodec(VIDEO_CODEC_NONE) , m_nAudioCodec(AUDIO_CODEC_NONE) , m_nSampleRate(0) , m_nChannel(0) , m_nBitPerSample(0) , m_pSnapFrame(NULL) , m_bRecording(FALSE) , m_bNalFlag(FALSE) , m_pAviCtx(NULL) , m_pAudioListMutex(NULL) , m_audioPlayFlag(FALSE) //, m_audioPlayThread(0) , m_pVideoListMutex(NULL) , m_videoPlayFlag(FALSE) //, m_videoPlayThread(0) , m_nLastAudioPts(AV_NOPTS_VALUE) , m_lastAudioTS(0) { m_Bbox.x = 0; m_Bbox.y = 0; m_Bbox.width = 0; m_Bbox.height = 0; m_bCrop = false; m_pRecordMutex = sys_os_create_mutex(); m_lastJpegImage = ""; m_jpegImage = ""; m_pts = 0; memset(&m_h26XParamSets, 0, sizeof(H26XParamSets)); memset(&m_audioClock, 0, sizeof(HTCLOCK)); memset(&m_videoClock, 0, sizeof(HTCLOCK)); this->_tjInstance = tjInitCompress(); } CVideoPlayer::~CVideoPlayer() { // Lock to ensure no other thread is mid-operation (getImage, getJpegImage, onVideoFrame) // before we free resources. close() stops the decoder which prevents new callbacks. { std::lock_guard lock(_mutex); close(); // Stop decoder first — prevents new onVideoFrame callbacks g_frameQueue.clearQueue(); if (swsCtx != nullptr) { sws_freeContext(swsCtx); swsCtx = nullptr; } { // Lock order: _mutex (held) → m_nv12SwsMutex. Protects against a // consumer thread still inside avframeNV12ToCvMat's sws_scale. std::lock_guard nv12Lock(m_nv12SwsMutex); if (m_nv12SwsCtx != nullptr) { sws_freeContext(m_nv12SwsCtx); m_nv12SwsCtx = nullptr; } } if (this->_tjInstance) { tjDestroy(this->_tjInstance); this->_tjInstance = nullptr; } } // _mutex is destroyed after this block — no other thread should be accessing this object } BOOL CVideoPlayer::open(std::string fileName) { m_sFileName = fileName; return TRUE; } BOOL CVideoPlayer::open(std::string _username, std::string _password, std::string _url) { m_acct = _username; m_pass = _password; m_sFileName = _url; return TRUE; } AVFrame* CVideoPlayer::getNV12Frame() { // Return a CLONE so multiple consumers (tasks sharing the same stream) // each get their own copy. The original m_currentNV12Frame stays valid // until the next getImage() call overwrites it. // (Previously used ownership transfer — only the first caller got NV12, // and the second caller fell back to BGR.) std::lock_guard lock(_mutex); AVFrame* clone = m_currentNV12Frame ? av_frame_clone(m_currentNV12Frame) : nullptr; if (clone) g_nv12Escapes.fetch_add(1, std::memory_order_relaxed); return clone; } AVFrame* CVideoPlayer::getCudaHWFrame() { // Return a clone of the CUDA HW frame captured by onVideoFrame(). // Clone (not ownership transfer) because multiple callers may request // the frame between onVideoFrame updates (e.g., during warmup when // GetRTSPCVImage is called faster than the decode rate). // extra_hw_frames=2 in the decoder provides surface pool headroom // for the 3 concurrent clones (decoder + player + registry). std::lock_guard lock(_mutex); AVFrame* clone = m_currentCudaHWFrame ? av_frame_clone(m_currentCudaHWFrame) : nullptr; if (clone) g_cudaHWEscapes.fetch_add(1, std::memory_order_relaxed); return clone; } bool CVideoPlayer::isCudaHWAccel() const { return m_pVideoDecoder && m_pVideoDecoder->isCudaHWAccel(); } void CVideoPlayer::close() { closeVideo(); closeAudio(); if (m_currentNV12Frame) { av_frame_free(&m_currentNV12Frame); g_nv12Frees.fetch_add(1, std::memory_order_relaxed); m_currentNV12Frame = nullptr; } if (m_currentCudaHWFrame) { av_frame_free(&m_currentCudaHWFrame); g_cudaHWFrees.fetch_add(1, std::memory_order_relaxed); m_currentCudaHWFrame = nullptr; } if (m_pSnapFrame) { av_frame_free(&m_pSnapFrame); m_pSnapFrame = nullptr; } stopRecord(); if (m_pRecordMutex) { sys_os_destroy_sig_mutex(m_pRecordMutex); m_pRecordMutex = NULL; } } void CVideoPlayer::setVolume(int volume) { if (m_pAudioPlay) { m_pAudioPlay->setVolume(volume); } } void CVideoPlayer::snapshot(int videofmt) { m_bSnapshot = TRUE; m_nSnapVideoFmt = videofmt; } BOOL CVideoPlayer::record(std::string baseName) { if (m_bRecording) { return TRUE; } //std::string path = getRecordPath(); std::string file = baseName;// path + "/" + getTempFile(baseName, ".avi"); m_sBaseName = baseName; m_pAviCtx = avi_write_open(file.c_str()); if (NULL == m_pAviCtx) { log_print(HT_LOG_ERR, "%s, avi_write_open failed. %s\r\n", __FUNCTION__, file.c_str()); return FALSE; } if (!onRecord()) { avi_write_close(m_pAviCtx); m_pAviCtx = NULL; return FALSE; } m_bRecording = TRUE; return m_bRecording; } void CVideoPlayer::stopRecord() { sys_os_mutex_enter(m_pRecordMutex); m_bRecording = FALSE; m_bNalFlag = FALSE; memset(&m_h26XParamSets, 0, sizeof(H26XParamSets)); if (m_pAviCtx) { avi_write_close(m_pAviCtx); m_pAviCtx = NULL; } sys_os_mutex_leave(m_pRecordMutex); } void CVideoPlayer::recordVideo(uint8* data, int len, uint32 ts, uint16 seq) { int codec = VIDEO_CODEC_NONE; if (!memcmp(m_pAviCtx->v_fcc, "H264", 4)) { codec = VIDEO_CODEC_H264; } else if (!memcmp(m_pAviCtx->v_fcc, "H265", 4)) { codec = VIDEO_CODEC_H265; } if ((VIDEO_CODEC_H264 == codec || VIDEO_CODEC_H265 == codec) && !m_bNalFlag) { if (avc_get_h26x_paramsets(data, len, codec, &m_h26XParamSets)) { avi_write_nalu(m_pAviCtx, m_h26XParamSets.vps, m_h26XParamSets.vps_size, m_h26XParamSets.sps, m_h26XParamSets.sps_size, m_h26XParamSets.pps, m_h26XParamSets.pps_size); m_bNalFlag = 1; } } recordVideoEx(data, len, ts, seq); if (recordSwitchCheck()) { recordFileSwitch(); } } void CVideoPlayer::recordVideoEx(uint8* data, int len, uint32 ts, uint16 seq) { AVICTX* p_avictx = m_pAviCtx; if (p_avictx->v_width == 0 || p_avictx->v_height == 0) { int codec = VIDEO_CODEC_NONE; if (memcmp(p_avictx->v_fcc, "H264", 4) == 0) { codec = VIDEO_CODEC_H264; } else if (memcmp(p_avictx->v_fcc, "H265", 4) == 0) { codec = VIDEO_CODEC_H265; } else if (memcmp(p_avictx->v_fcc, "JPEG", 4) == 0) { codec = VIDEO_CODEC_JPEG; } else if (memcmp(p_avictx->v_fcc, "MP4V", 4) == 0) { codec = VIDEO_CODEC_MP4; } avc_parse_video_size(codec, data, len, &p_avictx->v_width, &p_avictx->v_height); if (p_avictx->v_width && p_avictx->v_height) { avi_update_header(p_avictx); } } int key = 0; if (memcmp(p_avictx->v_fcc, "H264", 4) == 0) { uint8 nalu_t = (data[4] & 0x1F); key = (nalu_t == 5 || nalu_t == 7 || nalu_t == 8); } else if (memcmp(p_avictx->v_fcc, "H265", 4) == 0) { uint8 nalu_t = (data[4] >> 1) & 0x3F; key = ((nalu_t >= 16 && nalu_t <= 21) || nalu_t == 32 || nalu_t == 33 || nalu_t == 34); } else if (memcmp(p_avictx->v_fcc, "MP4V", 4) == 0) { key = 1; } else if (memcmp(p_avictx->v_fcc, "JPEG", 4) == 0) { key = 1; } avi_write_video(p_avictx, data, len, ts, key); } void CVideoPlayer::recordAudio(uint8* data, int len, uint32 ts, uint16 seq) { AVICTX* p_avictx = m_pAviCtx; avi_write_audio(p_avictx, data, len, ts); if (recordSwitchCheck()) { recordFileSwitch(); } } BOOL CVideoPlayer::recordSwitchCheck() { uint64 tlen = avi_get_file_length(m_pAviCtx); uint32 mtime = avi_get_media_time(m_pAviCtx); uint32 recordSize = 0;// getRecordSize(); if (recordSize == 0) { recordSize = 1048576; // max 1G file size } // Switch according to the recording size if (tlen > recordSize * 1024) { return TRUE; } uint32 recordTime = 0;// getRecordTime(); // Switch according to the recording duration if (recordTime > 0 && mtime > recordTime * 1000) { return TRUE; } return FALSE; } void CVideoPlayer::recordFileSwitch() { AVICTX* p_ctx; AVICTX* p_oldctx = m_pAviCtx; //std::string path = getRecordPath(); std::string file = m_sBaseName;// path + "/" + getTempFile(m_sBaseName, ".avi"); p_ctx = avi_write_open(file.c_str()); if (NULL == p_ctx) { return; } p_ctx->ctxf_video = p_oldctx->ctxf_video; p_ctx->ctxf_audio = p_oldctx->ctxf_audio; if (p_ctx->ctxf_video) { avi_calc_fps(p_oldctx); avi_set_video_info(p_ctx, p_oldctx->v_fps, p_oldctx->v_width, p_oldctx->v_height, p_oldctx->v_fcc); avi_set_video_extra_info(p_ctx, p_oldctx->v_extra, p_oldctx->v_extra_len); } if (p_ctx->ctxf_audio) { avi_set_audio_info(p_ctx, p_oldctx->a_chns, p_oldctx->a_rate, p_oldctx->a_fmt); avi_set_audio_extra_info(p_ctx, p_oldctx->a_extra, p_oldctx->a_extra_len); } avi_write_close(p_oldctx); avi_update_header(p_ctx); m_pAviCtx = p_ctx; if (m_h26XParamSets.vps_size > 0 || m_h26XParamSets.sps_size > 0 || m_h26XParamSets.pps_size > 0) { avi_write_nalu(m_pAviCtx, m_h26XParamSets.vps, m_h26XParamSets.vps_size, m_h26XParamSets.sps, m_h26XParamSets.sps_size, m_h26XParamSets.pps, m_h26XParamSets.pps_size); } } BOOL CVideoPlayer::openVideo(enum AVCodecID codec, uint8* extradata, int extradata_size) { if (m_bVideoInited) { return TRUE; } if (m_pVideoDecoder) { m_bVideoInited = m_pVideoDecoder->init(codec, extradata, extradata_size, m_nHWDecoding, m_nPreferredGpu); } if (m_bVideoInited) { m_pVideoDecoder->setCallback(VideoDecoderCallback, this); m_pVideoListMutex = sys_os_create_mutex(); m_videoPlayFlag = TRUE; //m_videoPlayThread = sys_os_create_thread((void*)VideoPlayThread, this); } m_nVideoCodec = to_video_codec(codec); return m_bVideoInited; } BOOL CVideoPlayer::openVideo(int codec, uint8* extradata, int extradata_size) { return openVideo(to_video_avcodecid(codec), extradata, extradata_size); } void CVideoPlayer::closeVideo() { // Stop decoder outside the player lock to avoid the same lock-ordering // deadlock as StopVideoDecoder() (see comment there). CVideoDecoder* decoder = nullptr; { std::lock_guard lock(_mutex); decoder = m_pVideoDecoder.get(); } if (decoder) { decoder->Stop(); decoder->flush(); } // Now clean up resources under the lock std::lock_guard lock(_mutex); m_videoPlayFlag = FALSE; if (m_pVideoListMutex) { sys_os_destroy_sig_mutex(m_pVideoListMutex); m_pVideoListMutex = NULL; } if (!g_frameQueue.isEmpty())g_frameQueue.clearQueue(); m_bVideoInited = FALSE; } void CVideoPlayer::StartVideoDecoder() { std::lock_guard lock(_mutex); // Clear queue but KEEP m_currentImage — it holds the last good frame // which we'll return while the decoder stabilizes after restart g_frameQueue.clearQueue(); m_lastFrameSeq = 0; m_bWaitingForKeyframe = true; // Skip frames until first keyframe m_cleanFrameCount = 0; // Reset settle counter // Reset jitter buffer state so a restart starts measuring rate fresh m_arrivalIdx = 0; m_arrivalCount = 0; m_observedFrameIntervalMs = 40.0; // re-seed at 25 fps m_lastReleaseTimeSet = false; m_jitterBufferRefilling = true; // new fill phase on (re)start if (m_pVideoDecoder) { m_pVideoDecoder->Start(); } } void CVideoPlayer::StopVideoDecoder() { // Get decoder pointer under lock, then release BEFORE calling decoder methods. // This avoids a lock-ordering deadlock: // Thread 1 (here): CVideoPlayer::_mutex -> CVideoDecoder::_mutex // Thread 2 (TCP rx decode -> onVideoFrame callback): CVideoDecoder::_mutex -> CVideoPlayer::_mutex CVideoDecoder* decoder = nullptr; { std::lock_guard lock(_mutex); decoder = m_pVideoDecoder.get(); } if (decoder) { decoder->Stop(); // Flush decoder to drain and discard any buffered frames, // so stale reference frames don't corrupt the next session decoder->flush(); // Free NVDEC decoder context and all GPU surfaces (DPB buffers). // Stopped cameras should not hold VRAM — with 100 cameras created // but only 5 running, the 95 idle decoders would consume ~5-10 GB. // The decoder will be re-initialized automatically when the next // video packet arrives after Start() is called. decoder->uninit(); m_bVideoInited = FALSE; } // Clear queue but KEEP m_currentImage and m_lastJpegImage — // getImage()/getJpegImage() will return the last good frame while decoder stabilizes { std::lock_guard lock(_mutex); g_frameQueue.clearQueue(); m_lastFrameSeq = 0; } } BOOL CVideoPlayer::openAudio(enum AVCodecID codec, int samplerate, int channels, int bitpersample) { if (m_bAudioInited) { return TRUE; } if (m_pAudioDecoder) { m_bAudioInited = m_pAudioDecoder->init(codec, samplerate, channels, bitpersample); } if (m_bAudioInited) { m_pAudioDecoder->setCallback(AudioDecoderCallback, this); #if __WINDOWS_OS__ m_pAudioPlay = std::make_unique();/// new CWAudioPlay(); #elif defined(IOS) m_pAudioPlay = std::make_unique(); #elif __LINUX_OS__ m_pAudioPlay = std::make_unique(); #endif if (m_pAudioPlay) { m_pAudioPlay->startPlay(samplerate, channels); } m_pAudioListMutex = sys_os_create_mutex(); m_audioPlayFlag = FALSE;//disable by default //m_audioPlayThread = sys_os_create_thread((void*)AudioPlayThread, this); } m_nAudioCodec = to_audio_codec(codec); m_nSampleRate = samplerate; m_nChannel = channels; m_nBitPerSample = bitpersample; return m_bAudioInited; } void CVideoPlayer::enableAudio(bool status) { if (status)m_audioPlayFlag = TRUE; else m_audioPlayFlag = FALSE; } BOOL CVideoPlayer::openAudio(int codec, int samplerate, int channels, int bitpersample) { return openAudio(to_audio_avcodecid(codec), samplerate, channels, bitpersample); } void CVideoPlayer::closeAudio() { m_audioPlayFlag = FALSE; if (m_pAudioListMutex) { sys_os_destroy_sig_mutex(m_pAudioListMutex); m_pAudioListMutex = NULL; } if (!a_frameQueue.isEmpty())a_frameQueue.clearQueue(); m_bAudioInited = FALSE; } int CVideoPlayer::getVideoWidth() { if (m_pVideoDecoder) { return m_pVideoDecoder->getWidth(); } return 0; } int CVideoPlayer::getVideoHeight() { if (m_pVideoDecoder) { return m_pVideoDecoder->getHeight(); } return 0; } double CVideoPlayer::getFrameRate() { if (m_pVideoDecoder) { return m_pVideoDecoder->getFrameRate(); } return 0; } void CVideoPlayer::setTargetFPS(double intervalMs) { std::lock_guard lock(_mutex); m_targetIntervalMs = intervalMs; m_targetFPSInitialized = false; // reset timing on change } void CVideoPlayer::setJitterBufferMs(int ms) { std::lock_guard lock(_mutex); if (ms < 0) ms = 0; if (ms > 2000) ms = 2000; // sanity ceiling — beyond this is just lag m_jitterBufferMs = ms; // Reset release timing and re-enter fill phase so the new setting takes // effect cleanly on the next call. m_lastReleaseTimeSet = false; m_jitterBufferRefilling = (ms > 0); } double CVideoPlayer::getLastFrameAgeMs() { std::lock_guard lock(_mutex); if (!m_lastDecoderFrameTimeSet) return 0.0; auto now = std::chrono::steady_clock::now(); return std::chrono::duration(now - m_lastDecoderFrameTime).count(); } void CVideoPlayer::playVideo(uint8* data, int len, uint32 ts, uint16 seq) { if (m_bRecording) { sys_os_mutex_enter(m_pRecordMutex); recordVideo(data, len, ts, seq); sys_os_mutex_leave(m_pRecordMutex); } updateClock(&m_videoClock, ts, getVideoClock()); if (m_bVideoInited) { if (m_bPlaying) { m_pVideoDecoder->decode(data, len, m_videoClock.SyncTime.tv_sec * 1000000 + m_videoClock.SyncTime.tv_usec); } } } void CVideoPlayer::playAudio(uint8* data, int len, uint32 ts, uint16 seq) { if (m_bRecording) { sys_os_mutex_enter(m_pRecordMutex); recordAudio(data, len, ts, seq); sys_os_mutex_leave(m_pRecordMutex); } updateClock(&m_audioClock, ts, getAudioClock()); if (m_bAudioInited) { m_pAudioDecoder->decode(data, len, m_audioClock.SyncTime.tv_sec * 1000000 + m_audioClock.SyncTime.tv_usec); } } void CVideoPlayer::updateClock(HTCLOCK* clock, uint32 ts, int frequency) { if (ts == 0) { return; } if (clock->SyncTime.tv_sec == 0 && clock->SyncTime.tv_usec == 0) { clock->SyncTimestamp = ts; gettimeofday(&clock->SyncTime, NULL); } int timestampDiff = ts - clock->SyncTimestamp; // Divide this by the timestamp frequency to get real time: double timeDiff = timestampDiff / (double)frequency; uint32 const million = 1000000; uint32 seconds, uSeconds; if (timeDiff >= 0.0) { seconds = clock->SyncTime.tv_sec + (uint32)(timeDiff); uSeconds = clock->SyncTime.tv_usec + (uint32)((timeDiff - (uint32)timeDiff) * million); if (uSeconds >= million) { uSeconds -= million; ++seconds; } } else { timeDiff = -timeDiff; seconds = clock->SyncTime.tv_sec - (uint32)(timeDiff); uSeconds = clock->SyncTime.tv_usec - (uint32)((timeDiff - (uint32)timeDiff) * million); if ((int)uSeconds < 0) { uSeconds += million; --seconds; } } // Save these as the new synchronization timestamp & time: clock->SyncTimestamp = ts; clock->SyncTime.tv_sec = seconds; clock->SyncTime.tv_usec = uSeconds; } BOOL CVideoPlayer::initFrame(AVFrame*& frame, int width, int height, AVPixelFormat pixfmt) { if (width == 0 || height == 0 || pixfmt == AV_PIX_FMT_NONE) { return FALSE; } if (NULL == frame || frame->width != width || frame->height != height || frame->format != pixfmt) { if (frame) { av_frame_free(&frame); } frame = av_frame_alloc(); if (NULL == frame) { return FALSE; } frame->format = pixfmt; frame->width = width; frame->height = height; if (0 != av_frame_get_buffer(frame, 0)) { av_frame_free(&frame); return FALSE; } av_frame_make_writable(frame); } return TRUE; } BOOL CVideoPlayer::doSnapshot(AVFrame* frame) { if (m_pSnapFrame) { av_frame_free(&m_pSnapFrame); // Free the previous snapshot frame if it exists } if (!initFrame(m_pSnapFrame, frame->width, frame->height, to_avpixelformat(m_nSnapVideoFmt))) { return FALSE; } if (NULL == convertFrame(frame, m_pSnapFrame, FALSE)) { return FALSE; } return TRUE; } AVFrame* CVideoPlayer::convertFrame(AVFrame* srcframe, AVFrame* dstframe, BOOL updown) { if (!srcframe || !dstframe) { return NULL; } SwsContext* _swsctx = sws_getContext(srcframe->width, srcframe->height, (enum AVPixelFormat)srcframe->format, srcframe->width, srcframe->height, (enum AVPixelFormat)dstframe->format, SWS_BICUBIC, NULL, NULL, NULL); if (!_swsctx) { return NULL; } if (updown) { srcframe->data[0] += srcframe->linesize[0] * (srcframe->height - 1); srcframe->linesize[0] *= -1; srcframe->data[1] += srcframe->linesize[1] * (srcframe->height / 2 - 1); srcframe->linesize[1] *= -1; srcframe->data[2] += srcframe->linesize[2] * (srcframe->height / 2 - 1); srcframe->linesize[2] *= -1; } int ret = sws_scale(_swsctx, srcframe->data, srcframe->linesize, 0, srcframe->height, dstframe->data, dstframe->linesize); sws_freeContext(_swsctx); // Free context after scaling attempt if (ret > 0) { dstframe->pts = srcframe->pts; dstframe->pkt_dts = srcframe->pkt_dts; return dstframe; } else { log_print(HT_LOG_ERR, "%s, sws_scale failed\r\n", __FUNCTION__); return NULL; } } void CVideoPlayer::onVideoFrame(AVFrame* frame) { std::lock_guard lock(_mutex); // Protect against concurrent access if (!frame) return; // Check for null pointer if (m_bSnapshot) { if (doSnapshot(frame)) { m_bSnapshot = FALSE; } } if (m_bPlaying && m_videoPlayFlag) { // MEDIA_Pulse: count every frame the decoder hands us, before any drop checks. g_pulseArrived.fetch_add(1, std::memory_order_relaxed); note_arrival_gap(); // tracks max gap between consecutive arrivals in this 1s window // --- Arrival-rate measurement for the jitter buffer --- // Record this arrival in the ring buffer, then update the observed // inter-frame interval from the median delta of the last samples. // Median is robust to bursts (TCP unstall delivers several frames in // rapid succession; mean would be skewed, median picks up the typical // in-burst spacing). Adapts within ~1 s when camera rate changes. { using clk = std::chrono::steady_clock; const long long now = clk::now().time_since_epoch().count(); m_arrivalTicks[m_arrivalIdx % kArrivalRingSize] = now; m_arrivalIdx++; if (m_arrivalCount < kArrivalRingSize) m_arrivalCount++; // Recompute every 8 frames (cheap; ring of 32 ints). if (m_arrivalCount >= 4 && (m_arrivalIdx % 8) == 0) { const size_t n = m_arrivalCount; std::array sortedCopy = m_arrivalTicks; std::sort(sortedCopy.begin(), sortedCopy.begin() + n); std::array deltas{}; for (size_t i = 1; i < n; ++i) deltas[i - 1] = sortedCopy[i] - sortedCopy[i - 1]; // Median delta std::sort(deltas.begin(), deltas.begin() + (n - 1)); const long long medianTicks = deltas[(n - 1) / 2]; const double medianMs = std::chrono::duration( clk::duration(medianTicks)).count(); // Clamp to sane camera framerate range (5–120 fps) so a // degenerate measurement can't poison the buffer logic. if (medianMs >= 8.0 && medianMs <= 200.0) { m_observedFrameIntervalMs = medianMs; } } } // Pass frames with decode_error_flags through. The codec is configured // with AV_CODEC_FLAG_OUTPUT_CORRUPT + AV_EF_IGNORE_ERR (video_decoder.cpp) // so it has already applied concealment for missing references — the // frame data is usable even if the flag is set. // // Why this matters: HW decoders (NVDEC/D3D11VA/DXVA2) silently conceal // without setting the flag, but the SW decoder (default since commit // e134ebd) sets the flag on every UDP packet loss. Dropping flagged // frames meant every lost packet froze the display until the next IDR // (~1-2s typical), reproducing the periodic freeze the user saw under // UDP+SW. VLC plays through these frames — so do we now. if (frame->decode_error_flags != 0) { g_pulseDropDecodeErr.fetch_add(1, std::memory_order_relaxed); // counted, not dropped ANS_DBG("MEDIA_DecodeErr", "frame had decode_error_flags=0x%x — passing through with concealment " "(pict_type=%d key=%d)", frame->decode_error_flags, (int)frame->pict_type, (int)frame->key_frame); } // After start/restart, skip corrupted frames until first keyframe (IDR) arrives. // HEVC/H.264 P/B frames received before the first I-frame will produce visual // corruption ("Could not find ref with POC", green/grey artifacts). if (m_bWaitingForKeyframe) { if (frame->key_frame || frame->pict_type == AV_PICTURE_TYPE_I) { m_bWaitingForKeyframe = false; m_cleanFrameCount = 0; ANS_DBG("MEDIA_Lifecycle", "first keyframe received, settling for %d frames", SETTLE_FRAME_COUNT); } else { g_pulseDropKeyframeWait.fetch_add(1, std::memory_order_relaxed); emit_media_pulse_if_due(g_frameQueue.size()); return; // Drop this frame — not yet safe to decode } } // Record wall-clock time of every decoded frame (even rate-limited ones). // Used by getLastFrameAgeMs() to detect truly stale cameras. m_lastDecoderFrameTime = std::chrono::steady_clock::now(); m_lastDecoderFrameTimeSet = true; // --- Frame rate limiting --- // Skip post-decode processing (clone, queue push, CUDA clone) if not enough // time has elapsed since the last processed frame. The decode itself still // runs for every packet to maintain the H.264/H.265 reference frame chain. if (m_targetIntervalMs > 0.0) { auto now = std::chrono::steady_clock::now(); if (!m_targetFPSInitialized) { m_lastProcessedTime = now; m_targetFPSInitialized = true; } else { auto elapsed = std::chrono::duration(now - m_lastProcessedTime).count(); if (elapsed < m_targetIntervalMs) { g_pulseDropRateLimit.fetch_add(1, std::memory_order_relaxed); emit_media_pulse_if_due(g_frameQueue.size()); return; // Skip this frame — too soon } } m_lastProcessedTime = now; } // --- End frame rate limiting --- // Push frame to queue; during settle period getImage() will ignore the queue // and keep returning the last good cached image g_frameQueue.pushFrame(frame); // pushFrame() clones the frame internally g_pulsePushed.fetch_add(1, std::memory_order_relaxed); // Capture CUDA HW frame for zero-copy inference. // We're inside decode()'s lock scope (decoder._mutex held) AND onVideoFrame // holds player._mutex — so this is the ONE place where both locks are held // and we can safely clone the CUDA frame without deadlock risk. // cloneCudaHWFrame_unlocked() is safe because decoder._mutex is already held. if (m_pVideoDecoder && m_pVideoDecoder->isCudaHWAccel()) { if (m_currentCudaHWFrame) { av_frame_free(&m_currentCudaHWFrame); g_cudaHWFrees.fetch_add(1, std::memory_order_relaxed); } m_currentCudaHWFrame = m_pVideoDecoder->cloneCudaHWFrame_unlocked(); if (m_currentCudaHWFrame) g_cudaHWClones.fetch_add(1, std::memory_order_relaxed); } // Track how many clean frames have arrived since keyframe if (m_cleanFrameCount < SETTLE_FRAME_COUNT) { m_cleanFrameCount++; if (m_cleanFrameCount == SETTLE_FRAME_COUNT) { ANS_DBG("MEDIA_Lifecycle", "settle complete, delivering new frames"); } } emit_media_pulse_if_due(g_frameQueue.size()); } } void CVideoPlayer::onAudioFrame(AVFrame* frame) { // Support for audio playback std::lock_guard lock(_mutex); // Protect against concurrent access if (!frame) return; // Check for null pointer if (m_bSnapshot) { if (doSnapshot(frame)) { m_bSnapshot = FALSE; } } if (m_bPlaying && m_audioPlayFlag) { a_frameQueue.pushFrame(frame); // pushFrame() clones the frame internally } } cv::Mat CVideoPlayer::getImage(int& width, int& height, int64_t& pts) { try { AVFrame* frameToProcess = nullptr; uint64_t currentSeq = 0; // MEDIA_Pulse: count every getImage call for the per-second pulse, // and try to emit on entry so the line still prints during a freeze // where every call lands on a cached-return path. The once-per-second // CAS inside makes the actual cost trivial. g_pulseGetCalls.fetch_add(1, std::memory_order_relaxed); emit_media_pulse_if_due(g_frameQueue.size()); // Timing breakdown — gated by ANSCORE_DEBUGVIEW (zero overhead in production). // t0 = entry, t1 = after pulling frame from queue, t2 = after YUV->BGR, // t3 = after publish. Throttled to every 30 full-path calls (~1/sec @30fps). using clk = std::chrono::steady_clock; const auto t0 = clk::now(); // --- Phase 1: short locked section — examine state, pull frame --- { std::lock_guard lock(_mutex); if (!m_bPlaying) { width = m_currentImage.cols; height = m_currentImage.rows; pts = m_pts; g_pulseGetCachedNotPlaying.fetch_add(1, std::memory_order_relaxed); return m_currentImage; // Shallow copy (reference counted) } // While waiting for keyframe or during settle period after restart, // return the last good cached image to avoid showing corrupted frames if (m_bWaitingForKeyframe || m_cleanFrameCount < SETTLE_FRAME_COUNT) { width = m_currentImage.cols; height = m_currentImage.rows; pts = m_pts; g_pulseGetCachedSettle.fetch_add(1, std::memory_order_relaxed); return m_currentImage; } currentSeq = g_frameQueue.getSequence(); if (m_jitterBufferMs > 0 && m_observedFrameIntervalMs > 0.0) { // ========================================================= // Jitter-buffer mode: FIFO + rate-paced + adaptive depth // + fill-phase refill // ========================================================= const double frameMs = m_observedFrameIntervalMs; const size_t targetN = std::max(1, static_cast(static_cast(m_jitterBufferMs) / frameMs)); const size_t maxAllowed = 2 * targetN; const size_t qSize = g_frameQueue.size(); // (1) Adaptive drop-ahead: if the consumer is slower than the // producer, the queue overshoots target. Pop & discard // down to targetN so latency stays bounded. Each discard // counts as a "skipped" frame in the pulse. if (qSize > maxAllowed) { while (g_frameQueue.size() > targetN) { AVFrame* old = g_frameQueue.popFrame(); if (old) { av_frame_free(&old); g_queueFrees.fetch_add(1, std::memory_order_relaxed); g_pulseSeqJump.fetch_add(1, std::memory_order_relaxed); } } } // (2) Fill phase: hold all releases until the queue reaches // target depth. Triggered at startup and again whenever // the queue drains to empty mid-stream. Without this, // 1.0x pacing keeps the queue at whatever depth it // happens to be (zero, after a drain) — the buffer // never refills. if (m_jitterBufferRefilling) { if (g_frameQueue.size() >= targetN) { m_jitterBufferRefilling = false; } else { width = m_currentImage.cols; height = m_currentImage.rows; pts = m_pts; g_pulseGetCachedFast.fetch_add(1, std::memory_order_relaxed); return m_currentImage; } } // (3) Rate-paced release at 1.0x of observed producer interval. // Anything <1.0 leaks the buffer (consumer faster than // producer); >1.0 keeps growing the queue until drop-ahead // trips. 1.0 keeps the queue at its current depth in // steady state. if (m_lastReleaseTimeSet) { const double sinceLastMs = std::chrono::duration( t0 - m_lastReleaseTime).count(); if (sinceLastMs < frameMs) { width = m_currentImage.cols; height = m_currentImage.rows; pts = m_pts; g_pulseGetCachedFast.fetch_add(1, std::memory_order_relaxed); return m_currentImage; } } // (4) Empty queue (network gap drained the buffer): re-enter // fill phase so the next producer pushes are held back to // rebuild the buffer to target depth before the next // release. Without this, a one-time gap permanently // shrinks the buffer and we lose absorption capacity. if (g_frameQueue.isEmpty()) { m_jitterBufferRefilling = true; width = m_currentImage.cols; height = m_currentImage.rows; pts = m_pts; g_pulseGetCachedEmpty.fetch_add(1, std::memory_order_relaxed); return cv::Mat(); } // (5) FIFO pop — the frame is now ours; we must free it after // conversion (popFrame returns the original, not a clone). frameToProcess = g_frameQueue.popFrame(); if (!frameToProcess) { m_jitterBufferRefilling = true; width = m_currentImage.cols; height = m_currentImage.rows; pts = m_pts; g_pulseGetCachedEmpty.fetch_add(1, std::memory_order_relaxed); return cv::Mat(); } m_lastReleaseTime = t0; m_lastReleaseTimeSet = true; } else { // ========================================================= // Legacy mode (m_jitterBufferMs == 0): getLatest semantics // ========================================================= // Fast path: same frame as last call — skip clone + BGR conversion if (currentSeq == m_lastFrameSeq && !m_currentImage.empty()) { width = m_currentImage.cols; height = m_currentImage.rows; pts = m_pts; g_pulseGetCachedFast.fetch_add(1, std::memory_order_relaxed); return m_currentImage; } if (g_frameQueue.isEmpty()) { width = m_currentImage.cols; height = m_currentImage.rows; pts = m_pts; g_pulseGetCachedEmpty.fetch_add(1, std::memory_order_relaxed); // Common case during network jitter — gated to avoid stderr spam. ANS_DBG("MEDIA_GetImage", "queue empty, returning cached"); return cv::Mat(); } // getLatestFrame() clones the AVFrame — we own it from here frameToProcess = g_frameQueue.getLatestFrame(); if (!frameToProcess) { width = m_currentImage.cols; height = m_currentImage.rows; pts = m_pts; g_pulseGetCachedEmpty.fetch_add(1, std::memory_order_relaxed); return cv::Mat(); } } } // --- _mutex released here --- // At 4K NV12, cvtColorTwoPlane takes ~100–300 ms on CPU; during that // window the decoder callback (onVideoFrame) is free to push the next // frame and the CUDA HW capture path can run in parallel. const auto t1 = clk::now(); cv::Mat converted; try { converted = avframeToCVMat(frameToProcess); } catch (const std::exception& e) { ANS_DBG("MEDIA_GetImage", "avframeToCVMat threw: %s — returning empty Mat", e.what()); } const auto t2 = clk::now(); // --- Phase 2: short locked section — publish new frame state --- cv::Mat result; // Snapshot taken under the lock, returned after release. { std::lock_guard lock(_mutex); if (!converted.empty()) { // Skip-tracking: only meaningful in legacy mode where the // consumer reads the back of the queue. In jitter-buffer mode // we already counted drop-ahead discards; sequence numbers // don't track displayed-vs-skipped because pop is FIFO. if (m_jitterBufferMs == 0) { if (m_lastFrameSeq != 0 && currentSeq > m_lastFrameSeq + 1) { g_pulseSeqJump.fetch_add(currentSeq - m_lastFrameSeq - 1, std::memory_order_relaxed); } m_lastFrameSeq = currentSeq; } m_currentImage = converted; m_pts++; g_pulseGetNew.fetch_add(1, std::memory_order_relaxed); } // Preserve raw YUV/NV12 frame for GPU fast-path inference // (NV12 from HW decode, YUV420P/YUVJ420P from SW decode) if (frameToProcess && (frameToProcess->format == AV_PIX_FMT_NV12 || frameToProcess->format == AV_PIX_FMT_YUV420P || frameToProcess->format == AV_PIX_FMT_YUVJ420P)) { if (m_currentNV12Frame) { av_frame_free(&m_currentNV12Frame); g_nv12Frees.fetch_add(1, std::memory_order_relaxed); } m_currentNV12Frame = av_frame_clone(frameToProcess); if (m_currentNV12Frame) g_nv12Clones.fetch_add(1, std::memory_order_relaxed); } width = m_currentImage.cols; height = m_currentImage.rows; pts = m_pts; result = m_currentImage; // Shallow copy under lock — refcount keeps buffer alive } av_frame_free(&frameToProcess); g_queueFrees.fetch_add(1, std::memory_order_relaxed); // Leak diagnostics — one heartbeat every 60 s across the whole process. // Each counter pair (allocs, frees) should stay balanced. A monotonic // rise in (allocs - frees) identifies the leaking pool. Bytes field // covers the ~12 MB/frame contiguous YUV420P buffers specifically — // watch for steady climb while the counters look balanced (refcount // leak in a held clone would show that shape). { using clk = std::chrono::steady_clock; static std::atomic s_nextLeakLogTick{0}; const long long tick = clk::now().time_since_epoch().count(); long long expected = s_nextLeakLogTick.load(std::memory_order_relaxed); if (tick >= expected) { const long long deadline = tick + std::chrono::duration_cast( std::chrono::seconds(60)).count(); // Claim the next window — first writer wins so only one thread logs. if (s_nextLeakLogTick.compare_exchange_strong( expected, deadline, std::memory_order_relaxed)) { const int64_t qA = g_queueClones.load(std::memory_order_relaxed); const int64_t qF = g_queueFrees.load(std::memory_order_relaxed); const int64_t nvA = g_nv12Clones.load(std::memory_order_relaxed); const int64_t nvF = g_nv12Frees.load(std::memory_order_relaxed); const int64_t nvE = g_nv12Escapes.load(std::memory_order_relaxed); const int64_t cuA = g_cudaHWClones.load(std::memory_order_relaxed); const int64_t cuF = g_cudaHWFrees.load(std::memory_order_relaxed); const int64_t cuE = g_cudaHWEscapes.load(std::memory_order_relaxed); const int64_t pR = g_avframePendingReturns.load(std::memory_order_relaxed); const int64_t cgA = g_contiguousAllocs.load(std::memory_order_relaxed); const int64_t cgF = g_contiguousFrees.load(std::memory_order_relaxed); const int64_t cgB = g_contiguousBytesInFlight.load(std::memory_order_relaxed); // escapeBalance = (nv12Esc + cuHWEsc) - pendingReturns. // Positive growing value = external callers hold clones they // never returned to the pendingFree drain → escape-path leak. const int64_t escBal = (nvE + cuE) - pR; ANS_DBG("MEDIA_Leak", "queue(C=%lld F=%lld net=%lld depth=%zu) " "nv12(C=%lld F=%lld net=%lld esc=%lld) " "cudaHW(C=%lld F=%lld net=%lld esc=%lld) " "pendingReturns=%lld escBal=%lld " "contig(A=%lld F=%lld net=%lld bytesMB=%.1f)", (long long)qA, (long long)qF, (long long)(qA - qF), g_frameQueue.size(), (long long)nvA, (long long)nvF, (long long)(nvA - nvF), (long long)nvE, (long long)cuA, (long long)cuF, (long long)(cuA - cuF), (long long)cuE, (long long)pR, (long long)escBal, (long long)cgA, (long long)cgF, (long long)(cgA - cgF), (double)cgB / (1024.0 * 1024.0)); } } } // Emit timing breakdown. Throttled so DebugView / stderr stay readable. { static std::atomic s_timingCount{0}; const uint64_t n = s_timingCount.fetch_add(1, std::memory_order_relaxed); if ((n % 30) == 0) { const auto t3 = clk::now(); auto ms = [](clk::time_point a, clk::time_point b) { return std::chrono::duration(b - a).count(); }; ANS_DBG("MEDIA_Timing", "getImage call#%llu pull=%.2fms convert=%.2fms publish=%.2fms total=%.2fms " "size=%dx%d seq=%llu (this=%p)", (unsigned long long)n, ms(t0, t1), ms(t1, t2), ms(t2, t3), ms(t0, t3), width, height, (unsigned long long)currentSeq, (void*)this); } } return result; } catch (const std::exception& e) { ANS_DBG("MEDIA_GetImage", "outer catch: %s", e.what()); return cv::Mat(); // Return an empty cv::Mat if an exception occurs } catch (...) { ANS_DBG("MEDIA_GetImage", "outer catch: unknown exception"); return cv::Mat(); // Return an empty cv::Mat if an exception occurs } } std::string CVideoPlayer::getJpegImage(int& width, int& height, int64_t& pts) { try { // Timing breakdown — gated by ANSCORE_DEBUGVIEW (zero overhead in production). using clk = std::chrono::steady_clock; const auto t0 = clk::now(); // Use same _mutex as getImage() to protect shared state consistently // recursive_mutex allows nested calls to avframeToJpegString → _mutex std::lock_guard lock(_mutex); const auto t1 = clk::now(); // While waiting for keyframe or during settle period after restart, // return the last good cached JPEG to avoid showing corrupted frames if (m_bWaitingForKeyframe || m_cleanFrameCount < SETTLE_FRAME_COUNT) { width = m_Width; height = m_Height; pts = m_pts; return m_lastJpegImage; // Last good JPEG (may be empty on first-ever start) } AVFrame* frameToProcess = g_frameQueue.getLatestFrame(); // Get a safe copy if (!frameToProcess) { return m_lastJpegImage; // Return the last valid JPEG image if no frame is available } const auto t2 = clk::now(); const int frameFmt = frameToProcess->format; const int frameW = frameToProcess->width; const int frameH = frameToProcess->height; try { if (frameToProcess->format == AV_PIX_FMT_NV12) { m_jpegImage = avframeToJpegString(frameToProcess); // Convert frame to JPEG from NV12 } else { m_jpegImage = avframeYUVJ420PToJpegString(frameToProcess); // Convert frame to JPEG from YUVJ420P } } catch (const std::exception& e) { ANS_DBG("MEDIA_GetJpeg", "encode threw: %s — returning cached JPEG", e.what()); av_frame_free(&frameToProcess); g_queueFrees.fetch_add(1, std::memory_order_relaxed); return m_lastJpegImage; } const auto t3 = clk::now(); av_frame_free(&frameToProcess); g_queueFrees.fetch_add(1, std::memory_order_relaxed); if (m_pts < INT64_MAX) { m_pts++; } else { m_pts = 0; // Reset to zero when max is reached } // Update the width, height, and pts width = m_Width; height = m_Height; pts = m_pts; if (!m_jpegImage.empty()) { m_lastJpegImage = std::move(m_jpegImage); // Move instead of copy } // Throttled timing breakdown for the JPEG hot path. { static std::atomic s_jpegTimingCount{0}; const uint64_t n = s_jpegTimingCount.fetch_add(1, std::memory_order_relaxed); if ((n % 30) == 0) { const auto t4 = clk::now(); auto ms = [](clk::time_point a, clk::time_point b) { return std::chrono::duration(b - a).count(); }; const char* fmtName = av_get_pix_fmt_name((AVPixelFormat)frameFmt); ANS_DBG("MEDIA_JpegTiming", "getJpegImage call#%llu lock=%.2fms pull=%.2fms encode=%.2fms publish=%.2fms " "total=%.2fms src_fmt=%s %dx%d jpeg_bytes=%zu (this=%p)", (unsigned long long)n, ms(t0, t1), ms(t1, t2), ms(t2, t3), ms(t3, t4), ms(t0, t4), fmtName ? fmtName : "?", frameW, frameH, m_lastJpegImage.size(), (void*)this); } } // Return the most recent valid JPEG image return m_lastJpegImage; } catch (const std::exception& e) { ANS_DBG("MEDIA_GetJpeg", "outer catch: %s", e.what()); } catch (...) { ANS_DBG("MEDIA_GetJpeg", "outer catch: unknown exception"); } // If any exception occurs, return the last valid JPEG image return m_lastJpegImage; }