diff --git a/MediaClient/media/video_player.cpp b/MediaClient/media/video_player.cpp index ebfd2cc..7aecd14 100644 --- a/MediaClient/media/video_player.cpp +++ b/MediaClient/media/video_player.cpp @@ -27,6 +27,8 @@ extern "C" #endif #include #include +#include +#include #include #include #include @@ -56,6 +58,112 @@ extern std::atomic g_contiguousAllocs; extern std::atomic g_contiguousFrees; extern std::atomic g_contiguousBytesInFlight; +// --------------------------------------------------------------------------- +// Freeze diagnostics — once-per-second pulse showing both producer +// (onVideoFrame) and consumer (getImage) accounting in a single line so a +// 1-2 second freeze can be localized in DebugView. Filter on "MEDIA_Pulse". +// +// Reading the line during a freeze: +// arrived high, pushed=0 ............ all frames dropped at producer +// drop(err=N) UDP packet loss → SW decoder set decode_error_flags +// drop(keyWait=N) waiting for first IDR after start/restart +// drop(rate=N) SetTargetFPS throttle skipped them +// arrived=0 .................. decoder silent (network/rx stall) +// arrived≈pushed, gets high, new=0 ... consumer not seeing the new push +// (mutex contention, seq stuck, ...) +// arrived≈pushed, new high, smooth ..... pipeline healthy +// --------------------------------------------------------------------------- +namespace { + std::atomic g_pulseArrived{0}; // onVideoFrame entries + std::atomic g_pulsePushed{0}; // pushFrame() calls (frame survived all drop checks) + std::atomic g_pulseDropDecodeErr{0}; // decode_error_flags != 0 (now also pushed; counted for visibility) + std::atomic g_pulseDropKeyframeWait{0}; // m_bWaitingForKeyframe and not IDR + std::atomic g_pulseDropRateLimit{0}; // SetTargetFPS interval skip + std::atomic g_pulseGetCalls{0}; // getImage entries + std::atomic g_pulseGetNew{0}; // returned a brand-new frame + std::atomic g_pulseGetCachedFast{0}; // currentSeq == m_lastFrameSeq (no new push) + std::atomic g_pulseGetCachedEmpty{0}; // queue empty + std::atomic g_pulseGetCachedNotPlaying{0};// !m_bPlaying + std::atomic g_pulseGetCachedSettle{0}; // waiting for keyframe / settle period + std::atomic g_pulseSeqJump{0}; // total frames the consumer flew past via getLatestFrame + // (sum of (currentSeq - m_lastFrameSeq - 1) across all reads) + std::atomic g_pulseNextEmitTick{0}; // monotonic deadline (steady_clock::now ticks) + + // Inter-arrival timing — exposes "21 frames bunched then 900ms silence" patterns + // that aggregate counters hide. Both stored as steady_clock raw ticks. + std::atomic g_pulseLastArrivalTick{0}; // tick of previous onVideoFrame entry (0 = none) + std::atomic g_pulseMaxGapTicks{0}; // max gap seen in current 1s window + + // Called from onVideoFrame on every arrival. Updates "max gap" if the gap + // since previous arrival exceeds the current window's running max. + inline void note_arrival_gap() { + using clk = std::chrono::steady_clock; + const long long now = clk::now().time_since_epoch().count(); + const long long prev = g_pulseLastArrivalTick.exchange(now, std::memory_order_relaxed); + if (prev == 0) return; // first arrival ever — no gap + const long long gap = now - prev; + // Atomic max: only update if larger + long long cur = g_pulseMaxGapTicks.load(std::memory_order_relaxed); + while (gap > cur && + !g_pulseMaxGapTicks.compare_exchange_weak(cur, gap, std::memory_order_relaxed)) { + // CAS retry until we win or someone else wrote a larger value + } + } + + // Emit if 1s has elapsed since the previous emission. CAS makes only one + // thread per second do the print; the rest just no-op. + inline void emit_media_pulse_if_due(size_t queueDepth) { + using clk = std::chrono::steady_clock; + const long long tick = clk::now().time_since_epoch().count(); + long long expected = g_pulseNextEmitTick.load(std::memory_order_relaxed); + if (tick < expected) return; + const long long deadline = tick + std::chrono::duration_cast( + std::chrono::milliseconds(1000)).count(); + if (!g_pulseNextEmitTick.compare_exchange_strong( + expected, deadline, std::memory_order_relaxed)) { + return; // another thread won the window + } + // Snapshot + reset (delta semantics — every line is "events in last ~1s") + const uint64_t arr = g_pulseArrived.exchange(0, std::memory_order_relaxed); + const uint64_t psh = g_pulsePushed.exchange(0, std::memory_order_relaxed); + const uint64_t dErr = g_pulseDropDecodeErr.exchange(0, std::memory_order_relaxed); + const uint64_t dKey = g_pulseDropKeyframeWait.exchange(0, std::memory_order_relaxed); + const uint64_t dRat = g_pulseDropRateLimit.exchange(0, std::memory_order_relaxed); + const uint64_t gtl = g_pulseGetCalls.exchange(0, std::memory_order_relaxed); + const uint64_t gNew = g_pulseGetNew.exchange(0, std::memory_order_relaxed); + const uint64_t gFst = g_pulseGetCachedFast.exchange(0, std::memory_order_relaxed); + const uint64_t gEmt = g_pulseGetCachedEmpty.exchange(0, std::memory_order_relaxed); + const uint64_t gNp = g_pulseGetCachedNotPlaying.exchange(0, std::memory_order_relaxed); + const uint64_t gStl = g_pulseGetCachedSettle.exchange(0, std::memory_order_relaxed); + const uint64_t evct = FrameQueue::evictedCounter().exchange(0, std::memory_order_relaxed); + const uint64_t skip = g_pulseSeqJump.exchange(0, std::memory_order_relaxed); + const long long maxGapTicks = g_pulseMaxGapTicks.exchange(0, std::memory_order_relaxed); + const double maxGapMs = std::chrono::duration( + clk::duration(maxGapTicks)).count(); + // Production: route through ANS_DBG so the pulse is gated by the + // ANSCORE_DEBUGVIEW runtime toggle (env var ANSCENTER_DBGVIEW=1 or + // sentinel file C:\ProgramData\ANSCENTER\ansvisdebugview.txt). + // Silent by default; one-line snapshot per second when enabled. + // + // maxGap is the largest interval between two consecutive arrivals in + // this second. Healthy ~25fps streams should sit at maxGap≈40-60ms. + // A spike to 1000+ ms is the smoking gun for a producer-side stall: + // RTP rx hung, decoder waiting on references, etc. + ANS_DBG("MEDIA_Pulse", + "[1s] producer arrived=%llu pushed=%llu drop(err=%llu keyWait=%llu rate=%llu) " + "maxGap=%.0fms evicted=%llu | " + "consumer gets=%llu new=%llu skipped=%llu cached(fast=%llu empty=%llu notPlay=%llu settle=%llu) " + "| queueDepth=%zu", + (unsigned long long)arr, (unsigned long long)psh, + (unsigned long long)dErr, (unsigned long long)dKey, (unsigned long long)dRat, + maxGapMs, (unsigned long long)evct, + (unsigned long long)gtl, (unsigned long long)gNew, (unsigned long long)skip, + (unsigned long long)gFst, (unsigned long long)gEmt, + (unsigned long long)gNp, (unsigned long long)gStl, + queueDepth); + } +} // anonymous namespace + // libyuv: SIMD-accelerated YUV↔RGB conversion with native strided-plane input. // Replaces the memcpy-into-staging + cv::cvtColor(COLOR_YUV2BGR_I420) chain // in avframeYUV420PToCvMat with a direct I420→RGB24 (== OpenCV BGR memory @@ -1920,14 +2028,13 @@ cv::Mat CVideoPlayer::avframeToCVMat(const AVFrame* pFrame) { return cv::Mat(); } - // One-shot diagnostic: print the pixel format the first time through so - // we can see which branch of the switch below is taken. Remove after use. + // One-shot diagnostic: log the pixel format the first time through so + // we can see which branch of the switch below is taken. Production: only + // emitted when ANSCORE_DEBUGVIEW is enabled. static bool s_loggedFmt = false; if (!s_loggedFmt) { s_loggedFmt = true; const char* name = av_get_pix_fmt_name((AVPixelFormat)pFrame->format); - fprintf(stderr, "[avframeToCVMat] first frame format=%d (%s) %dx%d\n", - pFrame->format, name ? name : "?", pFrame->width, pFrame->height); ANS_DBG("MEDIA_Convert", "avframeToCVMat FIRST-FRAME fmt=%d(%s) %dx%d HWDecoding=%d (this=%p)", pFrame->format, name ? name : "?", @@ -2465,6 +2572,12 @@ void CVideoPlayer::StartVideoDecoder() { m_lastFrameSeq = 0; m_bWaitingForKeyframe = true; // Skip frames until first keyframe m_cleanFrameCount = 0; // Reset settle counter + // Reset jitter buffer state so a restart starts measuring rate fresh + m_arrivalIdx = 0; + m_arrivalCount = 0; + m_observedFrameIntervalMs = 40.0; // re-seed at 25 fps + m_lastReleaseTimeSet = false; + m_jitterBufferRefilling = true; // new fill phase on (re)start if (m_pVideoDecoder) { m_pVideoDecoder->Start(); @@ -2595,6 +2708,17 @@ void CVideoPlayer::setTargetFPS(double intervalMs) m_targetIntervalMs = intervalMs; m_targetFPSInitialized = false; // reset timing on change } +void CVideoPlayer::setJitterBufferMs(int ms) +{ + std::lock_guard lock(_mutex); + if (ms < 0) ms = 0; + if (ms > 2000) ms = 2000; // sanity ceiling — beyond this is just lag + m_jitterBufferMs = ms; + // Reset release timing and re-enter fill phase so the new setting takes + // effect cleanly on the next call. + m_lastReleaseTimeSet = false; + m_jitterBufferRefilling = (ms > 0); +} double CVideoPlayer::getLastFrameAgeMs() { std::lock_guard lock(_mutex); @@ -2802,10 +2926,60 @@ void CVideoPlayer::onVideoFrame(AVFrame* frame) } if (m_bPlaying && m_videoPlayFlag) { - // Drop any frame with decode errors (corrupted reference frames, etc.) + // MEDIA_Pulse: count every frame the decoder hands us, before any drop checks. + g_pulseArrived.fetch_add(1, std::memory_order_relaxed); + note_arrival_gap(); // tracks max gap between consecutive arrivals in this 1s window + + // --- Arrival-rate measurement for the jitter buffer --- + // Record this arrival in the ring buffer, then update the observed + // inter-frame interval from the median delta of the last samples. + // Median is robust to bursts (TCP unstall delivers several frames in + // rapid succession; mean would be skewed, median picks up the typical + // in-burst spacing). Adapts within ~1 s when camera rate changes. + { + using clk = std::chrono::steady_clock; + const long long now = clk::now().time_since_epoch().count(); + m_arrivalTicks[m_arrivalIdx % kArrivalRingSize] = now; + m_arrivalIdx++; + if (m_arrivalCount < kArrivalRingSize) m_arrivalCount++; + // Recompute every 8 frames (cheap; ring of 32 ints). + if (m_arrivalCount >= 4 && (m_arrivalIdx % 8) == 0) { + const size_t n = m_arrivalCount; + std::array sortedCopy = m_arrivalTicks; + std::sort(sortedCopy.begin(), sortedCopy.begin() + n); + std::array deltas{}; + for (size_t i = 1; i < n; ++i) deltas[i - 1] = sortedCopy[i] - sortedCopy[i - 1]; + // Median delta + std::sort(deltas.begin(), deltas.begin() + (n - 1)); + const long long medianTicks = deltas[(n - 1) / 2]; + const double medianMs = std::chrono::duration( + clk::duration(medianTicks)).count(); + // Clamp to sane camera framerate range (5–120 fps) so a + // degenerate measurement can't poison the buffer logic. + if (medianMs >= 8.0 && medianMs <= 200.0) { + m_observedFrameIntervalMs = medianMs; + } + } + } + + // Pass frames with decode_error_flags through. The codec is configured + // with AV_CODEC_FLAG_OUTPUT_CORRUPT + AV_EF_IGNORE_ERR (video_decoder.cpp) + // so it has already applied concealment for missing references — the + // frame data is usable even if the flag is set. + // + // Why this matters: HW decoders (NVDEC/D3D11VA/DXVA2) silently conceal + // without setting the flag, but the SW decoder (default since commit + // e134ebd) sets the flag on every UDP packet loss. Dropping flagged + // frames meant every lost packet froze the display until the next IDR + // (~1-2s typical), reproducing the periodic freeze the user saw under + // UDP+SW. VLC plays through these frames — so do we now. if (frame->decode_error_flags != 0) { - fprintf(stderr, "[HWDecode] Dropping frame with decode errors (flags=0x%x)\n", frame->decode_error_flags); - return; + g_pulseDropDecodeErr.fetch_add(1, std::memory_order_relaxed); // counted, not dropped + ANS_DBG("MEDIA_DecodeErr", + "frame had decode_error_flags=0x%x — passing through with concealment " + "(pict_type=%d key=%d)", + frame->decode_error_flags, + (int)frame->pict_type, (int)frame->key_frame); } // After start/restart, skip corrupted frames until first keyframe (IDR) arrives. @@ -2815,8 +2989,11 @@ void CVideoPlayer::onVideoFrame(AVFrame* frame) if (frame->key_frame || frame->pict_type == AV_PICTURE_TYPE_I) { m_bWaitingForKeyframe = false; m_cleanFrameCount = 0; - fprintf(stderr, "[HWDecode] First keyframe received, settling for %d frames\n", SETTLE_FRAME_COUNT); + ANS_DBG("MEDIA_Lifecycle", + "first keyframe received, settling for %d frames", SETTLE_FRAME_COUNT); } else { + g_pulseDropKeyframeWait.fetch_add(1, std::memory_order_relaxed); + emit_media_pulse_if_due(g_frameQueue.size()); return; // Drop this frame — not yet safe to decode } } @@ -2838,6 +3015,8 @@ void CVideoPlayer::onVideoFrame(AVFrame* frame) } else { auto elapsed = std::chrono::duration(now - m_lastProcessedTime).count(); if (elapsed < m_targetIntervalMs) { + g_pulseDropRateLimit.fetch_add(1, std::memory_order_relaxed); + emit_media_pulse_if_due(g_frameQueue.size()); return; // Skip this frame — too soon } } @@ -2848,6 +3027,7 @@ void CVideoPlayer::onVideoFrame(AVFrame* frame) // Push frame to queue; during settle period getImage() will ignore the queue // and keep returning the last good cached image g_frameQueue.pushFrame(frame); // pushFrame() clones the frame internally + g_pulsePushed.fetch_add(1, std::memory_order_relaxed); // Capture CUDA HW frame for zero-copy inference. // We're inside decode()'s lock scope (decoder._mutex held) AND onVideoFrame @@ -2867,9 +3047,11 @@ void CVideoPlayer::onVideoFrame(AVFrame* frame) if (m_cleanFrameCount < SETTLE_FRAME_COUNT) { m_cleanFrameCount++; if (m_cleanFrameCount == SETTLE_FRAME_COUNT) { - fprintf(stderr, "[HWDecode] Settle complete, delivering new frames\n"); + ANS_DBG("MEDIA_Lifecycle", "settle complete, delivering new frames"); } } + + emit_media_pulse_if_due(g_frameQueue.size()); } } @@ -2901,13 +3083,20 @@ cv::Mat CVideoPlayer::getImage(int& width, int& height, int64_t& pts) { AVFrame* frameToProcess = nullptr; uint64_t currentSeq = 0; + // MEDIA_Pulse: count every getImage call for the per-second pulse, + // and try to emit on entry so the line still prints during a freeze + // where every call lands on a cached-return path. The once-per-second + // CAS inside makes the actual cost trivial. + g_pulseGetCalls.fetch_add(1, std::memory_order_relaxed); + emit_media_pulse_if_due(g_frameQueue.size()); + // Timing breakdown — gated by ANSCORE_DEBUGVIEW (zero overhead in production). // t0 = entry, t1 = after pulling frame from queue, t2 = after YUV->BGR, // t3 = after publish. Throttled to every 30 full-path calls (~1/sec @30fps). using clk = std::chrono::steady_clock; const auto t0 = clk::now(); - // --- Phase 1: short locked section — examine state, pull latest frame --- + // --- Phase 1: short locked section — examine state, pull frame --- { std::lock_guard lock(_mutex); @@ -2915,6 +3104,7 @@ cv::Mat CVideoPlayer::getImage(int& width, int& height, int64_t& pts) { width = m_currentImage.cols; height = m_currentImage.rows; pts = m_pts; + g_pulseGetCachedNotPlaying.fetch_add(1, std::memory_order_relaxed); return m_currentImage; // Shallow copy (reference counted) } @@ -2924,33 +3114,132 @@ cv::Mat CVideoPlayer::getImage(int& width, int& height, int64_t& pts) { width = m_currentImage.cols; height = m_currentImage.rows; pts = m_pts; + g_pulseGetCachedSettle.fetch_add(1, std::memory_order_relaxed); return m_currentImage; } - // Fast path: same frame as last call — skip clone + BGR conversion currentSeq = g_frameQueue.getSequence(); - if (currentSeq == m_lastFrameSeq && !m_currentImage.empty()) { - width = m_currentImage.cols; - height = m_currentImage.rows; - pts = m_pts; - return m_currentImage; - } - if (g_frameQueue.isEmpty()) { - width = m_currentImage.cols; - height = m_currentImage.rows; - pts = m_pts; - std::cerr << "No frame available in getImage()" << std::endl; - return cv::Mat(); - } + if (m_jitterBufferMs > 0 && m_observedFrameIntervalMs > 0.0) { + // ========================================================= + // Jitter-buffer mode: FIFO + rate-paced + adaptive depth + // + fill-phase refill + // ========================================================= + const double frameMs = m_observedFrameIntervalMs; + const size_t targetN = std::max(1, + static_cast(static_cast(m_jitterBufferMs) / frameMs)); + const size_t maxAllowed = 2 * targetN; + const size_t qSize = g_frameQueue.size(); - // getLatestFrame() clones the AVFrame — we own it from here - frameToProcess = g_frameQueue.getLatestFrame(); - if (!frameToProcess) { - width = m_currentImage.cols; - height = m_currentImage.rows; - pts = m_pts; - return cv::Mat(); + // (1) Adaptive drop-ahead: if the consumer is slower than the + // producer, the queue overshoots target. Pop & discard + // down to targetN so latency stays bounded. Each discard + // counts as a "skipped" frame in the pulse. + if (qSize > maxAllowed) { + while (g_frameQueue.size() > targetN) { + AVFrame* old = g_frameQueue.popFrame(); + if (old) { + av_frame_free(&old); + g_queueFrees.fetch_add(1, std::memory_order_relaxed); + g_pulseSeqJump.fetch_add(1, std::memory_order_relaxed); + } + } + } + + // (2) Fill phase: hold all releases until the queue reaches + // target depth. Triggered at startup and again whenever + // the queue drains to empty mid-stream. Without this, + // 1.0x pacing keeps the queue at whatever depth it + // happens to be (zero, after a drain) — the buffer + // never refills. + if (m_jitterBufferRefilling) { + if (g_frameQueue.size() >= targetN) { + m_jitterBufferRefilling = false; + } else { + width = m_currentImage.cols; + height = m_currentImage.rows; + pts = m_pts; + g_pulseGetCachedFast.fetch_add(1, std::memory_order_relaxed); + return m_currentImage; + } + } + + // (3) Rate-paced release at 1.0x of observed producer interval. + // Anything <1.0 leaks the buffer (consumer faster than + // producer); >1.0 keeps growing the queue until drop-ahead + // trips. 1.0 keeps the queue at its current depth in + // steady state. + if (m_lastReleaseTimeSet) { + const double sinceLastMs = std::chrono::duration( + t0 - m_lastReleaseTime).count(); + if (sinceLastMs < frameMs) { + width = m_currentImage.cols; + height = m_currentImage.rows; + pts = m_pts; + g_pulseGetCachedFast.fetch_add(1, std::memory_order_relaxed); + return m_currentImage; + } + } + + // (4) Empty queue (network gap drained the buffer): re-enter + // fill phase so the next producer pushes are held back to + // rebuild the buffer to target depth before the next + // release. Without this, a one-time gap permanently + // shrinks the buffer and we lose absorption capacity. + if (g_frameQueue.isEmpty()) { + m_jitterBufferRefilling = true; + width = m_currentImage.cols; + height = m_currentImage.rows; + pts = m_pts; + g_pulseGetCachedEmpty.fetch_add(1, std::memory_order_relaxed); + return cv::Mat(); + } + + // (5) FIFO pop — the frame is now ours; we must free it after + // conversion (popFrame returns the original, not a clone). + frameToProcess = g_frameQueue.popFrame(); + if (!frameToProcess) { + m_jitterBufferRefilling = true; + width = m_currentImage.cols; + height = m_currentImage.rows; + pts = m_pts; + g_pulseGetCachedEmpty.fetch_add(1, std::memory_order_relaxed); + return cv::Mat(); + } + m_lastReleaseTime = t0; + m_lastReleaseTimeSet = true; + } else { + // ========================================================= + // Legacy mode (m_jitterBufferMs == 0): getLatest semantics + // ========================================================= + // Fast path: same frame as last call — skip clone + BGR conversion + if (currentSeq == m_lastFrameSeq && !m_currentImage.empty()) { + width = m_currentImage.cols; + height = m_currentImage.rows; + pts = m_pts; + g_pulseGetCachedFast.fetch_add(1, std::memory_order_relaxed); + return m_currentImage; + } + + if (g_frameQueue.isEmpty()) { + width = m_currentImage.cols; + height = m_currentImage.rows; + pts = m_pts; + g_pulseGetCachedEmpty.fetch_add(1, std::memory_order_relaxed); + // Common case during network jitter — gated to avoid stderr spam. + ANS_DBG("MEDIA_GetImage", "queue empty, returning cached"); + return cv::Mat(); + } + + // getLatestFrame() clones the AVFrame — we own it from here + frameToProcess = g_frameQueue.getLatestFrame(); + if (!frameToProcess) { + width = m_currentImage.cols; + height = m_currentImage.rows; + pts = m_pts; + g_pulseGetCachedEmpty.fetch_add(1, std::memory_order_relaxed); + return cv::Mat(); + } } } // --- _mutex released here --- @@ -2964,7 +3253,8 @@ cv::Mat CVideoPlayer::getImage(int& width, int& height, int64_t& pts) { converted = avframeToCVMat(frameToProcess); } catch (const std::exception& e) { - std::cerr << "Exception while converting AVFrame to cv::Mat: " << e.what() << std::endl; + ANS_DBG("MEDIA_GetImage", + "avframeToCVMat threw: %s — returning empty Mat", e.what()); } const auto t2 = clk::now(); @@ -2974,9 +3264,20 @@ cv::Mat CVideoPlayer::getImage(int& width, int& height, int64_t& pts) { std::lock_guard lock(_mutex); if (!converted.empty()) { + // Skip-tracking: only meaningful in legacy mode where the + // consumer reads the back of the queue. In jitter-buffer mode + // we already counted drop-ahead discards; sequence numbers + // don't track displayed-vs-skipped because pop is FIFO. + if (m_jitterBufferMs == 0) { + if (m_lastFrameSeq != 0 && currentSeq > m_lastFrameSeq + 1) { + g_pulseSeqJump.fetch_add(currentSeq - m_lastFrameSeq - 1, + std::memory_order_relaxed); + } + m_lastFrameSeq = currentSeq; + } m_currentImage = converted; m_pts++; - m_lastFrameSeq = currentSeq; + g_pulseGetNew.fetch_add(1, std::memory_order_relaxed); } // Preserve raw YUV/NV12 frame for GPU fast-path inference @@ -3076,11 +3377,11 @@ cv::Mat CVideoPlayer::getImage(int& width, int& height, int64_t& pts) { return result; } catch (const std::exception& e) { - std::cerr << "Unexpected exception in getImage(): " << e.what() << std::endl; + ANS_DBG("MEDIA_GetImage", "outer catch: %s", e.what()); return cv::Mat(); // Return an empty cv::Mat if an exception occurs } catch (...) { - std::cerr << "Unknown exception in getImage()" << std::endl; + ANS_DBG("MEDIA_GetImage", "outer catch: unknown exception"); return cv::Mat(); // Return an empty cv::Mat if an exception occurs } @@ -3124,7 +3425,7 @@ std::string CVideoPlayer::getJpegImage(int& width, int& height, int64_t& pts) { } } catch (const std::exception& e) { - std::cerr << "Exception while converting AVFrame to JPEG string: " << e.what() << std::endl; + ANS_DBG("MEDIA_GetJpeg", "encode threw: %s — returning cached JPEG", e.what()); av_frame_free(&frameToProcess); g_queueFrees.fetch_add(1, std::memory_order_relaxed); return m_lastJpegImage; @@ -3176,10 +3477,10 @@ std::string CVideoPlayer::getJpegImage(int& width, int& height, int64_t& pts) { return m_lastJpegImage; } catch (const std::exception& e) { - std::cerr << "Unexpected exception in getJpegImage(): " << e.what() << std::endl; + ANS_DBG("MEDIA_GetJpeg", "outer catch: %s", e.what()); } catch (...) { - std::cerr << "Unknown exception in getJpegImage()" << std::endl; + ANS_DBG("MEDIA_GetJpeg", "outer catch: unknown exception"); } // If any exception occurs, return the last valid JPEG image diff --git a/MediaClient/media/video_player.h b/MediaClient/media/video_player.h index 468ea91..67c820f 100644 --- a/MediaClient/media/video_player.h +++ b/MediaClient/media/video_player.h @@ -11,6 +11,7 @@ #include "media_util.h" #include #include +#include #include #include #include @@ -51,6 +52,16 @@ private: uint64_t m_frameSeq = 0; // Sequence counter — increments on each new frame push public: + // Pulse counter for evicted-unread frames (instrumentation, not behavior). + // Incremented every time a push has to drop the oldest frame because the + // queue hit maxFrames AND the oldest never got read. With getLatestFrame() + // the consumer reads the back, so anything pushed before the consumer's + // last read sits in the middle/front and gets silently evicted here. + static std::atomic& evictedCounter() { + static std::atomic g{0}; + return g; + } + // Push a new frame to the queue (removes oldest if full) void pushFrame(AVFrame* newFrame) { if (!newFrame) return; @@ -74,6 +85,7 @@ public: frameQueue.pop(); av_frame_free(&oldFrame); g_queueFrees.fetch_add(1, std::memory_order_relaxed); + evictedCounter().fetch_add(1, std::memory_order_relaxed); } } @@ -177,6 +189,16 @@ public: virtual void setImageQuality(int mode) { m_nImageQuality = mode; } void setTargetFPS(double intervalMs); // Set minimum interval between processed frames in ms (0 = no limit, 100 = ~10 FPS) double getLastFrameAgeMs(); // Milliseconds since last frame arrived from decoder (0 if no frame yet) + // Jitter buffer: hold decoded frames for ~ms before delivering to consumer. + // Smooths network/decoder jitter at the cost of added end-to-end latency. + // 0 disables (today's getLatest semantics). Default 200 ms. + // Adaptive: if the consumer is slower than the producer, drop-ahead kicks in + // and bypasses the buffer (latency stays ~0 frames). + // Note: applies only to the BGR cv::Mat returned by getImage(); the NV12/CUDA + // fast-path frames (m_currentNV12Frame / m_currentCudaHWFrame) remain + // updated at push time so inference always sees the latest. + void setJitterBufferMs(int ms); + int getJitterBufferMs() const { return m_jitterBufferMs; } virtual void setRtpMulticast(BOOL flag) {} virtual void setRtpOverUdp(BOOL flag) {} @@ -317,6 +339,31 @@ protected: std::chrono::steady_clock::time_point m_lastDecoderFrameTime; bool m_lastDecoderFrameTimeSet = false; + // --- Jitter buffer (BGR delivery only) --- + // Default 200 ms — absorbs typical network/decoder hiccups. + int m_jitterBufferMs = 200; + // Observed inter-arrival interval, ms. Seeds at 40 ms (=25 fps) and is + // refined to a rolling median of the last m_arrivalCount samples. + double m_observedFrameIntervalMs = 40.0; + // Ring buffer of recent arrival ticks (steady_clock raw ticks). Used to + // compute the rolling median frame interval. Sized at 32 — covers >1 s at + // 30 fps so a single GOP-boundary jitter doesn't dominate. + static constexpr size_t kArrivalRingSize = 32; + std::array m_arrivalTicks{}; + size_t m_arrivalIdx = 0; + size_t m_arrivalCount = 0; + // Wall-clock of the last frame released to the consumer. Used for rate-paced + // delivery — if the consumer polls faster than the producer rate, between + // releases we return the cached frame so the buffer doesn't drain instantly. + std::chrono::steady_clock::time_point m_lastReleaseTime; + bool m_lastReleaseTimeSet = false; + // Fill-phase: when true, hold releases until queue depth reaches target. + // Set true at startup and again whenever the queue drains to empty mid-stream. + // This is what makes the buffer actually buffer — without it, pacing alone + // can't grow the queue from empty (producer and consumer would just match + // rates with depth=0 forever). + bool m_jitterBufferRefilling = true; + BOOL m_bPlaying; BOOL m_bPaused; diff --git a/modules/ANSCV/ANSRTSP.cpp b/modules/ANSCV/ANSRTSP.cpp index b89b076..b07bce2 100644 --- a/modules/ANSCV/ANSRTSP.cpp +++ b/modules/ANSCV/ANSRTSP.cpp @@ -28,6 +28,51 @@ extern "C" // Note: per-instance thread safety is handled by ANSRTSPClient::_mutex // Mat registry thread safety is handled by anscv_mat_replace's internal registry_mutex +// --------------------------------------------------------------------------- +// Freeze diagnostics — once-per-second pulse for ANSRTSPClient::GetImage so +// we can see *which* return path fired during a 1-2s freeze. Filter on +// "RTSP_Pulse" in DebugView. Pairs with the producer/consumer pulse in +// video_player.cpp ("MEDIA_Pulse"): together they let you localize a stall +// to producer (decoder) vs internal player vs ANSRTSP wrapper. +// --------------------------------------------------------------------------- +namespace { + std::atomic g_rtspPulseGetCalls{0}; // entries to ANSRTSPClient::GetImage + std::atomic g_rtspPulseNotPlaying{0}; // !_isPlaying + std::atomic g_rtspPulseEarlyStale{0}; // ageMs >= 5000 → cached early + std::atomic g_rtspPulseInnerEmpty{0}; // _playerClient->getImage returned empty + std::atomic g_rtspPulseDupPts{0}; // currentPts == _pts → cached + std::atomic g_rtspPulseNewFrame{0}; // brand-new frame returned + std::atomic g_rtspPulseNextEmitTick{0}; + + inline void emit_rtsp_pulse_if_due(double ageMsLastFrame) { + using clk = std::chrono::steady_clock; + const long long tick = clk::now().time_since_epoch().count(); + long long expected = g_rtspPulseNextEmitTick.load(std::memory_order_relaxed); + if (tick < expected) return; + const long long deadline = tick + std::chrono::duration_cast( + std::chrono::milliseconds(1000)).count(); + if (!g_rtspPulseNextEmitTick.compare_exchange_strong( + expected, deadline, std::memory_order_relaxed)) { + return; + } + const uint64_t calls = g_rtspPulseGetCalls.exchange(0, std::memory_order_relaxed); + const uint64_t np = g_rtspPulseNotPlaying.exchange(0, std::memory_order_relaxed); + const uint64_t es = g_rtspPulseEarlyStale.exchange(0, std::memory_order_relaxed); + const uint64_t ie = g_rtspPulseInnerEmpty.exchange(0, std::memory_order_relaxed); + const uint64_t dp = g_rtspPulseDupPts.exchange(0, std::memory_order_relaxed); + const uint64_t nf = g_rtspPulseNewFrame.exchange(0, std::memory_order_relaxed); + // Production: route through ANS_DBG so the pulse is gated by the + // ANSCORE_DEBUGVIEW runtime toggle. Silent by default. + ANS_DBG("RTSP_Pulse", + "[1s] GetImage calls=%llu new=%llu cached(notPlay=%llu earlyStale=%llu " + "innerEmpty=%llu dupPts=%llu) lastFrameAge=%.0fms", + (unsigned long long)calls, (unsigned long long)nf, + (unsigned long long)np, (unsigned long long)es, + (unsigned long long)ie, (unsigned long long)dp, + ageMsLastFrame); + } +} // anonymous namespace + // Debug logging. Define ANSCORE_GPU_DEBUG=1 to enable verbose per-frame logging. #ifndef RTSP_DBG #if defined(ANSCORE_GPU_DEBUG) && ANSCORE_GPU_DEBUG @@ -450,11 +495,17 @@ namespace ANSCENTER { cv::Mat ANSRTSPClient::GetImage(int& width, int& height, int64_t& pts) { std::lock_guard lock(_mutex); + // RTSP_Pulse: count entry, attempt once-per-second emit on every call so + // the pulse keeps printing even when every path returns cached. + g_rtspPulseGetCalls.fetch_add(1, std::memory_order_relaxed); + emit_rtsp_pulse_if_due(_playerClient ? _playerClient->getLastFrameAgeMs() : -1.0); + // Return last known frame if not playing if (!_isPlaying) { width = _imageWidth; height = _imageHeight; pts = _pts; + g_rtspPulseNotPlaying.fetch_add(1, std::memory_order_relaxed); return _pLastFrame; // Shallow copy (fast) } @@ -473,6 +524,7 @@ namespace ANSCENTER { width = _imageWidth; height = _imageHeight; pts = _pts; + g_rtspPulseEarlyStale.fetch_add(1, std::memory_order_relaxed); return _pLastFrame; } } @@ -488,36 +540,40 @@ namespace ANSCENTER { width = _imageWidth; height = _imageHeight; pts = _pts; + g_rtspPulseInnerEmpty.fetch_add(1, std::memory_order_relaxed); return _pLastFrame; } - // Use PTS to detect duplicate frames (much faster than pixel comparison) - // The sequence-based check in getImage() already skips conversion for same frames, - // so if PTS hasn't changed, it's the same frame — no need for expensive memcmp - if (currentPts == _pts && !_pLastFrame.empty()) { + // Duplicate-frame detection. We track the inner player's currentPts + // (in _lastInnerPts) separately from our own _pts. This is important: + // _pts gets synthetically advanced on every duplicate so LabVIEW's + // staleness check sees a changing timestamp. If we compared currentPts + // against _pts, the synthetic advance would make the comparison fail + // on the next call and route the same cached frame through the + // "new frame" path — pointless resize + storage on identical pixels. + if (currentPts == _lastInnerPts && !_pLastFrame.empty()) { width = _imageWidth; height = _imageHeight; - // Return timestamp based on decoder frame age so LabVIEW can distinguish - // "rate-limited duplicate" from "camera truly stale". - // If decoder is still receiving frames (age < 5s), advance PTS so LabVIEW - // sees a changing timestamp and doesn't trigger false reconnect. - // If decoder is stale (age > 5s), return same PTS so LabVIEW detects it. double ageMs = _playerClient->getLastFrameAgeMs(); if (ageMs > 0.0 && ageMs < 5000.0) { - // Camera alive but rate-limited — advance PTS to prevent false stale detection - _pts++; + _pts++; // synthetic advance — LabVIEW staleness signal } else if (ageMs >= 5000.0) { - // Camera stale — keep same PTS so LabVIEW triggers reconnect ANS_DBG("RTSP_GetImage", "STALE PTS: ageMs=%.1f pts=%lld url=%s — not advancing PTS", ageMs, (long long)_pts, _url.c_str()); } pts = _pts; + g_rtspPulseDupPts.fetch_add(1, std::memory_order_relaxed); return _pLastFrame; } - // Update PTS first - _pts = currentPts; - pts = currentPts; + // Inner advanced — real new frame. Sync our duplicate-detection cursor + // and bump _pts monotonically (never assign to currentPts, which would + // make _pts go backwards if it had been advanced past currentPts on + // previous duplicates). + _lastInnerPts = currentPts; + _pts++; + pts = _pts; + g_rtspPulseNewFrame.fetch_add(1, std::memory_order_relaxed); // Handle non-rotated case if (_imageRotateDeg == 0) { @@ -992,6 +1048,24 @@ namespace ANSCENTER { std::lock_guard lock(_mutex); _playerClient->setTargetFPS(intervalMs); // 0=no limit, 100=~10FPS, 200=~5FPS } + void ANSRTSPClient::SetRtpOverUdp(bool useUdp) { + std::lock_guard lock(_mutex); + // Forwards to CRtspClient::set_rtp_over_udp via CRtspPlayer. The TRUE/FALSE + // mapping is inverted inside (rtp_tcp = useUdp ? 0 : 1) — see rtsp_cln.cpp:3578. + _playerClient->setRtpOverUdp(useUdp ? TRUE : FALSE); + ANS_DBG("RTSP_Lifecycle", + "SetRtpOverUdp: useUdp=%d url=%s — must be called before Start()", + (int)useUdp, _url.c_str()); + } + void ANSRTSPClient::SetJitterBufferMs(int ms) { + std::lock_guard lock(_mutex); + _playerClient->setJitterBufferMs(ms); + ANS_DBG("RTSP_Lifecycle", + "SetJitterBufferMs: ms=%d url=%s", ms, _url.c_str()); + } + int ANSRTSPClient::GetJitterBufferMs() const { + return _playerClient->getJitterBufferMs(); + } void ANSRTSPClient::SetNV12FastPath(bool enable) { std::lock_guard lock(_mutex); _useNV12FastPath = enable; @@ -1585,6 +1659,24 @@ extern "C" __declspec(dllexport) void SetRTSPTargetFPS(ANSCENTER::ANSRTSPClient* (*Handle)->SetTargetFPS(intervalMs); // 0=no limit, 100=~10FPS, 200=~5FPS } catch (...) { } } +extern "C" __declspec(dllexport) void SetRTSPRtpOverUdp(ANSCENTER::ANSRTSPClient** Handle, int useUdp) { + if (Handle == nullptr || *Handle == nullptr) return; + try { + (*Handle)->SetRtpOverUdp(useUdp != 0); + } catch (...) { } +} +extern "C" __declspec(dllexport) void SetRTSPJitterBuffer(ANSCENTER::ANSRTSPClient** Handle, int jitterMs) { + if (Handle == nullptr || *Handle == nullptr) return; + try { + (*Handle)->SetJitterBufferMs(jitterMs); + } catch (...) { } +} +extern "C" __declspec(dllexport) int GetRTSPJitterBuffer(ANSCENTER::ANSRTSPClient** Handle) { + if (Handle == nullptr || *Handle == nullptr) return -1; + try { + return (*Handle)->GetJitterBufferMs(); + } catch (...) { return -1; } +} extern "C" __declspec(dllexport) void SetRTSPNV12FastPath(ANSCENTER::ANSRTSPClient** Handle, int enable) { if (Handle == nullptr || *Handle == nullptr) return; try { diff --git a/modules/ANSCV/ANSRTSP.h b/modules/ANSCV/ANSRTSP.h index abe09bb..1b20606 100644 --- a/modules/ANSCV/ANSRTSP.h +++ b/modules/ANSCV/ANSRTSP.h @@ -37,6 +37,11 @@ namespace ANSCENTER std::string _lastJpegImage; int _imageWidth,_imageHeight; int64_t _pts; + // Last currentPts seen from the inner getImage(). Used for duplicate + // detection separately from _pts (which gets synthetically incremented + // for LabVIEW's staleness check). Keeping these decoupled fixes the + // "every other dup goes through new-frame path" inefficiency. + int64_t _lastInnerPts = -1; bool _isPlaying; bool _useNV12FastPath = false; // false = original stable CPU path, true = NV12 GPU fast path std::recursive_mutex _mutex; @@ -104,6 +109,21 @@ namespace ANSCENTER void SetDisplayResolution(int width, int height); // Set display output size; 0,0 = original (no resize) void SetImageQuality(int mode); // 0=fast (AI), 1=quality (display) void SetTargetFPS(double intervalMs); // Set min interval between processed frames in ms (0 = no limit, 100 = ~10 FPS, 200 = ~5 FPS) + // Jitter buffer for the BGR delivery path. Holds frames ms milliseconds + // before releasing to absorb network/decoder hiccups. 0 = today's + // getLatest semantics. Default in CVideoPlayer is 200 ms. + // Adaptive: if the consumer is slower than the producer, drop-ahead + // keeps latency bounded — slow callers (LabVIEW polling at 1-5 Hz) + // effectively bypass the buffer. + // Inference path (NV12/CUDA fast-path) is unaffected — it always + // receives the latest decoded frame regardless of this setting. + void SetJitterBufferMs(int ms); + int GetJitterBufferMs() const; + // Transport for RTP packets. true = RTP/UDP (lower latency, no TCP head-of-line + // blocking — recommended on LAN). false = RTP/RTSP/TCP interleaved (default, more + // firewall-friendly but bursts on packet loss / large I-frames). + // MUST be called BEFORE Start() / StartRTSP — the SETUP request is sent inside play(). + void SetRtpOverUdp(bool useUdp); void SetNV12FastPath(bool enable); // true = NV12 GPU fast path (zero-copy inference), false = original CPU path (stable) bool IsNV12FastPath() const { return _useNV12FastPath; } double GetLastFrameAgeMs(); // Milliseconds since last frame from decoder (detects truly stale cameras, unaffected by SetTargetFPS) @@ -145,6 +165,9 @@ extern "C" __declspec(dllexport) int GetRTSPHWDecodingGpuIndex(ANSCENTER::ANSRT extern "C" __declspec(dllexport) void SetRTSPImageQuality(ANSCENTER::ANSRTSPClient** Handle, int mode); extern "C" __declspec(dllexport) void SetRTSPDisplayResolution(ANSCENTER::ANSRTSPClient** Handle, int width, int height); extern "C" __declspec(dllexport) void SetRTSPTargetFPS(ANSCENTER::ANSRTSPClient** Handle, double intervalMs); +extern "C" __declspec(dllexport) void SetRTSPRtpOverUdp(ANSCENTER::ANSRTSPClient** Handle, int useUdp); +extern "C" __declspec(dllexport) void SetRTSPJitterBuffer(ANSCENTER::ANSRTSPClient** Handle, int jitterMs); +extern "C" __declspec(dllexport) int GetRTSPJitterBuffer(ANSCENTER::ANSRTSPClient** Handle); extern "C" __declspec(dllexport) void SetRTSPNV12FastPath(ANSCENTER::ANSRTSPClient** Handle, int enable); extern "C" __declspec(dllexport) double GetRTSPLastFrameAgeMs(ANSCENTER::ANSRTSPClient** Handle); #endif \ No newline at end of file diff --git a/tests/ANSCV-UnitTest/ANSCV-UnitTest.cpp b/tests/ANSCV-UnitTest/ANSCV-UnitTest.cpp index 2b3ad03..eccdbbc 100644 --- a/tests/ANSCV-UnitTest/ANSCV-UnitTest.cpp +++ b/tests/ANSCV-UnitTest/ANSCV-UnitTest.cpp @@ -833,14 +833,36 @@ int RSTPTestCVClient() { // std::string rtspUrl = "rtsp://admin:admin123@192.168.1.23:554";//"rtsp://alex:admin123@192.168.1.11:554";// //std::string rtspUrl = "rtsp://anscamuser:SonTay2020@anssontay.tplinkdns.com:554/Streaming/Channels/401";// "rtsp://admin:JA!uV_mx@42.1.117.99:8093/1/1"; - std::string rtspUrl = "rtsp://admin:AnsHome16%&@192.168.1.104:554";//"rtsp://alex:admin123@192.168.1.11:554";//112 + std::string rtspUrl = "rtsp://admin:Grb%401212@14.224.237.228:15541/Streaming/channels/101";//"rtsp://admin:AnsHome16%&@192.168.1.104:554";//"rtsp://alex:admin123@192.168.1.11:554";//112 CreateANSRTSPHandle(&rtspClient, "", "", "", rtspUrl.c_str()); //void SetRTSPImageQuality(ANSCENTER::ANSRTSPClient** Handle, int mode) SetRTSPImageQuality(&rtspClient,0); //SetBBoxRTSP(&rtspClient, 0, 0, 1920, 1080); //SetCropFlagRTSP(&rtspClient, 1); + // Hardware decode (auto-pick: CUDA if available, otherwise D3D11VA/DXVA2). + // Without this the player uses single-threaded SW decode (default since + // commit e134ebd "Use software decoder by default"), which on 4K HEVC + // can take 80-120ms/frame — too slow for 25/30fps. The decoder falls + // behind, the RTP buffer fills, and frames arrive in bursts (the + // "smooth → 1-2s freeze → catch-up" pattern). VLC uses HW decode by + // default, which is why it plays the same camera smoothly. + // MUST be called before StartRTSP — decoder is constructed inside play(). + SetRTSPHWDecoding(&rtspClient, /*HW_DECODING_AUTO=*/0, /*preferredGpu=*/-1); + + // Transport: default is RTP-over-RTSP/TCP (interleaved). On a LAN, UDP + // avoids TCP head-of-line stalls. Over WAN/NAT it typically does NOT + // work (port forwarding usually only covers TCP/554), so leave commented + // out for remote cameras. MUST be called before StartRTSP. + //SetRTSPRtpOverUdp(&rtspClient, 1); + StartRTSP(&rtspClient); + // Let ANSRTSP downscale internally (INTER_LINEAR, ~5x faster than the + // INTER_LANCZOS4 the test was doing). Without this, a 4K source plus + // LANCZOS4 resize stretches each loop iteration to ~150ms and the + // 20-slot decoder queue overruns — frames look "skipped" because + // intermediate ones get evicted before getLatestFrame() sees them. + SetRTSPDisplayResolution(&rtspClient, 1920, 1080); //SetRTSPImageRotation(&rtspClient,0); cv::namedWindow("Image", cv::WINDOW_NORMAL); // Create a resizable window. cv::resizeWindow("Image", 1920, 1080); // Set initial size of the window (landscape). @@ -854,12 +876,13 @@ int RSTPTestCVClient() { //if ((index == 1800) || (index == 2200) || (index == 2500)) { StartRTSP(&rtspClient); } //if ((index == 2000) || (index == 2300) || (index == 2700)) { StartRTSP(&rtspClient); } //if (index > 20000) break; - auto start = std::chrono::system_clock::now(); + auto loopStart = std::chrono::steady_clock::now(); + auto getStart = loopStart; cv::Mat* image = nullptr; // ✅ Use a pointer to hold the allocated image GetRTSPCVImage(&rtspClient, width, height, pts,&image); - auto end1 = std::chrono::system_clock::now(); - auto elapsed1 = std::chrono::duration_cast(end1 - start); - if (elapsed1.count() > 0)std::cout << "Time to get image:" << elapsed1.count() << "ms" << std::endl; + auto getEnd = std::chrono::steady_clock::now(); + auto getMs = std::chrono::duration_cast(getEnd - getStart).count(); + if (getMs > 0) std::cout << "Time to get image:" << getMs << "ms" << std::endl; // ✅ Check if the image is valid BEFORE accessing it if (!image || image->empty()) { @@ -868,23 +891,42 @@ int RSTPTestCVClient() { continue; // Skip processing if the image is empty } - // High-quality downscale for display: INTER_LANCZOS4 preserves sharpness and edges - cv::Mat displayImage; - if (image->cols > 1920) { - double scale = 1920.0 / image->cols; - cv::resize(*image, displayImage, cv::Size(), scale, scale, cv::INTER_LANCZOS4); - } - else { - displayImage = *image; - } - cv::imshow("Image", displayImage); + // ANSRTSP already returned a 1920x1080 frame — no client-side resize. + // Time the imshow specifically — on Windows it can occasionally block + // for 100s of ms (window message pump, GPU sync, monitor refresh + // alignment), and that would visibly look like a freeze even though + // the producer pulse stays healthy. + auto showStart = std::chrono::steady_clock::now(); + cv::imshow("Image", *image); + auto showEnd = std::chrono::steady_clock::now(); + auto showMs = std::chrono::duration_cast(showEnd - showStart).count(); ANSCV_ReleaseImage_S(&image); - //std::cout << "Index="<(waitEnd - waitStart).count(); + + // Print only when something is actually slow — keeps the log readable. + // Threshold: 30ms = noticeable stutter, 100ms = visible freeze contributor. + if (showMs >= 30 || waitMs >= 30) { + std::cout << "[SLOW DISPLAY] imshow=" << showMs << "ms waitKey=" << waitMs + << "ms (frame=" << index << ")" << std::endl; + } + + if (quit) { std::cout << "Break" << std::endl; break; } + + // Whole-iteration timing — should now be close to 1/cameraFps. + auto loopMs = std::chrono::duration_cast( + std::chrono::steady_clock::now() - loopStart).count(); + if (loopMs >= 50) std::cout << "Loop iteration: " << loopMs << "ms" << std::endl; } cv::destroyAllWindows(); // Destroy all OpenCV windows StopRTSP(&rtspClient);