Add jitter buffer

This commit is contained in:
2026-05-07 22:36:27 +10:00
parent 2e0fee5e3d
commit 75693cadd9
5 changed files with 575 additions and 70 deletions
+339 -38
View File
@@ -27,6 +27,8 @@ extern "C"
#endif
#include <string>
#include <vector>
#include <array>
#include <algorithm>
#include <chrono>
#include <atomic>
#include <libswscale/swscale.h>
@@ -56,6 +58,112 @@ extern std::atomic<int64_t> g_contiguousAllocs;
extern std::atomic<int64_t> g_contiguousFrees;
extern std::atomic<int64_t> g_contiguousBytesInFlight;
// ---------------------------------------------------------------------------
// Freeze diagnostics — once-per-second pulse showing both producer
// (onVideoFrame) and consumer (getImage) accounting in a single line so a
// 1-2 second freeze can be localized in DebugView. Filter on "MEDIA_Pulse".
//
// Reading the line during a freeze:
// arrived high, pushed=0 ............ all frames dropped at producer
// drop(err=N) UDP packet loss → SW decoder set decode_error_flags
// drop(keyWait=N) waiting for first IDR after start/restart
// drop(rate=N) SetTargetFPS throttle skipped them
// arrived=0 .................. decoder silent (network/rx stall)
// arrived≈pushed, gets high, new=0 ... consumer not seeing the new push
// (mutex contention, seq stuck, ...)
// arrived≈pushed, new high, smooth ..... pipeline healthy
// ---------------------------------------------------------------------------
namespace {
std::atomic<uint64_t> g_pulseArrived{0}; // onVideoFrame entries
std::atomic<uint64_t> g_pulsePushed{0}; // pushFrame() calls (frame survived all drop checks)
std::atomic<uint64_t> g_pulseDropDecodeErr{0}; // decode_error_flags != 0 (now also pushed; counted for visibility)
std::atomic<uint64_t> g_pulseDropKeyframeWait{0}; // m_bWaitingForKeyframe and not IDR
std::atomic<uint64_t> g_pulseDropRateLimit{0}; // SetTargetFPS interval skip
std::atomic<uint64_t> g_pulseGetCalls{0}; // getImage entries
std::atomic<uint64_t> g_pulseGetNew{0}; // returned a brand-new frame
std::atomic<uint64_t> g_pulseGetCachedFast{0}; // currentSeq == m_lastFrameSeq (no new push)
std::atomic<uint64_t> g_pulseGetCachedEmpty{0}; // queue empty
std::atomic<uint64_t> g_pulseGetCachedNotPlaying{0};// !m_bPlaying
std::atomic<uint64_t> g_pulseGetCachedSettle{0}; // waiting for keyframe / settle period
std::atomic<uint64_t> g_pulseSeqJump{0}; // total frames the consumer flew past via getLatestFrame
// (sum of (currentSeq - m_lastFrameSeq - 1) across all reads)
std::atomic<long long> g_pulseNextEmitTick{0}; // monotonic deadline (steady_clock::now ticks)
// Inter-arrival timing — exposes "21 frames bunched then 900ms silence" patterns
// that aggregate counters hide. Both stored as steady_clock raw ticks.
std::atomic<long long> g_pulseLastArrivalTick{0}; // tick of previous onVideoFrame entry (0 = none)
std::atomic<long long> g_pulseMaxGapTicks{0}; // max gap seen in current 1s window
// Called from onVideoFrame on every arrival. Updates "max gap" if the gap
// since previous arrival exceeds the current window's running max.
inline void note_arrival_gap() {
using clk = std::chrono::steady_clock;
const long long now = clk::now().time_since_epoch().count();
const long long prev = g_pulseLastArrivalTick.exchange(now, std::memory_order_relaxed);
if (prev == 0) return; // first arrival ever — no gap
const long long gap = now - prev;
// Atomic max: only update if larger
long long cur = g_pulseMaxGapTicks.load(std::memory_order_relaxed);
while (gap > cur &&
!g_pulseMaxGapTicks.compare_exchange_weak(cur, gap, std::memory_order_relaxed)) {
// CAS retry until we win or someone else wrote a larger value
}
}
// Emit if 1s has elapsed since the previous emission. CAS makes only one
// thread per second do the print; the rest just no-op.
inline void emit_media_pulse_if_due(size_t queueDepth) {
using clk = std::chrono::steady_clock;
const long long tick = clk::now().time_since_epoch().count();
long long expected = g_pulseNextEmitTick.load(std::memory_order_relaxed);
if (tick < expected) return;
const long long deadline = tick + std::chrono::duration_cast<clk::duration>(
std::chrono::milliseconds(1000)).count();
if (!g_pulseNextEmitTick.compare_exchange_strong(
expected, deadline, std::memory_order_relaxed)) {
return; // another thread won the window
}
// Snapshot + reset (delta semantics — every line is "events in last ~1s")
const uint64_t arr = g_pulseArrived.exchange(0, std::memory_order_relaxed);
const uint64_t psh = g_pulsePushed.exchange(0, std::memory_order_relaxed);
const uint64_t dErr = g_pulseDropDecodeErr.exchange(0, std::memory_order_relaxed);
const uint64_t dKey = g_pulseDropKeyframeWait.exchange(0, std::memory_order_relaxed);
const uint64_t dRat = g_pulseDropRateLimit.exchange(0, std::memory_order_relaxed);
const uint64_t gtl = g_pulseGetCalls.exchange(0, std::memory_order_relaxed);
const uint64_t gNew = g_pulseGetNew.exchange(0, std::memory_order_relaxed);
const uint64_t gFst = g_pulseGetCachedFast.exchange(0, std::memory_order_relaxed);
const uint64_t gEmt = g_pulseGetCachedEmpty.exchange(0, std::memory_order_relaxed);
const uint64_t gNp = g_pulseGetCachedNotPlaying.exchange(0, std::memory_order_relaxed);
const uint64_t gStl = g_pulseGetCachedSettle.exchange(0, std::memory_order_relaxed);
const uint64_t evct = FrameQueue::evictedCounter().exchange(0, std::memory_order_relaxed);
const uint64_t skip = g_pulseSeqJump.exchange(0, std::memory_order_relaxed);
const long long maxGapTicks = g_pulseMaxGapTicks.exchange(0, std::memory_order_relaxed);
const double maxGapMs = std::chrono::duration<double, std::milli>(
clk::duration(maxGapTicks)).count();
// Production: route through ANS_DBG so the pulse is gated by the
// ANSCORE_DEBUGVIEW runtime toggle (env var ANSCENTER_DBGVIEW=1 or
// sentinel file C:\ProgramData\ANSCENTER\ansvisdebugview.txt).
// Silent by default; one-line snapshot per second when enabled.
//
// maxGap is the largest interval between two consecutive arrivals in
// this second. Healthy ~25fps streams should sit at maxGap≈40-60ms.
// A spike to 1000+ ms is the smoking gun for a producer-side stall:
// RTP rx hung, decoder waiting on references, etc.
ANS_DBG("MEDIA_Pulse",
"[1s] producer arrived=%llu pushed=%llu drop(err=%llu keyWait=%llu rate=%llu) "
"maxGap=%.0fms evicted=%llu | "
"consumer gets=%llu new=%llu skipped=%llu cached(fast=%llu empty=%llu notPlay=%llu settle=%llu) "
"| queueDepth=%zu",
(unsigned long long)arr, (unsigned long long)psh,
(unsigned long long)dErr, (unsigned long long)dKey, (unsigned long long)dRat,
maxGapMs, (unsigned long long)evct,
(unsigned long long)gtl, (unsigned long long)gNew, (unsigned long long)skip,
(unsigned long long)gFst, (unsigned long long)gEmt,
(unsigned long long)gNp, (unsigned long long)gStl,
queueDepth);
}
} // anonymous namespace
// libyuv: SIMD-accelerated YUV↔RGB conversion with native strided-plane input.
// Replaces the memcpy-into-staging + cv::cvtColor(COLOR_YUV2BGR_I420) chain
// in avframeYUV420PToCvMat with a direct I420→RGB24 (== OpenCV BGR memory
@@ -1920,14 +2028,13 @@ cv::Mat CVideoPlayer::avframeToCVMat(const AVFrame* pFrame) {
return cv::Mat();
}
// One-shot diagnostic: print the pixel format the first time through so
// we can see which branch of the switch below is taken. Remove after use.
// One-shot diagnostic: log the pixel format the first time through so
// we can see which branch of the switch below is taken. Production: only
// emitted when ANSCORE_DEBUGVIEW is enabled.
static bool s_loggedFmt = false;
if (!s_loggedFmt) {
s_loggedFmt = true;
const char* name = av_get_pix_fmt_name((AVPixelFormat)pFrame->format);
fprintf(stderr, "[avframeToCVMat] first frame format=%d (%s) %dx%d\n",
pFrame->format, name ? name : "?", pFrame->width, pFrame->height);
ANS_DBG("MEDIA_Convert",
"avframeToCVMat FIRST-FRAME fmt=%d(%s) %dx%d HWDecoding=%d (this=%p)",
pFrame->format, name ? name : "?",
@@ -2465,6 +2572,12 @@ void CVideoPlayer::StartVideoDecoder() {
m_lastFrameSeq = 0;
m_bWaitingForKeyframe = true; // Skip frames until first keyframe
m_cleanFrameCount = 0; // Reset settle counter
// Reset jitter buffer state so a restart starts measuring rate fresh
m_arrivalIdx = 0;
m_arrivalCount = 0;
m_observedFrameIntervalMs = 40.0; // re-seed at 25 fps
m_lastReleaseTimeSet = false;
m_jitterBufferRefilling = true; // new fill phase on (re)start
if (m_pVideoDecoder)
{
m_pVideoDecoder->Start();
@@ -2595,6 +2708,17 @@ void CVideoPlayer::setTargetFPS(double intervalMs)
m_targetIntervalMs = intervalMs;
m_targetFPSInitialized = false; // reset timing on change
}
void CVideoPlayer::setJitterBufferMs(int ms)
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
if (ms < 0) ms = 0;
if (ms > 2000) ms = 2000; // sanity ceiling — beyond this is just lag
m_jitterBufferMs = ms;
// Reset release timing and re-enter fill phase so the new setting takes
// effect cleanly on the next call.
m_lastReleaseTimeSet = false;
m_jitterBufferRefilling = (ms > 0);
}
double CVideoPlayer::getLastFrameAgeMs()
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
@@ -2802,10 +2926,60 @@ void CVideoPlayer::onVideoFrame(AVFrame* frame)
}
if (m_bPlaying && m_videoPlayFlag) {
// Drop any frame with decode errors (corrupted reference frames, etc.)
// MEDIA_Pulse: count every frame the decoder hands us, before any drop checks.
g_pulseArrived.fetch_add(1, std::memory_order_relaxed);
note_arrival_gap(); // tracks max gap between consecutive arrivals in this 1s window
// --- Arrival-rate measurement for the jitter buffer ---
// Record this arrival in the ring buffer, then update the observed
// inter-frame interval from the median delta of the last samples.
// Median is robust to bursts (TCP unstall delivers several frames in
// rapid succession; mean would be skewed, median picks up the typical
// in-burst spacing). Adapts within ~1 s when camera rate changes.
{
using clk = std::chrono::steady_clock;
const long long now = clk::now().time_since_epoch().count();
m_arrivalTicks[m_arrivalIdx % kArrivalRingSize] = now;
m_arrivalIdx++;
if (m_arrivalCount < kArrivalRingSize) m_arrivalCount++;
// Recompute every 8 frames (cheap; ring of 32 ints).
if (m_arrivalCount >= 4 && (m_arrivalIdx % 8) == 0) {
const size_t n = m_arrivalCount;
std::array<long long, kArrivalRingSize> sortedCopy = m_arrivalTicks;
std::sort(sortedCopy.begin(), sortedCopy.begin() + n);
std::array<long long, kArrivalRingSize - 1> deltas{};
for (size_t i = 1; i < n; ++i) deltas[i - 1] = sortedCopy[i] - sortedCopy[i - 1];
// Median delta
std::sort(deltas.begin(), deltas.begin() + (n - 1));
const long long medianTicks = deltas[(n - 1) / 2];
const double medianMs = std::chrono::duration<double, std::milli>(
clk::duration(medianTicks)).count();
// Clamp to sane camera framerate range (5–120 fps) so a
// degenerate measurement can't poison the buffer logic.
if (medianMs >= 8.0 && medianMs <= 200.0) {
m_observedFrameIntervalMs = medianMs;
}
}
}
// Pass frames with decode_error_flags through. The codec is configured
// with AV_CODEC_FLAG_OUTPUT_CORRUPT + AV_EF_IGNORE_ERR (video_decoder.cpp)
// so it has already applied concealment for missing references — the
// frame data is usable even if the flag is set.
//
// Why this matters: HW decoders (NVDEC/D3D11VA/DXVA2) silently conceal
// without setting the flag, but the SW decoder (default since commit
// e134ebd) sets the flag on every UDP packet loss. Dropping flagged
// frames meant every lost packet froze the display until the next IDR
// (~1-2s typical), reproducing the periodic freeze the user saw under
// UDP+SW. VLC plays through these frames — so do we now.
if (frame->decode_error_flags != 0) {
fprintf(stderr, "[HWDecode] Dropping frame with decode errors (flags=0x%x)\n", frame->decode_error_flags);
return;
g_pulseDropDecodeErr.fetch_add(1, std::memory_order_relaxed); // counted, not dropped
ANS_DBG("MEDIA_DecodeErr",
"frame had decode_error_flags=0x%x — passing through with concealment "
"(pict_type=%d key=%d)",
frame->decode_error_flags,
(int)frame->pict_type, (int)frame->key_frame);
}
// After start/restart, skip corrupted frames until first keyframe (IDR) arrives.
@@ -2815,8 +2989,11 @@ void CVideoPlayer::onVideoFrame(AVFrame* frame)
if (frame->key_frame || frame->pict_type == AV_PICTURE_TYPE_I) {
m_bWaitingForKeyframe = false;
m_cleanFrameCount = 0;
fprintf(stderr, "[HWDecode] First keyframe received, settling for %d frames\n", SETTLE_FRAME_COUNT);
ANS_DBG("MEDIA_Lifecycle",
"first keyframe received, settling for %d frames", SETTLE_FRAME_COUNT);
} else {
g_pulseDropKeyframeWait.fetch_add(1, std::memory_order_relaxed);
emit_media_pulse_if_due(g_frameQueue.size());
return; // Drop this frame — not yet safe to decode
}
}
@@ -2838,6 +3015,8 @@ void CVideoPlayer::onVideoFrame(AVFrame* frame)
} else {
auto elapsed = std::chrono::duration<double, std::milli>(now - m_lastProcessedTime).count();
if (elapsed < m_targetIntervalMs) {
g_pulseDropRateLimit.fetch_add(1, std::memory_order_relaxed);
emit_media_pulse_if_due(g_frameQueue.size());
return; // Skip this frame — too soon
}
}
@@ -2848,6 +3027,7 @@ void CVideoPlayer::onVideoFrame(AVFrame* frame)
// Push frame to queue; during settle period getImage() will ignore the queue
// and keep returning the last good cached image
g_frameQueue.pushFrame(frame); // pushFrame() clones the frame internally
g_pulsePushed.fetch_add(1, std::memory_order_relaxed);
// Capture CUDA HW frame for zero-copy inference.
// We're inside decode()'s lock scope (decoder._mutex held) AND onVideoFrame
@@ -2867,9 +3047,11 @@ void CVideoPlayer::onVideoFrame(AVFrame* frame)
if (m_cleanFrameCount < SETTLE_FRAME_COUNT) {
m_cleanFrameCount++;
if (m_cleanFrameCount == SETTLE_FRAME_COUNT) {
fprintf(stderr, "[HWDecode] Settle complete, delivering new frames\n");
ANS_DBG("MEDIA_Lifecycle", "settle complete, delivering new frames");
}
}
emit_media_pulse_if_due(g_frameQueue.size());
}
}
@@ -2901,13 +3083,20 @@ cv::Mat CVideoPlayer::getImage(int& width, int& height, int64_t& pts) {
AVFrame* frameToProcess = nullptr;
uint64_t currentSeq = 0;
// MEDIA_Pulse: count every getImage call for the per-second pulse,
// and try to emit on entry so the line still prints during a freeze
// where every call lands on a cached-return path. The once-per-second
// CAS inside makes the actual cost trivial.
g_pulseGetCalls.fetch_add(1, std::memory_order_relaxed);
emit_media_pulse_if_due(g_frameQueue.size());
// Timing breakdown — gated by ANSCORE_DEBUGVIEW (zero overhead in production).
// t0 = entry, t1 = after pulling frame from queue, t2 = after YUV->BGR,
// t3 = after publish. Throttled to every 30 full-path calls (~1/sec @30fps).
using clk = std::chrono::steady_clock;
const auto t0 = clk::now();
// --- Phase 1: short locked section — examine state, pull latest frame ---
// --- Phase 1: short locked section — examine state, pull frame ---
{
std::lock_guard<std::recursive_mutex> lock(_mutex);
@@ -2915,6 +3104,7 @@ cv::Mat CVideoPlayer::getImage(int& width, int& height, int64_t& pts) {
width = m_currentImage.cols;
height = m_currentImage.rows;
pts = m_pts;
g_pulseGetCachedNotPlaying.fetch_add(1, std::memory_order_relaxed);
return m_currentImage; // Shallow copy (reference counted)
}
@@ -2924,33 +3114,132 @@ cv::Mat CVideoPlayer::getImage(int& width, int& height, int64_t& pts) {
width = m_currentImage.cols;
height = m_currentImage.rows;
pts = m_pts;
g_pulseGetCachedSettle.fetch_add(1, std::memory_order_relaxed);
return m_currentImage;
}
// Fast path: same frame as last call — skip clone + BGR conversion
currentSeq = g_frameQueue.getSequence();
if (currentSeq == m_lastFrameSeq && !m_currentImage.empty()) {
width = m_currentImage.cols;
height = m_currentImage.rows;
pts = m_pts;
return m_currentImage;
}
if (g_frameQueue.isEmpty()) {
width = m_currentImage.cols;
height = m_currentImage.rows;
pts = m_pts;
std::cerr << "No frame available in getImage()" << std::endl;
return cv::Mat();
}
if (m_jitterBufferMs > 0 && m_observedFrameIntervalMs > 0.0) {
// =========================================================
// Jitter-buffer mode: FIFO + rate-paced + adaptive depth
// + fill-phase refill
// =========================================================
const double frameMs = m_observedFrameIntervalMs;
const size_t targetN = std::max<size_t>(1,
static_cast<size_t>(static_cast<double>(m_jitterBufferMs) / frameMs));
const size_t maxAllowed = 2 * targetN;
const size_t qSize = g_frameQueue.size();
// getLatestFrame() clones the AVFrame — we own it from here
frameToProcess = g_frameQueue.getLatestFrame();
if (!frameToProcess) {
width = m_currentImage.cols;
height = m_currentImage.rows;
pts = m_pts;
return cv::Mat();
// (1) Adaptive drop-ahead: if the consumer is slower than the
// producer, the queue overshoots target. Pop & discard
// down to targetN so latency stays bounded. Each discard
// counts as a "skipped" frame in the pulse.
if (qSize > maxAllowed) {
while (g_frameQueue.size() > targetN) {
AVFrame* old = g_frameQueue.popFrame();
if (old) {
av_frame_free(&old);
g_queueFrees.fetch_add(1, std::memory_order_relaxed);
g_pulseSeqJump.fetch_add(1, std::memory_order_relaxed);
}
}
}
// (2) Fill phase: hold all releases until the queue reaches
// target depth. Triggered at startup and again whenever
// the queue drains to empty mid-stream. Without this,
// 1.0x pacing keeps the queue at whatever depth it
// happens to be (zero, after a drain) — the buffer
// never refills.
if (m_jitterBufferRefilling) {
if (g_frameQueue.size() >= targetN) {
m_jitterBufferRefilling = false;
} else {
width = m_currentImage.cols;
height = m_currentImage.rows;
pts = m_pts;
g_pulseGetCachedFast.fetch_add(1, std::memory_order_relaxed);
return m_currentImage;
}
}
// (3) Rate-paced release at 1.0x of observed producer interval.
// Anything <1.0 leaks the buffer (consumer faster than
// producer); >1.0 keeps growing the queue until drop-ahead
// trips. 1.0 keeps the queue at its current depth in
// steady state.
if (m_lastReleaseTimeSet) {
const double sinceLastMs = std::chrono::duration<double, std::milli>(
t0 - m_lastReleaseTime).count();
if (sinceLastMs < frameMs) {
width = m_currentImage.cols;
height = m_currentImage.rows;
pts = m_pts;
g_pulseGetCachedFast.fetch_add(1, std::memory_order_relaxed);
return m_currentImage;
}
}
// (4) Empty queue (network gap drained the buffer): re-enter
// fill phase so the next producer pushes are held back to
// rebuild the buffer to target depth before the next
// release. Without this, a one-time gap permanently
// shrinks the buffer and we lose absorption capacity.
if (g_frameQueue.isEmpty()) {
m_jitterBufferRefilling = true;
width = m_currentImage.cols;
height = m_currentImage.rows;
pts = m_pts;
g_pulseGetCachedEmpty.fetch_add(1, std::memory_order_relaxed);
return cv::Mat();
}
// (5) FIFO pop — the frame is now ours; we must free it after
// conversion (popFrame returns the original, not a clone).
frameToProcess = g_frameQueue.popFrame();
if (!frameToProcess) {
m_jitterBufferRefilling = true;
width = m_currentImage.cols;
height = m_currentImage.rows;
pts = m_pts;
g_pulseGetCachedEmpty.fetch_add(1, std::memory_order_relaxed);
return cv::Mat();
}
m_lastReleaseTime = t0;
m_lastReleaseTimeSet = true;
} else {
// =========================================================
// Legacy mode (m_jitterBufferMs == 0): getLatest semantics
// =========================================================
// Fast path: same frame as last call — skip clone + BGR conversion
if (currentSeq == m_lastFrameSeq && !m_currentImage.empty()) {
width = m_currentImage.cols;
height = m_currentImage.rows;
pts = m_pts;
g_pulseGetCachedFast.fetch_add(1, std::memory_order_relaxed);
return m_currentImage;
}
if (g_frameQueue.isEmpty()) {
width = m_currentImage.cols;
height = m_currentImage.rows;
pts = m_pts;
g_pulseGetCachedEmpty.fetch_add(1, std::memory_order_relaxed);
// Common case during network jitter — gated to avoid stderr spam.
ANS_DBG("MEDIA_GetImage", "queue empty, returning cached");
return cv::Mat();
}
// getLatestFrame() clones the AVFrame — we own it from here
frameToProcess = g_frameQueue.getLatestFrame();
if (!frameToProcess) {
width = m_currentImage.cols;
height = m_currentImage.rows;
pts = m_pts;
g_pulseGetCachedEmpty.fetch_add(1, std::memory_order_relaxed);
return cv::Mat();
}
}
}
// --- _mutex released here ---
@@ -2964,7 +3253,8 @@ cv::Mat CVideoPlayer::getImage(int& width, int& height, int64_t& pts) {
converted = avframeToCVMat(frameToProcess);
}
catch (const std::exception& e) {
std::cerr << "Exception while converting AVFrame to cv::Mat: " << e.what() << std::endl;
ANS_DBG("MEDIA_GetImage",
"avframeToCVMat threw: %s — returning empty Mat", e.what());
}
const auto t2 = clk::now();
@@ -2974,9 +3264,20 @@ cv::Mat CVideoPlayer::getImage(int& width, int& height, int64_t& pts) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
if (!converted.empty()) {
// Skip-tracking: only meaningful in legacy mode where the
// consumer reads the back of the queue. In jitter-buffer mode
// we already counted drop-ahead discards; sequence numbers
// don't track displayed-vs-skipped because pop is FIFO.
if (m_jitterBufferMs == 0) {
if (m_lastFrameSeq != 0 && currentSeq > m_lastFrameSeq + 1) {
g_pulseSeqJump.fetch_add(currentSeq - m_lastFrameSeq - 1,
std::memory_order_relaxed);
}
m_lastFrameSeq = currentSeq;
}
m_currentImage = converted;
m_pts++;
m_lastFrameSeq = currentSeq;
g_pulseGetNew.fetch_add(1, std::memory_order_relaxed);
}
// Preserve raw YUV/NV12 frame for GPU fast-path inference
@@ -3076,11 +3377,11 @@ cv::Mat CVideoPlayer::getImage(int& width, int& height, int64_t& pts) {
return result;
}
catch (const std::exception& e) {
std::cerr << "Unexpected exception in getImage(): " << e.what() << std::endl;
ANS_DBG("MEDIA_GetImage", "outer catch: %s", e.what());
return cv::Mat(); // Return an empty cv::Mat if an exception occurs
}
catch (...) {
std::cerr << "Unknown exception in getImage()" << std::endl;
ANS_DBG("MEDIA_GetImage", "outer catch: unknown exception");
return cv::Mat(); // Return an empty cv::Mat if an exception occurs
}
@@ -3124,7 +3425,7 @@ std::string CVideoPlayer::getJpegImage(int& width, int& height, int64_t& pts) {
}
}
catch (const std::exception& e) {
std::cerr << "Exception while converting AVFrame to JPEG string: " << e.what() << std::endl;
ANS_DBG("MEDIA_GetJpeg", "encode threw: %s — returning cached JPEG", e.what());
av_frame_free(&frameToProcess);
g_queueFrees.fetch_add(1, std::memory_order_relaxed);
return m_lastJpegImage;
@@ -3176,10 +3477,10 @@ std::string CVideoPlayer::getJpegImage(int& width, int& height, int64_t& pts) {
return m_lastJpegImage;
}
catch (const std::exception& e) {
std::cerr << "Unexpected exception in getJpegImage(): " << e.what() << std::endl;
ANS_DBG("MEDIA_GetJpeg", "outer catch: %s", e.what());
}
catch (...) {
std::cerr << "Unknown exception in getJpegImage()" << std::endl;
ANS_DBG("MEDIA_GetJpeg", "outer catch: unknown exception");
}
// If any exception occurs, return the last valid JPEG image
+47
View File
@@ -11,6 +11,7 @@
#include "media_util.h"
#include <list>
#include <string>
#include <array>
#include <opencv2/imgproc.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/opencv.hpp>
@@ -51,6 +52,16 @@ private:
uint64_t m_frameSeq = 0; // Sequence counter — increments on each new frame push
public:
// Pulse counter for evicted-unread frames (instrumentation, not behavior).
// Incremented every time a push has to drop the oldest frame because the
// queue hit maxFrames AND the oldest never got read. With getLatestFrame()
// the consumer reads the back, so anything pushed before the consumer's
// last read sits in the middle/front and gets silently evicted here.
static std::atomic<uint64_t>& evictedCounter() {
static std::atomic<uint64_t> g{0};
return g;
}
// Push a new frame to the queue (removes oldest if full)
void pushFrame(AVFrame* newFrame) {
if (!newFrame) return;
@@ -74,6 +85,7 @@ public:
frameQueue.pop();
av_frame_free(&oldFrame);
g_queueFrees.fetch_add(1, std::memory_order_relaxed);
evictedCounter().fetch_add(1, std::memory_order_relaxed);
}
}
@@ -177,6 +189,16 @@ public:
virtual void setImageQuality(int mode) { m_nImageQuality = mode; }
void setTargetFPS(double intervalMs); // Set minimum interval between processed frames in ms (0 = no limit, 100 = ~10 FPS)
double getLastFrameAgeMs(); // Milliseconds since last frame arrived from decoder (0 if no frame yet)
// Jitter buffer: hold decoded frames for ~ms before delivering to consumer.
// Smooths network/decoder jitter at the cost of added end-to-end latency.
// 0 disables (today's getLatest semantics). Default 200 ms.
// Adaptive: if the consumer is slower than the producer, drop-ahead kicks in
// and bypasses the buffer (latency stays ~0 frames).
// Note: applies only to the BGR cv::Mat returned by getImage(); the NV12/CUDA
// fast-path frames (m_currentNV12Frame / m_currentCudaHWFrame) remain
// updated at push time so inference always sees the latest.
void setJitterBufferMs(int ms);
int getJitterBufferMs() const { return m_jitterBufferMs; }
virtual void setRtpMulticast(BOOL flag) {}
virtual void setRtpOverUdp(BOOL flag) {}
@@ -317,6 +339,31 @@ protected:
std::chrono::steady_clock::time_point m_lastDecoderFrameTime;
bool m_lastDecoderFrameTimeSet = false;
// --- Jitter buffer (BGR delivery only) ---
// Default 200 ms — absorbs typical network/decoder hiccups.
int m_jitterBufferMs = 200;
// Observed inter-arrival interval, ms. Seeds at 40 ms (=25 fps) and is
// refined to a rolling median of the last m_arrivalCount samples.
double m_observedFrameIntervalMs = 40.0;
// Ring buffer of recent arrival ticks (steady_clock raw ticks). Used to
// compute the rolling median frame interval. Sized at 32 — covers >1 s at
// 30 fps so a single GOP-boundary jitter doesn't dominate.
static constexpr size_t kArrivalRingSize = 32;
std::array<long long, kArrivalRingSize> m_arrivalTicks{};
size_t m_arrivalIdx = 0;
size_t m_arrivalCount = 0;
// Wall-clock of the last frame released to the consumer. Used for rate-paced
// delivery — if the consumer polls faster than the producer rate, between
// releases we return the cached frame so the buffer doesn't drain instantly.
std::chrono::steady_clock::time_point m_lastReleaseTime;
bool m_lastReleaseTimeSet = false;
// Fill-phase: when true, hold releases until queue depth reaches target.
// Set true at startup and again whenever the queue drains to empty mid-stream.
// This is what makes the buffer actually buffer — without it, pacing alone
// can't grow the queue from empty (producer and consumer would just match
// rates with depth=0 forever).
bool m_jitterBufferRefilling = true;
BOOL m_bPlaying;
BOOL m_bPaused;