Add jitter buffer
This commit is contained in:
@@ -27,6 +27,8 @@ extern "C"
|
||||
#endif
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <array>
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <atomic>
|
||||
#include <libswscale/swscale.h>
|
||||
@@ -56,6 +58,112 @@ extern std::atomic<int64_t> g_contiguousAllocs;
|
||||
extern std::atomic<int64_t> g_contiguousFrees;
|
||||
extern std::atomic<int64_t> g_contiguousBytesInFlight;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Freeze diagnostics — once-per-second pulse showing both producer
|
||||
// (onVideoFrame) and consumer (getImage) accounting in a single line so a
|
||||
// 1-2 second freeze can be localized in DebugView. Filter on "MEDIA_Pulse".
|
||||
//
|
||||
// Reading the line during a freeze:
|
||||
// arrived high, pushed=0 ............ all frames dropped at producer
|
||||
// drop(err=N) UDP packet loss → SW decoder set decode_error_flags
|
||||
// drop(keyWait=N) waiting for first IDR after start/restart
|
||||
// drop(rate=N) SetTargetFPS throttle skipped them
|
||||
// arrived=0 .................. decoder silent (network/rx stall)
|
||||
// arrived≈pushed, gets high, new=0 ... consumer not seeing the new push
|
||||
// (mutex contention, seq stuck, ...)
|
||||
// arrived≈pushed, new high, smooth ..... pipeline healthy
|
||||
// ---------------------------------------------------------------------------
|
||||
namespace {
|
||||
std::atomic<uint64_t> g_pulseArrived{0}; // onVideoFrame entries
|
||||
std::atomic<uint64_t> g_pulsePushed{0}; // pushFrame() calls (frame survived all drop checks)
|
||||
std::atomic<uint64_t> g_pulseDropDecodeErr{0}; // decode_error_flags != 0 (now also pushed; counted for visibility)
|
||||
std::atomic<uint64_t> g_pulseDropKeyframeWait{0}; // m_bWaitingForKeyframe and not IDR
|
||||
std::atomic<uint64_t> g_pulseDropRateLimit{0}; // SetTargetFPS interval skip
|
||||
std::atomic<uint64_t> g_pulseGetCalls{0}; // getImage entries
|
||||
std::atomic<uint64_t> g_pulseGetNew{0}; // returned a brand-new frame
|
||||
std::atomic<uint64_t> g_pulseGetCachedFast{0}; // currentSeq == m_lastFrameSeq (no new push)
|
||||
std::atomic<uint64_t> g_pulseGetCachedEmpty{0}; // queue empty
|
||||
std::atomic<uint64_t> g_pulseGetCachedNotPlaying{0};// !m_bPlaying
|
||||
std::atomic<uint64_t> g_pulseGetCachedSettle{0}; // waiting for keyframe / settle period
|
||||
std::atomic<uint64_t> g_pulseSeqJump{0}; // total frames the consumer flew past via getLatestFrame
|
||||
// (sum of (currentSeq - m_lastFrameSeq - 1) across all reads)
|
||||
std::atomic<long long> g_pulseNextEmitTick{0}; // monotonic deadline (steady_clock::now ticks)
|
||||
|
||||
// Inter-arrival timing — exposes "21 frames bunched then 900ms silence" patterns
|
||||
// that aggregate counters hide. Both stored as steady_clock raw ticks.
|
||||
std::atomic<long long> g_pulseLastArrivalTick{0}; // tick of previous onVideoFrame entry (0 = none)
|
||||
std::atomic<long long> g_pulseMaxGapTicks{0}; // max gap seen in current 1s window
|
||||
|
||||
// Called from onVideoFrame on every arrival. Updates "max gap" if the gap
|
||||
// since previous arrival exceeds the current window's running max.
|
||||
inline void note_arrival_gap() {
|
||||
using clk = std::chrono::steady_clock;
|
||||
const long long now = clk::now().time_since_epoch().count();
|
||||
const long long prev = g_pulseLastArrivalTick.exchange(now, std::memory_order_relaxed);
|
||||
if (prev == 0) return; // first arrival ever — no gap
|
||||
const long long gap = now - prev;
|
||||
// Atomic max: only update if larger
|
||||
long long cur = g_pulseMaxGapTicks.load(std::memory_order_relaxed);
|
||||
while (gap > cur &&
|
||||
!g_pulseMaxGapTicks.compare_exchange_weak(cur, gap, std::memory_order_relaxed)) {
|
||||
// CAS retry until we win or someone else wrote a larger value
|
||||
}
|
||||
}
|
||||
|
||||
// Emit if 1s has elapsed since the previous emission. CAS makes only one
|
||||
// thread per second do the print; the rest just no-op.
|
||||
inline void emit_media_pulse_if_due(size_t queueDepth) {
|
||||
using clk = std::chrono::steady_clock;
|
||||
const long long tick = clk::now().time_since_epoch().count();
|
||||
long long expected = g_pulseNextEmitTick.load(std::memory_order_relaxed);
|
||||
if (tick < expected) return;
|
||||
const long long deadline = tick + std::chrono::duration_cast<clk::duration>(
|
||||
std::chrono::milliseconds(1000)).count();
|
||||
if (!g_pulseNextEmitTick.compare_exchange_strong(
|
||||
expected, deadline, std::memory_order_relaxed)) {
|
||||
return; // another thread won the window
|
||||
}
|
||||
// Snapshot + reset (delta semantics — every line is "events in last ~1s")
|
||||
const uint64_t arr = g_pulseArrived.exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t psh = g_pulsePushed.exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t dErr = g_pulseDropDecodeErr.exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t dKey = g_pulseDropKeyframeWait.exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t dRat = g_pulseDropRateLimit.exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t gtl = g_pulseGetCalls.exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t gNew = g_pulseGetNew.exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t gFst = g_pulseGetCachedFast.exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t gEmt = g_pulseGetCachedEmpty.exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t gNp = g_pulseGetCachedNotPlaying.exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t gStl = g_pulseGetCachedSettle.exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t evct = FrameQueue::evictedCounter().exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t skip = g_pulseSeqJump.exchange(0, std::memory_order_relaxed);
|
||||
const long long maxGapTicks = g_pulseMaxGapTicks.exchange(0, std::memory_order_relaxed);
|
||||
const double maxGapMs = std::chrono::duration<double, std::milli>(
|
||||
clk::duration(maxGapTicks)).count();
|
||||
// Production: route through ANS_DBG so the pulse is gated by the
|
||||
// ANSCORE_DEBUGVIEW runtime toggle (env var ANSCENTER_DBGVIEW=1 or
|
||||
// sentinel file C:\ProgramData\ANSCENTER\ansvisdebugview.txt).
|
||||
// Silent by default; one-line snapshot per second when enabled.
|
||||
//
|
||||
// maxGap is the largest interval between two consecutive arrivals in
|
||||
// this second. Healthy ~25fps streams should sit at maxGap≈40-60ms.
|
||||
// A spike to 1000+ ms is the smoking gun for a producer-side stall:
|
||||
// RTP rx hung, decoder waiting on references, etc.
|
||||
ANS_DBG("MEDIA_Pulse",
|
||||
"[1s] producer arrived=%llu pushed=%llu drop(err=%llu keyWait=%llu rate=%llu) "
|
||||
"maxGap=%.0fms evicted=%llu | "
|
||||
"consumer gets=%llu new=%llu skipped=%llu cached(fast=%llu empty=%llu notPlay=%llu settle=%llu) "
|
||||
"| queueDepth=%zu",
|
||||
(unsigned long long)arr, (unsigned long long)psh,
|
||||
(unsigned long long)dErr, (unsigned long long)dKey, (unsigned long long)dRat,
|
||||
maxGapMs, (unsigned long long)evct,
|
||||
(unsigned long long)gtl, (unsigned long long)gNew, (unsigned long long)skip,
|
||||
(unsigned long long)gFst, (unsigned long long)gEmt,
|
||||
(unsigned long long)gNp, (unsigned long long)gStl,
|
||||
queueDepth);
|
||||
}
|
||||
} // anonymous namespace
|
||||
|
||||
// libyuv: SIMD-accelerated YUV↔RGB conversion with native strided-plane input.
|
||||
// Replaces the memcpy-into-staging + cv::cvtColor(COLOR_YUV2BGR_I420) chain
|
||||
// in avframeYUV420PToCvMat with a direct I420→RGB24 (== OpenCV BGR memory
|
||||
@@ -1920,14 +2028,13 @@ cv::Mat CVideoPlayer::avframeToCVMat(const AVFrame* pFrame) {
|
||||
return cv::Mat();
|
||||
}
|
||||
|
||||
// One-shot diagnostic: print the pixel format the first time through so
|
||||
// we can see which branch of the switch below is taken. Remove after use.
|
||||
// One-shot diagnostic: log the pixel format the first time through so
|
||||
// we can see which branch of the switch below is taken. Production: only
|
||||
// emitted when ANSCORE_DEBUGVIEW is enabled.
|
||||
static bool s_loggedFmt = false;
|
||||
if (!s_loggedFmt) {
|
||||
s_loggedFmt = true;
|
||||
const char* name = av_get_pix_fmt_name((AVPixelFormat)pFrame->format);
|
||||
fprintf(stderr, "[avframeToCVMat] first frame format=%d (%s) %dx%d\n",
|
||||
pFrame->format, name ? name : "?", pFrame->width, pFrame->height);
|
||||
ANS_DBG("MEDIA_Convert",
|
||||
"avframeToCVMat FIRST-FRAME fmt=%d(%s) %dx%d HWDecoding=%d (this=%p)",
|
||||
pFrame->format, name ? name : "?",
|
||||
@@ -2465,6 +2572,12 @@ void CVideoPlayer::StartVideoDecoder() {
|
||||
m_lastFrameSeq = 0;
|
||||
m_bWaitingForKeyframe = true; // Skip frames until first keyframe
|
||||
m_cleanFrameCount = 0; // Reset settle counter
|
||||
// Reset jitter buffer state so a restart starts measuring rate fresh
|
||||
m_arrivalIdx = 0;
|
||||
m_arrivalCount = 0;
|
||||
m_observedFrameIntervalMs = 40.0; // re-seed at 25 fps
|
||||
m_lastReleaseTimeSet = false;
|
||||
m_jitterBufferRefilling = true; // new fill phase on (re)start
|
||||
if (m_pVideoDecoder)
|
||||
{
|
||||
m_pVideoDecoder->Start();
|
||||
@@ -2595,6 +2708,17 @@ void CVideoPlayer::setTargetFPS(double intervalMs)
|
||||
m_targetIntervalMs = intervalMs;
|
||||
m_targetFPSInitialized = false; // reset timing on change
|
||||
}
|
||||
void CVideoPlayer::setJitterBufferMs(int ms)
|
||||
{
|
||||
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||
if (ms < 0) ms = 0;
|
||||
if (ms > 2000) ms = 2000; // sanity ceiling — beyond this is just lag
|
||||
m_jitterBufferMs = ms;
|
||||
// Reset release timing and re-enter fill phase so the new setting takes
|
||||
// effect cleanly on the next call.
|
||||
m_lastReleaseTimeSet = false;
|
||||
m_jitterBufferRefilling = (ms > 0);
|
||||
}
|
||||
double CVideoPlayer::getLastFrameAgeMs()
|
||||
{
|
||||
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||
@@ -2802,10 +2926,60 @@ void CVideoPlayer::onVideoFrame(AVFrame* frame)
|
||||
}
|
||||
|
||||
if (m_bPlaying && m_videoPlayFlag) {
|
||||
// Drop any frame with decode errors (corrupted reference frames, etc.)
|
||||
// MEDIA_Pulse: count every frame the decoder hands us, before any drop checks.
|
||||
g_pulseArrived.fetch_add(1, std::memory_order_relaxed);
|
||||
note_arrival_gap(); // tracks max gap between consecutive arrivals in this 1s window
|
||||
|
||||
// --- Arrival-rate measurement for the jitter buffer ---
|
||||
// Record this arrival in the ring buffer, then update the observed
|
||||
// inter-frame interval from the median delta of the last samples.
|
||||
// Median is robust to bursts (TCP unstall delivers several frames in
|
||||
// rapid succession; mean would be skewed, median picks up the typical
|
||||
// in-burst spacing). Adapts within ~1 s when camera rate changes.
|
||||
{
|
||||
using clk = std::chrono::steady_clock;
|
||||
const long long now = clk::now().time_since_epoch().count();
|
||||
m_arrivalTicks[m_arrivalIdx % kArrivalRingSize] = now;
|
||||
m_arrivalIdx++;
|
||||
if (m_arrivalCount < kArrivalRingSize) m_arrivalCount++;
|
||||
// Recompute every 8 frames (cheap; ring of 32 ints).
|
||||
if (m_arrivalCount >= 4 && (m_arrivalIdx % 8) == 0) {
|
||||
const size_t n = m_arrivalCount;
|
||||
std::array<long long, kArrivalRingSize> sortedCopy = m_arrivalTicks;
|
||||
std::sort(sortedCopy.begin(), sortedCopy.begin() + n);
|
||||
std::array<long long, kArrivalRingSize - 1> deltas{};
|
||||
for (size_t i = 1; i < n; ++i) deltas[i - 1] = sortedCopy[i] - sortedCopy[i - 1];
|
||||
// Median delta
|
||||
std::sort(deltas.begin(), deltas.begin() + (n - 1));
|
||||
const long long medianTicks = deltas[(n - 1) / 2];
|
||||
const double medianMs = std::chrono::duration<double, std::milli>(
|
||||
clk::duration(medianTicks)).count();
|
||||
// Clamp to sane camera framerate range (5–120 fps) so a
|
||||
// degenerate measurement can't poison the buffer logic.
|
||||
if (medianMs >= 8.0 && medianMs <= 200.0) {
|
||||
m_observedFrameIntervalMs = medianMs;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Pass frames with decode_error_flags through. The codec is configured
|
||||
// with AV_CODEC_FLAG_OUTPUT_CORRUPT + AV_EF_IGNORE_ERR (video_decoder.cpp)
|
||||
// so it has already applied concealment for missing references — the
|
||||
// frame data is usable even if the flag is set.
|
||||
//
|
||||
// Why this matters: HW decoders (NVDEC/D3D11VA/DXVA2) silently conceal
|
||||
// without setting the flag, but the SW decoder (default since commit
|
||||
// e134ebd) sets the flag on every UDP packet loss. Dropping flagged
|
||||
// frames meant every lost packet froze the display until the next IDR
|
||||
// (~1-2s typical), reproducing the periodic freeze the user saw under
|
||||
// UDP+SW. VLC plays through these frames — so do we now.
|
||||
if (frame->decode_error_flags != 0) {
|
||||
fprintf(stderr, "[HWDecode] Dropping frame with decode errors (flags=0x%x)\n", frame->decode_error_flags);
|
||||
return;
|
||||
g_pulseDropDecodeErr.fetch_add(1, std::memory_order_relaxed); // counted, not dropped
|
||||
ANS_DBG("MEDIA_DecodeErr",
|
||||
"frame had decode_error_flags=0x%x — passing through with concealment "
|
||||
"(pict_type=%d key=%d)",
|
||||
frame->decode_error_flags,
|
||||
(int)frame->pict_type, (int)frame->key_frame);
|
||||
}
|
||||
|
||||
// After start/restart, skip corrupted frames until first keyframe (IDR) arrives.
|
||||
@@ -2815,8 +2989,11 @@ void CVideoPlayer::onVideoFrame(AVFrame* frame)
|
||||
if (frame->key_frame || frame->pict_type == AV_PICTURE_TYPE_I) {
|
||||
m_bWaitingForKeyframe = false;
|
||||
m_cleanFrameCount = 0;
|
||||
fprintf(stderr, "[HWDecode] First keyframe received, settling for %d frames\n", SETTLE_FRAME_COUNT);
|
||||
ANS_DBG("MEDIA_Lifecycle",
|
||||
"first keyframe received, settling for %d frames", SETTLE_FRAME_COUNT);
|
||||
} else {
|
||||
g_pulseDropKeyframeWait.fetch_add(1, std::memory_order_relaxed);
|
||||
emit_media_pulse_if_due(g_frameQueue.size());
|
||||
return; // Drop this frame — not yet safe to decode
|
||||
}
|
||||
}
|
||||
@@ -2838,6 +3015,8 @@ void CVideoPlayer::onVideoFrame(AVFrame* frame)
|
||||
} else {
|
||||
auto elapsed = std::chrono::duration<double, std::milli>(now - m_lastProcessedTime).count();
|
||||
if (elapsed < m_targetIntervalMs) {
|
||||
g_pulseDropRateLimit.fetch_add(1, std::memory_order_relaxed);
|
||||
emit_media_pulse_if_due(g_frameQueue.size());
|
||||
return; // Skip this frame — too soon
|
||||
}
|
||||
}
|
||||
@@ -2848,6 +3027,7 @@ void CVideoPlayer::onVideoFrame(AVFrame* frame)
|
||||
// Push frame to queue; during settle period getImage() will ignore the queue
|
||||
// and keep returning the last good cached image
|
||||
g_frameQueue.pushFrame(frame); // pushFrame() clones the frame internally
|
||||
g_pulsePushed.fetch_add(1, std::memory_order_relaxed);
|
||||
|
||||
// Capture CUDA HW frame for zero-copy inference.
|
||||
// We're inside decode()'s lock scope (decoder._mutex held) AND onVideoFrame
|
||||
@@ -2867,9 +3047,11 @@ void CVideoPlayer::onVideoFrame(AVFrame* frame)
|
||||
if (m_cleanFrameCount < SETTLE_FRAME_COUNT) {
|
||||
m_cleanFrameCount++;
|
||||
if (m_cleanFrameCount == SETTLE_FRAME_COUNT) {
|
||||
fprintf(stderr, "[HWDecode] Settle complete, delivering new frames\n");
|
||||
ANS_DBG("MEDIA_Lifecycle", "settle complete, delivering new frames");
|
||||
}
|
||||
}
|
||||
|
||||
emit_media_pulse_if_due(g_frameQueue.size());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2901,13 +3083,20 @@ cv::Mat CVideoPlayer::getImage(int& width, int& height, int64_t& pts) {
|
||||
AVFrame* frameToProcess = nullptr;
|
||||
uint64_t currentSeq = 0;
|
||||
|
||||
// MEDIA_Pulse: count every getImage call for the per-second pulse,
|
||||
// and try to emit on entry so the line still prints during a freeze
|
||||
// where every call lands on a cached-return path. The once-per-second
|
||||
// CAS inside makes the actual cost trivial.
|
||||
g_pulseGetCalls.fetch_add(1, std::memory_order_relaxed);
|
||||
emit_media_pulse_if_due(g_frameQueue.size());
|
||||
|
||||
// Timing breakdown — gated by ANSCORE_DEBUGVIEW (zero overhead in production).
|
||||
// t0 = entry, t1 = after pulling frame from queue, t2 = after YUV->BGR,
|
||||
// t3 = after publish. Throttled to every 30 full-path calls (~1/sec @30fps).
|
||||
using clk = std::chrono::steady_clock;
|
||||
const auto t0 = clk::now();
|
||||
|
||||
// --- Phase 1: short locked section — examine state, pull latest frame ---
|
||||
// --- Phase 1: short locked section — examine state, pull frame ---
|
||||
{
|
||||
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||
|
||||
@@ -2915,6 +3104,7 @@ cv::Mat CVideoPlayer::getImage(int& width, int& height, int64_t& pts) {
|
||||
width = m_currentImage.cols;
|
||||
height = m_currentImage.rows;
|
||||
pts = m_pts;
|
||||
g_pulseGetCachedNotPlaying.fetch_add(1, std::memory_order_relaxed);
|
||||
return m_currentImage; // Shallow copy (reference counted)
|
||||
}
|
||||
|
||||
@@ -2924,33 +3114,132 @@ cv::Mat CVideoPlayer::getImage(int& width, int& height, int64_t& pts) {
|
||||
width = m_currentImage.cols;
|
||||
height = m_currentImage.rows;
|
||||
pts = m_pts;
|
||||
g_pulseGetCachedSettle.fetch_add(1, std::memory_order_relaxed);
|
||||
return m_currentImage;
|
||||
}
|
||||
|
||||
// Fast path: same frame as last call — skip clone + BGR conversion
|
||||
currentSeq = g_frameQueue.getSequence();
|
||||
if (currentSeq == m_lastFrameSeq && !m_currentImage.empty()) {
|
||||
width = m_currentImage.cols;
|
||||
height = m_currentImage.rows;
|
||||
pts = m_pts;
|
||||
return m_currentImage;
|
||||
}
|
||||
|
||||
if (g_frameQueue.isEmpty()) {
|
||||
width = m_currentImage.cols;
|
||||
height = m_currentImage.rows;
|
||||
pts = m_pts;
|
||||
std::cerr << "No frame available in getImage()" << std::endl;
|
||||
return cv::Mat();
|
||||
}
|
||||
if (m_jitterBufferMs > 0 && m_observedFrameIntervalMs > 0.0) {
|
||||
// =========================================================
|
||||
// Jitter-buffer mode: FIFO + rate-paced + adaptive depth
|
||||
// + fill-phase refill
|
||||
// =========================================================
|
||||
const double frameMs = m_observedFrameIntervalMs;
|
||||
const size_t targetN = std::max<size_t>(1,
|
||||
static_cast<size_t>(static_cast<double>(m_jitterBufferMs) / frameMs));
|
||||
const size_t maxAllowed = 2 * targetN;
|
||||
const size_t qSize = g_frameQueue.size();
|
||||
|
||||
// getLatestFrame() clones the AVFrame — we own it from here
|
||||
frameToProcess = g_frameQueue.getLatestFrame();
|
||||
if (!frameToProcess) {
|
||||
width = m_currentImage.cols;
|
||||
height = m_currentImage.rows;
|
||||
pts = m_pts;
|
||||
return cv::Mat();
|
||||
// (1) Adaptive drop-ahead: if the consumer is slower than the
|
||||
// producer, the queue overshoots target. Pop & discard
|
||||
// down to targetN so latency stays bounded. Each discard
|
||||
// counts as a "skipped" frame in the pulse.
|
||||
if (qSize > maxAllowed) {
|
||||
while (g_frameQueue.size() > targetN) {
|
||||
AVFrame* old = g_frameQueue.popFrame();
|
||||
if (old) {
|
||||
av_frame_free(&old);
|
||||
g_queueFrees.fetch_add(1, std::memory_order_relaxed);
|
||||
g_pulseSeqJump.fetch_add(1, std::memory_order_relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// (2) Fill phase: hold all releases until the queue reaches
|
||||
// target depth. Triggered at startup and again whenever
|
||||
// the queue drains to empty mid-stream. Without this,
|
||||
// 1.0x pacing keeps the queue at whatever depth it
|
||||
// happens to be (zero, after a drain) — the buffer
|
||||
// never refills.
|
||||
if (m_jitterBufferRefilling) {
|
||||
if (g_frameQueue.size() >= targetN) {
|
||||
m_jitterBufferRefilling = false;
|
||||
} else {
|
||||
width = m_currentImage.cols;
|
||||
height = m_currentImage.rows;
|
||||
pts = m_pts;
|
||||
g_pulseGetCachedFast.fetch_add(1, std::memory_order_relaxed);
|
||||
return m_currentImage;
|
||||
}
|
||||
}
|
||||
|
||||
// (3) Rate-paced release at 1.0x of observed producer interval.
|
||||
// Anything <1.0 leaks the buffer (consumer faster than
|
||||
// producer); >1.0 keeps growing the queue until drop-ahead
|
||||
// trips. 1.0 keeps the queue at its current depth in
|
||||
// steady state.
|
||||
if (m_lastReleaseTimeSet) {
|
||||
const double sinceLastMs = std::chrono::duration<double, std::milli>(
|
||||
t0 - m_lastReleaseTime).count();
|
||||
if (sinceLastMs < frameMs) {
|
||||
width = m_currentImage.cols;
|
||||
height = m_currentImage.rows;
|
||||
pts = m_pts;
|
||||
g_pulseGetCachedFast.fetch_add(1, std::memory_order_relaxed);
|
||||
return m_currentImage;
|
||||
}
|
||||
}
|
||||
|
||||
// (4) Empty queue (network gap drained the buffer): re-enter
|
||||
// fill phase so the next producer pushes are held back to
|
||||
// rebuild the buffer to target depth before the next
|
||||
// release. Without this, a one-time gap permanently
|
||||
// shrinks the buffer and we lose absorption capacity.
|
||||
if (g_frameQueue.isEmpty()) {
|
||||
m_jitterBufferRefilling = true;
|
||||
width = m_currentImage.cols;
|
||||
height = m_currentImage.rows;
|
||||
pts = m_pts;
|
||||
g_pulseGetCachedEmpty.fetch_add(1, std::memory_order_relaxed);
|
||||
return cv::Mat();
|
||||
}
|
||||
|
||||
// (5) FIFO pop — the frame is now ours; we must free it after
|
||||
// conversion (popFrame returns the original, not a clone).
|
||||
frameToProcess = g_frameQueue.popFrame();
|
||||
if (!frameToProcess) {
|
||||
m_jitterBufferRefilling = true;
|
||||
width = m_currentImage.cols;
|
||||
height = m_currentImage.rows;
|
||||
pts = m_pts;
|
||||
g_pulseGetCachedEmpty.fetch_add(1, std::memory_order_relaxed);
|
||||
return cv::Mat();
|
||||
}
|
||||
m_lastReleaseTime = t0;
|
||||
m_lastReleaseTimeSet = true;
|
||||
} else {
|
||||
// =========================================================
|
||||
// Legacy mode (m_jitterBufferMs == 0): getLatest semantics
|
||||
// =========================================================
|
||||
// Fast path: same frame as last call — skip clone + BGR conversion
|
||||
if (currentSeq == m_lastFrameSeq && !m_currentImage.empty()) {
|
||||
width = m_currentImage.cols;
|
||||
height = m_currentImage.rows;
|
||||
pts = m_pts;
|
||||
g_pulseGetCachedFast.fetch_add(1, std::memory_order_relaxed);
|
||||
return m_currentImage;
|
||||
}
|
||||
|
||||
if (g_frameQueue.isEmpty()) {
|
||||
width = m_currentImage.cols;
|
||||
height = m_currentImage.rows;
|
||||
pts = m_pts;
|
||||
g_pulseGetCachedEmpty.fetch_add(1, std::memory_order_relaxed);
|
||||
// Common case during network jitter — gated to avoid stderr spam.
|
||||
ANS_DBG("MEDIA_GetImage", "queue empty, returning cached");
|
||||
return cv::Mat();
|
||||
}
|
||||
|
||||
// getLatestFrame() clones the AVFrame — we own it from here
|
||||
frameToProcess = g_frameQueue.getLatestFrame();
|
||||
if (!frameToProcess) {
|
||||
width = m_currentImage.cols;
|
||||
height = m_currentImage.rows;
|
||||
pts = m_pts;
|
||||
g_pulseGetCachedEmpty.fetch_add(1, std::memory_order_relaxed);
|
||||
return cv::Mat();
|
||||
}
|
||||
}
|
||||
}
|
||||
// --- _mutex released here ---
|
||||
@@ -2964,7 +3253,8 @@ cv::Mat CVideoPlayer::getImage(int& width, int& height, int64_t& pts) {
|
||||
converted = avframeToCVMat(frameToProcess);
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
std::cerr << "Exception while converting AVFrame to cv::Mat: " << e.what() << std::endl;
|
||||
ANS_DBG("MEDIA_GetImage",
|
||||
"avframeToCVMat threw: %s — returning empty Mat", e.what());
|
||||
}
|
||||
const auto t2 = clk::now();
|
||||
|
||||
@@ -2974,9 +3264,20 @@ cv::Mat CVideoPlayer::getImage(int& width, int& height, int64_t& pts) {
|
||||
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||
|
||||
if (!converted.empty()) {
|
||||
// Skip-tracking: only meaningful in legacy mode where the
|
||||
// consumer reads the back of the queue. In jitter-buffer mode
|
||||
// we already counted drop-ahead discards; sequence numbers
|
||||
// don't track displayed-vs-skipped because pop is FIFO.
|
||||
if (m_jitterBufferMs == 0) {
|
||||
if (m_lastFrameSeq != 0 && currentSeq > m_lastFrameSeq + 1) {
|
||||
g_pulseSeqJump.fetch_add(currentSeq - m_lastFrameSeq - 1,
|
||||
std::memory_order_relaxed);
|
||||
}
|
||||
m_lastFrameSeq = currentSeq;
|
||||
}
|
||||
m_currentImage = converted;
|
||||
m_pts++;
|
||||
m_lastFrameSeq = currentSeq;
|
||||
g_pulseGetNew.fetch_add(1, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
// Preserve raw YUV/NV12 frame for GPU fast-path inference
|
||||
@@ -3076,11 +3377,11 @@ cv::Mat CVideoPlayer::getImage(int& width, int& height, int64_t& pts) {
|
||||
return result;
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
std::cerr << "Unexpected exception in getImage(): " << e.what() << std::endl;
|
||||
ANS_DBG("MEDIA_GetImage", "outer catch: %s", e.what());
|
||||
return cv::Mat(); // Return an empty cv::Mat if an exception occurs
|
||||
}
|
||||
catch (...) {
|
||||
std::cerr << "Unknown exception in getImage()" << std::endl;
|
||||
ANS_DBG("MEDIA_GetImage", "outer catch: unknown exception");
|
||||
return cv::Mat(); // Return an empty cv::Mat if an exception occurs
|
||||
|
||||
}
|
||||
@@ -3124,7 +3425,7 @@ std::string CVideoPlayer::getJpegImage(int& width, int& height, int64_t& pts) {
|
||||
}
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
std::cerr << "Exception while converting AVFrame to JPEG string: " << e.what() << std::endl;
|
||||
ANS_DBG("MEDIA_GetJpeg", "encode threw: %s — returning cached JPEG", e.what());
|
||||
av_frame_free(&frameToProcess);
|
||||
g_queueFrees.fetch_add(1, std::memory_order_relaxed);
|
||||
return m_lastJpegImage;
|
||||
@@ -3176,10 +3477,10 @@ std::string CVideoPlayer::getJpegImage(int& width, int& height, int64_t& pts) {
|
||||
return m_lastJpegImage;
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
std::cerr << "Unexpected exception in getJpegImage(): " << e.what() << std::endl;
|
||||
ANS_DBG("MEDIA_GetJpeg", "outer catch: %s", e.what());
|
||||
}
|
||||
catch (...) {
|
||||
std::cerr << "Unknown exception in getJpegImage()" << std::endl;
|
||||
ANS_DBG("MEDIA_GetJpeg", "outer catch: unknown exception");
|
||||
}
|
||||
|
||||
// If any exception occurs, return the last valid JPEG image
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
#include "media_util.h"
|
||||
#include <list>
|
||||
#include <string>
|
||||
#include <array>
|
||||
#include <opencv2/imgproc.hpp>
|
||||
#include <opencv2/highgui.hpp>
|
||||
#include <opencv2/opencv.hpp>
|
||||
@@ -51,6 +52,16 @@ private:
|
||||
uint64_t m_frameSeq = 0; // Sequence counter — increments on each new frame push
|
||||
|
||||
public:
|
||||
// Pulse counter for evicted-unread frames (instrumentation, not behavior).
|
||||
// Incremented every time a push has to drop the oldest frame because the
|
||||
// queue hit maxFrames AND the oldest never got read. With getLatestFrame()
|
||||
// the consumer reads the back, so anything pushed before the consumer's
|
||||
// last read sits in the middle/front and gets silently evicted here.
|
||||
static std::atomic<uint64_t>& evictedCounter() {
|
||||
static std::atomic<uint64_t> g{0};
|
||||
return g;
|
||||
}
|
||||
|
||||
// Push a new frame to the queue (removes oldest if full)
|
||||
void pushFrame(AVFrame* newFrame) {
|
||||
if (!newFrame) return;
|
||||
@@ -74,6 +85,7 @@ public:
|
||||
frameQueue.pop();
|
||||
av_frame_free(&oldFrame);
|
||||
g_queueFrees.fetch_add(1, std::memory_order_relaxed);
|
||||
evictedCounter().fetch_add(1, std::memory_order_relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -177,6 +189,16 @@ public:
|
||||
virtual void setImageQuality(int mode) { m_nImageQuality = mode; }
|
||||
void setTargetFPS(double intervalMs); // Set minimum interval between processed frames in ms (0 = no limit, 100 = ~10 FPS)
|
||||
double getLastFrameAgeMs(); // Milliseconds since last frame arrived from decoder (0 if no frame yet)
|
||||
// Jitter buffer: hold decoded frames for ~ms before delivering to consumer.
|
||||
// Smooths network/decoder jitter at the cost of added end-to-end latency.
|
||||
// 0 disables (today's getLatest semantics). Default 200 ms.
|
||||
// Adaptive: if the consumer is slower than the producer, drop-ahead kicks in
|
||||
// and bypasses the buffer (latency stays ~0 frames).
|
||||
// Note: applies only to the BGR cv::Mat returned by getImage(); the NV12/CUDA
|
||||
// fast-path frames (m_currentNV12Frame / m_currentCudaHWFrame) remain
|
||||
// updated at push time so inference always sees the latest.
|
||||
void setJitterBufferMs(int ms);
|
||||
int getJitterBufferMs() const { return m_jitterBufferMs; }
|
||||
virtual void setRtpMulticast(BOOL flag) {}
|
||||
virtual void setRtpOverUdp(BOOL flag) {}
|
||||
|
||||
@@ -317,6 +339,31 @@ protected:
|
||||
std::chrono::steady_clock::time_point m_lastDecoderFrameTime;
|
||||
bool m_lastDecoderFrameTimeSet = false;
|
||||
|
||||
// --- Jitter buffer (BGR delivery only) ---
|
||||
// Default 200 ms — absorbs typical network/decoder hiccups.
|
||||
int m_jitterBufferMs = 200;
|
||||
// Observed inter-arrival interval, ms. Seeds at 40 ms (=25 fps) and is
|
||||
// refined to a rolling median of the last m_arrivalCount samples.
|
||||
double m_observedFrameIntervalMs = 40.0;
|
||||
// Ring buffer of recent arrival ticks (steady_clock raw ticks). Used to
|
||||
// compute the rolling median frame interval. Sized at 32 — covers >1 s at
|
||||
// 30 fps so a single GOP-boundary jitter doesn't dominate.
|
||||
static constexpr size_t kArrivalRingSize = 32;
|
||||
std::array<long long, kArrivalRingSize> m_arrivalTicks{};
|
||||
size_t m_arrivalIdx = 0;
|
||||
size_t m_arrivalCount = 0;
|
||||
// Wall-clock of the last frame released to the consumer. Used for rate-paced
|
||||
// delivery — if the consumer polls faster than the producer rate, between
|
||||
// releases we return the cached frame so the buffer doesn't drain instantly.
|
||||
std::chrono::steady_clock::time_point m_lastReleaseTime;
|
||||
bool m_lastReleaseTimeSet = false;
|
||||
// Fill-phase: when true, hold releases until queue depth reaches target.
|
||||
// Set true at startup and again whenever the queue drains to empty mid-stream.
|
||||
// This is what makes the buffer actually buffer — without it, pacing alone
|
||||
// can't grow the queue from empty (producer and consumer would just match
|
||||
// rates with depth=0 forever).
|
||||
bool m_jitterBufferRefilling = true;
|
||||
|
||||
BOOL m_bPlaying;
|
||||
BOOL m_bPaused;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user