Add jitter buffer
This commit is contained in:
@@ -27,6 +27,8 @@ extern "C"
|
||||
#endif
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <array>
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <atomic>
|
||||
#include <libswscale/swscale.h>
|
||||
@@ -56,6 +58,112 @@ extern std::atomic<int64_t> g_contiguousAllocs;
|
||||
extern std::atomic<int64_t> g_contiguousFrees;
|
||||
extern std::atomic<int64_t> g_contiguousBytesInFlight;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Freeze diagnostics — once-per-second pulse showing both producer
|
||||
// (onVideoFrame) and consumer (getImage) accounting in a single line so a
|
||||
// 1-2 second freeze can be localized in DebugView. Filter on "MEDIA_Pulse".
|
||||
//
|
||||
// Reading the line during a freeze:
|
||||
// arrived high, pushed=0 ............ all frames dropped at producer
|
||||
// drop(err=N) UDP packet loss → SW decoder set decode_error_flags
|
||||
// drop(keyWait=N) waiting for first IDR after start/restart
|
||||
// drop(rate=N) SetTargetFPS throttle skipped them
|
||||
// arrived=0 .................. decoder silent (network/rx stall)
|
||||
// arrived≈pushed, gets high, new=0 ... consumer not seeing the new push
|
||||
// (mutex contention, seq stuck, ...)
|
||||
// arrived≈pushed, new high, smooth ..... pipeline healthy
|
||||
// ---------------------------------------------------------------------------
|
||||
namespace {
|
||||
std::atomic<uint64_t> g_pulseArrived{0}; // onVideoFrame entries
|
||||
std::atomic<uint64_t> g_pulsePushed{0}; // pushFrame() calls (frame survived all drop checks)
|
||||
std::atomic<uint64_t> g_pulseDropDecodeErr{0}; // decode_error_flags != 0 (now also pushed; counted for visibility)
|
||||
std::atomic<uint64_t> g_pulseDropKeyframeWait{0}; // m_bWaitingForKeyframe and not IDR
|
||||
std::atomic<uint64_t> g_pulseDropRateLimit{0}; // SetTargetFPS interval skip
|
||||
std::atomic<uint64_t> g_pulseGetCalls{0}; // getImage entries
|
||||
std::atomic<uint64_t> g_pulseGetNew{0}; // returned a brand-new frame
|
||||
std::atomic<uint64_t> g_pulseGetCachedFast{0}; // currentSeq == m_lastFrameSeq (no new push)
|
||||
std::atomic<uint64_t> g_pulseGetCachedEmpty{0}; // queue empty
|
||||
std::atomic<uint64_t> g_pulseGetCachedNotPlaying{0};// !m_bPlaying
|
||||
std::atomic<uint64_t> g_pulseGetCachedSettle{0}; // waiting for keyframe / settle period
|
||||
std::atomic<uint64_t> g_pulseSeqJump{0}; // total frames the consumer flew past via getLatestFrame
|
||||
// (sum of (currentSeq - m_lastFrameSeq - 1) across all reads)
|
||||
std::atomic<long long> g_pulseNextEmitTick{0}; // monotonic deadline (steady_clock::now ticks)
|
||||
|
||||
// Inter-arrival timing — exposes "21 frames bunched then 900ms silence" patterns
|
||||
// that aggregate counters hide. Both stored as steady_clock raw ticks.
|
||||
std::atomic<long long> g_pulseLastArrivalTick{0}; // tick of previous onVideoFrame entry (0 = none)
|
||||
std::atomic<long long> g_pulseMaxGapTicks{0}; // max gap seen in current 1s window
|
||||
|
||||
// Called from onVideoFrame on every arrival. Updates "max gap" if the gap
|
||||
// since previous arrival exceeds the current window's running max.
|
||||
inline void note_arrival_gap() {
|
||||
using clk = std::chrono::steady_clock;
|
||||
const long long now = clk::now().time_since_epoch().count();
|
||||
const long long prev = g_pulseLastArrivalTick.exchange(now, std::memory_order_relaxed);
|
||||
if (prev == 0) return; // first arrival ever — no gap
|
||||
const long long gap = now - prev;
|
||||
// Atomic max: only update if larger
|
||||
long long cur = g_pulseMaxGapTicks.load(std::memory_order_relaxed);
|
||||
while (gap > cur &&
|
||||
!g_pulseMaxGapTicks.compare_exchange_weak(cur, gap, std::memory_order_relaxed)) {
|
||||
// CAS retry until we win or someone else wrote a larger value
|
||||
}
|
||||
}
|
||||
|
||||
// Emit if 1s has elapsed since the previous emission. CAS makes only one
|
||||
// thread per second do the print; the rest just no-op.
|
||||
inline void emit_media_pulse_if_due(size_t queueDepth) {
|
||||
using clk = std::chrono::steady_clock;
|
||||
const long long tick = clk::now().time_since_epoch().count();
|
||||
long long expected = g_pulseNextEmitTick.load(std::memory_order_relaxed);
|
||||
if (tick < expected) return;
|
||||
const long long deadline = tick + std::chrono::duration_cast<clk::duration>(
|
||||
std::chrono::milliseconds(1000)).count();
|
||||
if (!g_pulseNextEmitTick.compare_exchange_strong(
|
||||
expected, deadline, std::memory_order_relaxed)) {
|
||||
return; // another thread won the window
|
||||
}
|
||||
// Snapshot + reset (delta semantics — every line is "events in last ~1s")
|
||||
const uint64_t arr = g_pulseArrived.exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t psh = g_pulsePushed.exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t dErr = g_pulseDropDecodeErr.exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t dKey = g_pulseDropKeyframeWait.exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t dRat = g_pulseDropRateLimit.exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t gtl = g_pulseGetCalls.exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t gNew = g_pulseGetNew.exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t gFst = g_pulseGetCachedFast.exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t gEmt = g_pulseGetCachedEmpty.exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t gNp = g_pulseGetCachedNotPlaying.exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t gStl = g_pulseGetCachedSettle.exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t evct = FrameQueue::evictedCounter().exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t skip = g_pulseSeqJump.exchange(0, std::memory_order_relaxed);
|
||||
const long long maxGapTicks = g_pulseMaxGapTicks.exchange(0, std::memory_order_relaxed);
|
||||
const double maxGapMs = std::chrono::duration<double, std::milli>(
|
||||
clk::duration(maxGapTicks)).count();
|
||||
// Production: route through ANS_DBG so the pulse is gated by the
|
||||
// ANSCORE_DEBUGVIEW runtime toggle (env var ANSCENTER_DBGVIEW=1 or
|
||||
// sentinel file C:\ProgramData\ANSCENTER\ansvisdebugview.txt).
|
||||
// Silent by default; one-line snapshot per second when enabled.
|
||||
//
|
||||
// maxGap is the largest interval between two consecutive arrivals in
|
||||
// this second. Healthy ~25fps streams should sit at maxGap≈40-60ms.
|
||||
// A spike to 1000+ ms is the smoking gun for a producer-side stall:
|
||||
// RTP rx hung, decoder waiting on references, etc.
|
||||
ANS_DBG("MEDIA_Pulse",
|
||||
"[1s] producer arrived=%llu pushed=%llu drop(err=%llu keyWait=%llu rate=%llu) "
|
||||
"maxGap=%.0fms evicted=%llu | "
|
||||
"consumer gets=%llu new=%llu skipped=%llu cached(fast=%llu empty=%llu notPlay=%llu settle=%llu) "
|
||||
"| queueDepth=%zu",
|
||||
(unsigned long long)arr, (unsigned long long)psh,
|
||||
(unsigned long long)dErr, (unsigned long long)dKey, (unsigned long long)dRat,
|
||||
maxGapMs, (unsigned long long)evct,
|
||||
(unsigned long long)gtl, (unsigned long long)gNew, (unsigned long long)skip,
|
||||
(unsigned long long)gFst, (unsigned long long)gEmt,
|
||||
(unsigned long long)gNp, (unsigned long long)gStl,
|
||||
queueDepth);
|
||||
}
|
||||
} // anonymous namespace
|
||||
|
||||
// libyuv: SIMD-accelerated YUV↔RGB conversion with native strided-plane input.
|
||||
// Replaces the memcpy-into-staging + cv::cvtColor(COLOR_YUV2BGR_I420) chain
|
||||
// in avframeYUV420PToCvMat with a direct I420→RGB24 (== OpenCV BGR memory
|
||||
@@ -1920,14 +2028,13 @@ cv::Mat CVideoPlayer::avframeToCVMat(const AVFrame* pFrame) {
|
||||
return cv::Mat();
|
||||
}
|
||||
|
||||
// One-shot diagnostic: print the pixel format the first time through so
|
||||
// we can see which branch of the switch below is taken. Remove after use.
|
||||
// One-shot diagnostic: log the pixel format the first time through so
|
||||
// we can see which branch of the switch below is taken. Production: only
|
||||
// emitted when ANSCORE_DEBUGVIEW is enabled.
|
||||
static bool s_loggedFmt = false;
|
||||
if (!s_loggedFmt) {
|
||||
s_loggedFmt = true;
|
||||
const char* name = av_get_pix_fmt_name((AVPixelFormat)pFrame->format);
|
||||
fprintf(stderr, "[avframeToCVMat] first frame format=%d (%s) %dx%d\n",
|
||||
pFrame->format, name ? name : "?", pFrame->width, pFrame->height);
|
||||
ANS_DBG("MEDIA_Convert",
|
||||
"avframeToCVMat FIRST-FRAME fmt=%d(%s) %dx%d HWDecoding=%d (this=%p)",
|
||||
pFrame->format, name ? name : "?",
|
||||
@@ -2465,6 +2572,12 @@ void CVideoPlayer::StartVideoDecoder() {
|
||||
m_lastFrameSeq = 0;
|
||||
m_bWaitingForKeyframe = true; // Skip frames until first keyframe
|
||||
m_cleanFrameCount = 0; // Reset settle counter
|
||||
// Reset jitter buffer state so a restart starts measuring rate fresh
|
||||
m_arrivalIdx = 0;
|
||||
m_arrivalCount = 0;
|
||||
m_observedFrameIntervalMs = 40.0; // re-seed at 25 fps
|
||||
m_lastReleaseTimeSet = false;
|
||||
m_jitterBufferRefilling = true; // new fill phase on (re)start
|
||||
if (m_pVideoDecoder)
|
||||
{
|
||||
m_pVideoDecoder->Start();
|
||||
@@ -2595,6 +2708,17 @@ void CVideoPlayer::setTargetFPS(double intervalMs)
|
||||
m_targetIntervalMs = intervalMs;
|
||||
m_targetFPSInitialized = false; // reset timing on change
|
||||
}
|
||||
void CVideoPlayer::setJitterBufferMs(int ms)
|
||||
{
|
||||
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||
if (ms < 0) ms = 0;
|
||||
if (ms > 2000) ms = 2000; // sanity ceiling — beyond this is just lag
|
||||
m_jitterBufferMs = ms;
|
||||
// Reset release timing and re-enter fill phase so the new setting takes
|
||||
// effect cleanly on the next call.
|
||||
m_lastReleaseTimeSet = false;
|
||||
m_jitterBufferRefilling = (ms > 0);
|
||||
}
|
||||
double CVideoPlayer::getLastFrameAgeMs()
|
||||
{
|
||||
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||
@@ -2802,10 +2926,60 @@ void CVideoPlayer::onVideoFrame(AVFrame* frame)
|
||||
}
|
||||
|
||||
if (m_bPlaying && m_videoPlayFlag) {
|
||||
// Drop any frame with decode errors (corrupted reference frames, etc.)
|
||||
// MEDIA_Pulse: count every frame the decoder hands us, before any drop checks.
|
||||
g_pulseArrived.fetch_add(1, std::memory_order_relaxed);
|
||||
note_arrival_gap(); // tracks max gap between consecutive arrivals in this 1s window
|
||||
|
||||
// --- Arrival-rate measurement for the jitter buffer ---
|
||||
// Record this arrival in the ring buffer, then update the observed
|
||||
// inter-frame interval from the median delta of the last samples.
|
||||
// Median is robust to bursts (TCP unstall delivers several frames in
|
||||
// rapid succession; mean would be skewed, median picks up the typical
|
||||
// in-burst spacing). Adapts within ~1 s when camera rate changes.
|
||||
{
|
||||
using clk = std::chrono::steady_clock;
|
||||
const long long now = clk::now().time_since_epoch().count();
|
||||
m_arrivalTicks[m_arrivalIdx % kArrivalRingSize] = now;
|
||||
m_arrivalIdx++;
|
||||
if (m_arrivalCount < kArrivalRingSize) m_arrivalCount++;
|
||||
// Recompute every 8 frames (cheap; ring of 32 ints).
|
||||
if (m_arrivalCount >= 4 && (m_arrivalIdx % 8) == 0) {
|
||||
const size_t n = m_arrivalCount;
|
||||
std::array<long long, kArrivalRingSize> sortedCopy = m_arrivalTicks;
|
||||
std::sort(sortedCopy.begin(), sortedCopy.begin() + n);
|
||||
std::array<long long, kArrivalRingSize - 1> deltas{};
|
||||
for (size_t i = 1; i < n; ++i) deltas[i - 1] = sortedCopy[i] - sortedCopy[i - 1];
|
||||
// Median delta
|
||||
std::sort(deltas.begin(), deltas.begin() + (n - 1));
|
||||
const long long medianTicks = deltas[(n - 1) / 2];
|
||||
const double medianMs = std::chrono::duration<double, std::milli>(
|
||||
clk::duration(medianTicks)).count();
|
||||
// Clamp to sane camera framerate range (5–120 fps) so a
|
||||
// degenerate measurement can't poison the buffer logic.
|
||||
if (medianMs >= 8.0 && medianMs <= 200.0) {
|
||||
m_observedFrameIntervalMs = medianMs;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Pass frames with decode_error_flags through. The codec is configured
|
||||
// with AV_CODEC_FLAG_OUTPUT_CORRUPT + AV_EF_IGNORE_ERR (video_decoder.cpp)
|
||||
// so it has already applied concealment for missing references — the
|
||||
// frame data is usable even if the flag is set.
|
||||
//
|
||||
// Why this matters: HW decoders (NVDEC/D3D11VA/DXVA2) silently conceal
|
||||
// without setting the flag, but the SW decoder (default since commit
|
||||
// e134ebd) sets the flag on every UDP packet loss. Dropping flagged
|
||||
// frames meant every lost packet froze the display until the next IDR
|
||||
// (~1-2s typical), reproducing the periodic freeze the user saw under
|
||||
// UDP+SW. VLC plays through these frames — so do we now.
|
||||
if (frame->decode_error_flags != 0) {
|
||||
fprintf(stderr, "[HWDecode] Dropping frame with decode errors (flags=0x%x)\n", frame->decode_error_flags);
|
||||
return;
|
||||
g_pulseDropDecodeErr.fetch_add(1, std::memory_order_relaxed); // counted, not dropped
|
||||
ANS_DBG("MEDIA_DecodeErr",
|
||||
"frame had decode_error_flags=0x%x — passing through with concealment "
|
||||
"(pict_type=%d key=%d)",
|
||||
frame->decode_error_flags,
|
||||
(int)frame->pict_type, (int)frame->key_frame);
|
||||
}
|
||||
|
||||
// After start/restart, skip corrupted frames until first keyframe (IDR) arrives.
|
||||
@@ -2815,8 +2989,11 @@ void CVideoPlayer::onVideoFrame(AVFrame* frame)
|
||||
if (frame->key_frame || frame->pict_type == AV_PICTURE_TYPE_I) {
|
||||
m_bWaitingForKeyframe = false;
|
||||
m_cleanFrameCount = 0;
|
||||
fprintf(stderr, "[HWDecode] First keyframe received, settling for %d frames\n", SETTLE_FRAME_COUNT);
|
||||
ANS_DBG("MEDIA_Lifecycle",
|
||||
"first keyframe received, settling for %d frames", SETTLE_FRAME_COUNT);
|
||||
} else {
|
||||
g_pulseDropKeyframeWait.fetch_add(1, std::memory_order_relaxed);
|
||||
emit_media_pulse_if_due(g_frameQueue.size());
|
||||
return; // Drop this frame — not yet safe to decode
|
||||
}
|
||||
}
|
||||
@@ -2838,6 +3015,8 @@ void CVideoPlayer::onVideoFrame(AVFrame* frame)
|
||||
} else {
|
||||
auto elapsed = std::chrono::duration<double, std::milli>(now - m_lastProcessedTime).count();
|
||||
if (elapsed < m_targetIntervalMs) {
|
||||
g_pulseDropRateLimit.fetch_add(1, std::memory_order_relaxed);
|
||||
emit_media_pulse_if_due(g_frameQueue.size());
|
||||
return; // Skip this frame — too soon
|
||||
}
|
||||
}
|
||||
@@ -2848,6 +3027,7 @@ void CVideoPlayer::onVideoFrame(AVFrame* frame)
|
||||
// Push frame to queue; during settle period getImage() will ignore the queue
|
||||
// and keep returning the last good cached image
|
||||
g_frameQueue.pushFrame(frame); // pushFrame() clones the frame internally
|
||||
g_pulsePushed.fetch_add(1, std::memory_order_relaxed);
|
||||
|
||||
// Capture CUDA HW frame for zero-copy inference.
|
||||
// We're inside decode()'s lock scope (decoder._mutex held) AND onVideoFrame
|
||||
@@ -2867,9 +3047,11 @@ void CVideoPlayer::onVideoFrame(AVFrame* frame)
|
||||
if (m_cleanFrameCount < SETTLE_FRAME_COUNT) {
|
||||
m_cleanFrameCount++;
|
||||
if (m_cleanFrameCount == SETTLE_FRAME_COUNT) {
|
||||
fprintf(stderr, "[HWDecode] Settle complete, delivering new frames\n");
|
||||
ANS_DBG("MEDIA_Lifecycle", "settle complete, delivering new frames");
|
||||
}
|
||||
}
|
||||
|
||||
emit_media_pulse_if_due(g_frameQueue.size());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2901,13 +3083,20 @@ cv::Mat CVideoPlayer::getImage(int& width, int& height, int64_t& pts) {
|
||||
AVFrame* frameToProcess = nullptr;
|
||||
uint64_t currentSeq = 0;
|
||||
|
||||
// MEDIA_Pulse: count every getImage call for the per-second pulse,
|
||||
// and try to emit on entry so the line still prints during a freeze
|
||||
// where every call lands on a cached-return path. The once-per-second
|
||||
// CAS inside makes the actual cost trivial.
|
||||
g_pulseGetCalls.fetch_add(1, std::memory_order_relaxed);
|
||||
emit_media_pulse_if_due(g_frameQueue.size());
|
||||
|
||||
// Timing breakdown — gated by ANSCORE_DEBUGVIEW (zero overhead in production).
|
||||
// t0 = entry, t1 = after pulling frame from queue, t2 = after YUV->BGR,
|
||||
// t3 = after publish. Throttled to every 30 full-path calls (~1/sec @30fps).
|
||||
using clk = std::chrono::steady_clock;
|
||||
const auto t0 = clk::now();
|
||||
|
||||
// --- Phase 1: short locked section — examine state, pull latest frame ---
|
||||
// --- Phase 1: short locked section — examine state, pull frame ---
|
||||
{
|
||||
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||
|
||||
@@ -2915,6 +3104,7 @@ cv::Mat CVideoPlayer::getImage(int& width, int& height, int64_t& pts) {
|
||||
width = m_currentImage.cols;
|
||||
height = m_currentImage.rows;
|
||||
pts = m_pts;
|
||||
g_pulseGetCachedNotPlaying.fetch_add(1, std::memory_order_relaxed);
|
||||
return m_currentImage; // Shallow copy (reference counted)
|
||||
}
|
||||
|
||||
@@ -2924,33 +3114,132 @@ cv::Mat CVideoPlayer::getImage(int& width, int& height, int64_t& pts) {
|
||||
width = m_currentImage.cols;
|
||||
height = m_currentImage.rows;
|
||||
pts = m_pts;
|
||||
g_pulseGetCachedSettle.fetch_add(1, std::memory_order_relaxed);
|
||||
return m_currentImage;
|
||||
}
|
||||
|
||||
// Fast path: same frame as last call — skip clone + BGR conversion
|
||||
currentSeq = g_frameQueue.getSequence();
|
||||
if (currentSeq == m_lastFrameSeq && !m_currentImage.empty()) {
|
||||
width = m_currentImage.cols;
|
||||
height = m_currentImage.rows;
|
||||
pts = m_pts;
|
||||
return m_currentImage;
|
||||
}
|
||||
|
||||
if (g_frameQueue.isEmpty()) {
|
||||
width = m_currentImage.cols;
|
||||
height = m_currentImage.rows;
|
||||
pts = m_pts;
|
||||
std::cerr << "No frame available in getImage()" << std::endl;
|
||||
return cv::Mat();
|
||||
}
|
||||
if (m_jitterBufferMs > 0 && m_observedFrameIntervalMs > 0.0) {
|
||||
// =========================================================
|
||||
// Jitter-buffer mode: FIFO + rate-paced + adaptive depth
|
||||
// + fill-phase refill
|
||||
// =========================================================
|
||||
const double frameMs = m_observedFrameIntervalMs;
|
||||
const size_t targetN = std::max<size_t>(1,
|
||||
static_cast<size_t>(static_cast<double>(m_jitterBufferMs) / frameMs));
|
||||
const size_t maxAllowed = 2 * targetN;
|
||||
const size_t qSize = g_frameQueue.size();
|
||||
|
||||
// getLatestFrame() clones the AVFrame — we own it from here
|
||||
frameToProcess = g_frameQueue.getLatestFrame();
|
||||
if (!frameToProcess) {
|
||||
width = m_currentImage.cols;
|
||||
height = m_currentImage.rows;
|
||||
pts = m_pts;
|
||||
return cv::Mat();
|
||||
// (1) Adaptive drop-ahead: if the consumer is slower than the
|
||||
// producer, the queue overshoots target. Pop & discard
|
||||
// down to targetN so latency stays bounded. Each discard
|
||||
// counts as a "skipped" frame in the pulse.
|
||||
if (qSize > maxAllowed) {
|
||||
while (g_frameQueue.size() > targetN) {
|
||||
AVFrame* old = g_frameQueue.popFrame();
|
||||
if (old) {
|
||||
av_frame_free(&old);
|
||||
g_queueFrees.fetch_add(1, std::memory_order_relaxed);
|
||||
g_pulseSeqJump.fetch_add(1, std::memory_order_relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// (2) Fill phase: hold all releases until the queue reaches
|
||||
// target depth. Triggered at startup and again whenever
|
||||
// the queue drains to empty mid-stream. Without this,
|
||||
// 1.0x pacing keeps the queue at whatever depth it
|
||||
// happens to be (zero, after a drain) — the buffer
|
||||
// never refills.
|
||||
if (m_jitterBufferRefilling) {
|
||||
if (g_frameQueue.size() >= targetN) {
|
||||
m_jitterBufferRefilling = false;
|
||||
} else {
|
||||
width = m_currentImage.cols;
|
||||
height = m_currentImage.rows;
|
||||
pts = m_pts;
|
||||
g_pulseGetCachedFast.fetch_add(1, std::memory_order_relaxed);
|
||||
return m_currentImage;
|
||||
}
|
||||
}
|
||||
|
||||
// (3) Rate-paced release at 1.0x of observed producer interval.
|
||||
// Anything <1.0 leaks the buffer (consumer faster than
|
||||
// producer); >1.0 keeps growing the queue until drop-ahead
|
||||
// trips. 1.0 keeps the queue at its current depth in
|
||||
// steady state.
|
||||
if (m_lastReleaseTimeSet) {
|
||||
const double sinceLastMs = std::chrono::duration<double, std::milli>(
|
||||
t0 - m_lastReleaseTime).count();
|
||||
if (sinceLastMs < frameMs) {
|
||||
width = m_currentImage.cols;
|
||||
height = m_currentImage.rows;
|
||||
pts = m_pts;
|
||||
g_pulseGetCachedFast.fetch_add(1, std::memory_order_relaxed);
|
||||
return m_currentImage;
|
||||
}
|
||||
}
|
||||
|
||||
// (4) Empty queue (network gap drained the buffer): re-enter
|
||||
// fill phase so the next producer pushes are held back to
|
||||
// rebuild the buffer to target depth before the next
|
||||
// release. Without this, a one-time gap permanently
|
||||
// shrinks the buffer and we lose absorption capacity.
|
||||
if (g_frameQueue.isEmpty()) {
|
||||
m_jitterBufferRefilling = true;
|
||||
width = m_currentImage.cols;
|
||||
height = m_currentImage.rows;
|
||||
pts = m_pts;
|
||||
g_pulseGetCachedEmpty.fetch_add(1, std::memory_order_relaxed);
|
||||
return cv::Mat();
|
||||
}
|
||||
|
||||
// (5) FIFO pop — the frame is now ours; we must free it after
|
||||
// conversion (popFrame returns the original, not a clone).
|
||||
frameToProcess = g_frameQueue.popFrame();
|
||||
if (!frameToProcess) {
|
||||
m_jitterBufferRefilling = true;
|
||||
width = m_currentImage.cols;
|
||||
height = m_currentImage.rows;
|
||||
pts = m_pts;
|
||||
g_pulseGetCachedEmpty.fetch_add(1, std::memory_order_relaxed);
|
||||
return cv::Mat();
|
||||
}
|
||||
m_lastReleaseTime = t0;
|
||||
m_lastReleaseTimeSet = true;
|
||||
} else {
|
||||
// =========================================================
|
||||
// Legacy mode (m_jitterBufferMs == 0): getLatest semantics
|
||||
// =========================================================
|
||||
// Fast path: same frame as last call — skip clone + BGR conversion
|
||||
if (currentSeq == m_lastFrameSeq && !m_currentImage.empty()) {
|
||||
width = m_currentImage.cols;
|
||||
height = m_currentImage.rows;
|
||||
pts = m_pts;
|
||||
g_pulseGetCachedFast.fetch_add(1, std::memory_order_relaxed);
|
||||
return m_currentImage;
|
||||
}
|
||||
|
||||
if (g_frameQueue.isEmpty()) {
|
||||
width = m_currentImage.cols;
|
||||
height = m_currentImage.rows;
|
||||
pts = m_pts;
|
||||
g_pulseGetCachedEmpty.fetch_add(1, std::memory_order_relaxed);
|
||||
// Common case during network jitter — gated to avoid stderr spam.
|
||||
ANS_DBG("MEDIA_GetImage", "queue empty, returning cached");
|
||||
return cv::Mat();
|
||||
}
|
||||
|
||||
// getLatestFrame() clones the AVFrame — we own it from here
|
||||
frameToProcess = g_frameQueue.getLatestFrame();
|
||||
if (!frameToProcess) {
|
||||
width = m_currentImage.cols;
|
||||
height = m_currentImage.rows;
|
||||
pts = m_pts;
|
||||
g_pulseGetCachedEmpty.fetch_add(1, std::memory_order_relaxed);
|
||||
return cv::Mat();
|
||||
}
|
||||
}
|
||||
}
|
||||
// --- _mutex released here ---
|
||||
@@ -2964,7 +3253,8 @@ cv::Mat CVideoPlayer::getImage(int& width, int& height, int64_t& pts) {
|
||||
converted = avframeToCVMat(frameToProcess);
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
std::cerr << "Exception while converting AVFrame to cv::Mat: " << e.what() << std::endl;
|
||||
ANS_DBG("MEDIA_GetImage",
|
||||
"avframeToCVMat threw: %s — returning empty Mat", e.what());
|
||||
}
|
||||
const auto t2 = clk::now();
|
||||
|
||||
@@ -2974,9 +3264,20 @@ cv::Mat CVideoPlayer::getImage(int& width, int& height, int64_t& pts) {
|
||||
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||
|
||||
if (!converted.empty()) {
|
||||
// Skip-tracking: only meaningful in legacy mode where the
|
||||
// consumer reads the back of the queue. In jitter-buffer mode
|
||||
// we already counted drop-ahead discards; sequence numbers
|
||||
// don't track displayed-vs-skipped because pop is FIFO.
|
||||
if (m_jitterBufferMs == 0) {
|
||||
if (m_lastFrameSeq != 0 && currentSeq > m_lastFrameSeq + 1) {
|
||||
g_pulseSeqJump.fetch_add(currentSeq - m_lastFrameSeq - 1,
|
||||
std::memory_order_relaxed);
|
||||
}
|
||||
m_lastFrameSeq = currentSeq;
|
||||
}
|
||||
m_currentImage = converted;
|
||||
m_pts++;
|
||||
m_lastFrameSeq = currentSeq;
|
||||
g_pulseGetNew.fetch_add(1, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
// Preserve raw YUV/NV12 frame for GPU fast-path inference
|
||||
@@ -3076,11 +3377,11 @@ cv::Mat CVideoPlayer::getImage(int& width, int& height, int64_t& pts) {
|
||||
return result;
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
std::cerr << "Unexpected exception in getImage(): " << e.what() << std::endl;
|
||||
ANS_DBG("MEDIA_GetImage", "outer catch: %s", e.what());
|
||||
return cv::Mat(); // Return an empty cv::Mat if an exception occurs
|
||||
}
|
||||
catch (...) {
|
||||
std::cerr << "Unknown exception in getImage()" << std::endl;
|
||||
ANS_DBG("MEDIA_GetImage", "outer catch: unknown exception");
|
||||
return cv::Mat(); // Return an empty cv::Mat if an exception occurs
|
||||
|
||||
}
|
||||
@@ -3124,7 +3425,7 @@ std::string CVideoPlayer::getJpegImage(int& width, int& height, int64_t& pts) {
|
||||
}
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
std::cerr << "Exception while converting AVFrame to JPEG string: " << e.what() << std::endl;
|
||||
ANS_DBG("MEDIA_GetJpeg", "encode threw: %s — returning cached JPEG", e.what());
|
||||
av_frame_free(&frameToProcess);
|
||||
g_queueFrees.fetch_add(1, std::memory_order_relaxed);
|
||||
return m_lastJpegImage;
|
||||
@@ -3176,10 +3477,10 @@ std::string CVideoPlayer::getJpegImage(int& width, int& height, int64_t& pts) {
|
||||
return m_lastJpegImage;
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
std::cerr << "Unexpected exception in getJpegImage(): " << e.what() << std::endl;
|
||||
ANS_DBG("MEDIA_GetJpeg", "outer catch: %s", e.what());
|
||||
}
|
||||
catch (...) {
|
||||
std::cerr << "Unknown exception in getJpegImage()" << std::endl;
|
||||
ANS_DBG("MEDIA_GetJpeg", "outer catch: unknown exception");
|
||||
}
|
||||
|
||||
// If any exception occurs, return the last valid JPEG image
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
#include "media_util.h"
|
||||
#include <list>
|
||||
#include <string>
|
||||
#include <array>
|
||||
#include <opencv2/imgproc.hpp>
|
||||
#include <opencv2/highgui.hpp>
|
||||
#include <opencv2/opencv.hpp>
|
||||
@@ -51,6 +52,16 @@ private:
|
||||
uint64_t m_frameSeq = 0; // Sequence counter — increments on each new frame push
|
||||
|
||||
public:
|
||||
// Pulse counter for evicted-unread frames (instrumentation, not behavior).
|
||||
// Incremented every time a push has to drop the oldest frame because the
|
||||
// queue hit maxFrames AND the oldest never got read. With getLatestFrame()
|
||||
// the consumer reads the back, so anything pushed before the consumer's
|
||||
// last read sits in the middle/front and gets silently evicted here.
|
||||
static std::atomic<uint64_t>& evictedCounter() {
|
||||
static std::atomic<uint64_t> g{0};
|
||||
return g;
|
||||
}
|
||||
|
||||
// Push a new frame to the queue (removes oldest if full)
|
||||
void pushFrame(AVFrame* newFrame) {
|
||||
if (!newFrame) return;
|
||||
@@ -74,6 +85,7 @@ public:
|
||||
frameQueue.pop();
|
||||
av_frame_free(&oldFrame);
|
||||
g_queueFrees.fetch_add(1, std::memory_order_relaxed);
|
||||
evictedCounter().fetch_add(1, std::memory_order_relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -177,6 +189,16 @@ public:
|
||||
virtual void setImageQuality(int mode) { m_nImageQuality = mode; }
|
||||
void setTargetFPS(double intervalMs); // Set minimum interval between processed frames in ms (0 = no limit, 100 = ~10 FPS)
|
||||
double getLastFrameAgeMs(); // Milliseconds since last frame arrived from decoder (0 if no frame yet)
|
||||
// Jitter buffer: hold decoded frames for ~ms before delivering to consumer.
|
||||
// Smooths network/decoder jitter at the cost of added end-to-end latency.
|
||||
// 0 disables (today's getLatest semantics). Default 200 ms.
|
||||
// Adaptive: if the consumer is slower than the producer, drop-ahead kicks in
|
||||
// and bypasses the buffer (latency stays ~0 frames).
|
||||
// Note: applies only to the BGR cv::Mat returned by getImage(); the NV12/CUDA
|
||||
// fast-path frames (m_currentNV12Frame / m_currentCudaHWFrame) remain
|
||||
// updated at push time so inference always sees the latest.
|
||||
void setJitterBufferMs(int ms);
|
||||
int getJitterBufferMs() const { return m_jitterBufferMs; }
|
||||
virtual void setRtpMulticast(BOOL flag) {}
|
||||
virtual void setRtpOverUdp(BOOL flag) {}
|
||||
|
||||
@@ -317,6 +339,31 @@ protected:
|
||||
std::chrono::steady_clock::time_point m_lastDecoderFrameTime;
|
||||
bool m_lastDecoderFrameTimeSet = false;
|
||||
|
||||
// --- Jitter buffer (BGR delivery only) ---
|
||||
// Default 200 ms — absorbs typical network/decoder hiccups.
|
||||
int m_jitterBufferMs = 200;
|
||||
// Observed inter-arrival interval, ms. Seeds at 40 ms (=25 fps) and is
|
||||
// refined to a rolling median of the last m_arrivalCount samples.
|
||||
double m_observedFrameIntervalMs = 40.0;
|
||||
// Ring buffer of recent arrival ticks (steady_clock raw ticks). Used to
|
||||
// compute the rolling median frame interval. Sized at 32 — covers >1 s at
|
||||
// 30 fps so a single GOP-boundary jitter doesn't dominate.
|
||||
static constexpr size_t kArrivalRingSize = 32;
|
||||
std::array<long long, kArrivalRingSize> m_arrivalTicks{};
|
||||
size_t m_arrivalIdx = 0;
|
||||
size_t m_arrivalCount = 0;
|
||||
// Wall-clock of the last frame released to the consumer. Used for rate-paced
|
||||
// delivery — if the consumer polls faster than the producer rate, between
|
||||
// releases we return the cached frame so the buffer doesn't drain instantly.
|
||||
std::chrono::steady_clock::time_point m_lastReleaseTime;
|
||||
bool m_lastReleaseTimeSet = false;
|
||||
// Fill-phase: when true, hold releases until queue depth reaches target.
|
||||
// Set true at startup and again whenever the queue drains to empty mid-stream.
|
||||
// This is what makes the buffer actually buffer — without it, pacing alone
|
||||
// can't grow the queue from empty (producer and consumer would just match
|
||||
// rates with depth=0 forever).
|
||||
bool m_jitterBufferRefilling = true;
|
||||
|
||||
BOOL m_bPlaying;
|
||||
BOOL m_bPaused;
|
||||
|
||||
|
||||
+107
-15
@@ -28,6 +28,51 @@ extern "C"
|
||||
// Note: per-instance thread safety is handled by ANSRTSPClient::_mutex
|
||||
// Mat registry thread safety is handled by anscv_mat_replace's internal registry_mutex
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Freeze diagnostics — once-per-second pulse for ANSRTSPClient::GetImage so
|
||||
// we can see *which* return path fired during a 1-2s freeze. Filter on
|
||||
// "RTSP_Pulse" in DebugView. Pairs with the producer/consumer pulse in
|
||||
// video_player.cpp ("MEDIA_Pulse"): together they let you localize a stall
|
||||
// to producer (decoder) vs internal player vs ANSRTSP wrapper.
|
||||
// ---------------------------------------------------------------------------
|
||||
namespace {
|
||||
std::atomic<uint64_t> g_rtspPulseGetCalls{0}; // entries to ANSRTSPClient::GetImage
|
||||
std::atomic<uint64_t> g_rtspPulseNotPlaying{0}; // !_isPlaying
|
||||
std::atomic<uint64_t> g_rtspPulseEarlyStale{0}; // ageMs >= 5000 → cached early
|
||||
std::atomic<uint64_t> g_rtspPulseInnerEmpty{0}; // _playerClient->getImage returned empty
|
||||
std::atomic<uint64_t> g_rtspPulseDupPts{0}; // currentPts == _pts → cached
|
||||
std::atomic<uint64_t> g_rtspPulseNewFrame{0}; // brand-new frame returned
|
||||
std::atomic<long long> g_rtspPulseNextEmitTick{0};
|
||||
|
||||
inline void emit_rtsp_pulse_if_due(double ageMsLastFrame) {
|
||||
using clk = std::chrono::steady_clock;
|
||||
const long long tick = clk::now().time_since_epoch().count();
|
||||
long long expected = g_rtspPulseNextEmitTick.load(std::memory_order_relaxed);
|
||||
if (tick < expected) return;
|
||||
const long long deadline = tick + std::chrono::duration_cast<clk::duration>(
|
||||
std::chrono::milliseconds(1000)).count();
|
||||
if (!g_rtspPulseNextEmitTick.compare_exchange_strong(
|
||||
expected, deadline, std::memory_order_relaxed)) {
|
||||
return;
|
||||
}
|
||||
const uint64_t calls = g_rtspPulseGetCalls.exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t np = g_rtspPulseNotPlaying.exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t es = g_rtspPulseEarlyStale.exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t ie = g_rtspPulseInnerEmpty.exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t dp = g_rtspPulseDupPts.exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t nf = g_rtspPulseNewFrame.exchange(0, std::memory_order_relaxed);
|
||||
// Production: route through ANS_DBG so the pulse is gated by the
|
||||
// ANSCORE_DEBUGVIEW runtime toggle. Silent by default.
|
||||
ANS_DBG("RTSP_Pulse",
|
||||
"[1s] GetImage calls=%llu new=%llu cached(notPlay=%llu earlyStale=%llu "
|
||||
"innerEmpty=%llu dupPts=%llu) lastFrameAge=%.0fms",
|
||||
(unsigned long long)calls, (unsigned long long)nf,
|
||||
(unsigned long long)np, (unsigned long long)es,
|
||||
(unsigned long long)ie, (unsigned long long)dp,
|
||||
ageMsLastFrame);
|
||||
}
|
||||
} // anonymous namespace
|
||||
|
||||
// Debug logging. Define ANSCORE_GPU_DEBUG=1 to enable verbose per-frame logging.
|
||||
#ifndef RTSP_DBG
|
||||
#if defined(ANSCORE_GPU_DEBUG) && ANSCORE_GPU_DEBUG
|
||||
@@ -450,11 +495,17 @@ namespace ANSCENTER {
|
||||
cv::Mat ANSRTSPClient::GetImage(int& width, int& height, int64_t& pts) {
|
||||
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||
|
||||
// RTSP_Pulse: count entry, attempt once-per-second emit on every call so
|
||||
// the pulse keeps printing even when every path returns cached.
|
||||
g_rtspPulseGetCalls.fetch_add(1, std::memory_order_relaxed);
|
||||
emit_rtsp_pulse_if_due(_playerClient ? _playerClient->getLastFrameAgeMs() : -1.0);
|
||||
|
||||
// Return last known frame if not playing
|
||||
if (!_isPlaying) {
|
||||
width = _imageWidth;
|
||||
height = _imageHeight;
|
||||
pts = _pts;
|
||||
g_rtspPulseNotPlaying.fetch_add(1, std::memory_order_relaxed);
|
||||
return _pLastFrame; // Shallow copy (fast)
|
||||
}
|
||||
|
||||
@@ -473,6 +524,7 @@ namespace ANSCENTER {
|
||||
width = _imageWidth;
|
||||
height = _imageHeight;
|
||||
pts = _pts;
|
||||
g_rtspPulseEarlyStale.fetch_add(1, std::memory_order_relaxed);
|
||||
return _pLastFrame;
|
||||
}
|
||||
}
|
||||
@@ -488,36 +540,40 @@ namespace ANSCENTER {
|
||||
width = _imageWidth;
|
||||
height = _imageHeight;
|
||||
pts = _pts;
|
||||
g_rtspPulseInnerEmpty.fetch_add(1, std::memory_order_relaxed);
|
||||
return _pLastFrame;
|
||||
}
|
||||
|
||||
// Use PTS to detect duplicate frames (much faster than pixel comparison)
|
||||
// The sequence-based check in getImage() already skips conversion for same frames,
|
||||
// so if PTS hasn't changed, it's the same frame — no need for expensive memcmp
|
||||
if (currentPts == _pts && !_pLastFrame.empty()) {
|
||||
// Duplicate-frame detection. We track the inner player's currentPts
|
||||
// (in _lastInnerPts) separately from our own _pts. This is important:
|
||||
// _pts gets synthetically advanced on every duplicate so LabVIEW's
|
||||
// staleness check sees a changing timestamp. If we compared currentPts
|
||||
// against _pts, the synthetic advance would make the comparison fail
|
||||
// on the next call and route the same cached frame through the
|
||||
// "new frame" path — pointless resize + storage on identical pixels.
|
||||
if (currentPts == _lastInnerPts && !_pLastFrame.empty()) {
|
||||
width = _imageWidth;
|
||||
height = _imageHeight;
|
||||
// Return timestamp based on decoder frame age so LabVIEW can distinguish
|
||||
// "rate-limited duplicate" from "camera truly stale".
|
||||
// If decoder is still receiving frames (age < 5s), advance PTS so LabVIEW
|
||||
// sees a changing timestamp and doesn't trigger false reconnect.
|
||||
// If decoder is stale (age > 5s), return same PTS so LabVIEW detects it.
|
||||
double ageMs = _playerClient->getLastFrameAgeMs();
|
||||
if (ageMs > 0.0 && ageMs < 5000.0) {
|
||||
// Camera alive but rate-limited — advance PTS to prevent false stale detection
|
||||
_pts++;
|
||||
_pts++; // synthetic advance — LabVIEW staleness signal
|
||||
} else if (ageMs >= 5000.0) {
|
||||
// Camera stale — keep same PTS so LabVIEW triggers reconnect
|
||||
ANS_DBG("RTSP_GetImage", "STALE PTS: ageMs=%.1f pts=%lld url=%s — not advancing PTS",
|
||||
ageMs, (long long)_pts, _url.c_str());
|
||||
}
|
||||
pts = _pts;
|
||||
g_rtspPulseDupPts.fetch_add(1, std::memory_order_relaxed);
|
||||
return _pLastFrame;
|
||||
}
|
||||
|
||||
// Update PTS first
|
||||
_pts = currentPts;
|
||||
pts = currentPts;
|
||||
// Inner advanced — real new frame. Sync our duplicate-detection cursor
|
||||
// and bump _pts monotonically (never assign to currentPts, which would
|
||||
// make _pts go backwards if it had been advanced past currentPts on
|
||||
// previous duplicates).
|
||||
_lastInnerPts = currentPts;
|
||||
_pts++;
|
||||
pts = _pts;
|
||||
g_rtspPulseNewFrame.fetch_add(1, std::memory_order_relaxed);
|
||||
|
||||
// Handle non-rotated case
|
||||
if (_imageRotateDeg == 0) {
|
||||
@@ -992,6 +1048,24 @@ namespace ANSCENTER {
|
||||
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||
_playerClient->setTargetFPS(intervalMs); // 0=no limit, 100=~10FPS, 200=~5FPS
|
||||
}
|
||||
void ANSRTSPClient::SetRtpOverUdp(bool useUdp) {
|
||||
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||
// Forwards to CRtspClient::set_rtp_over_udp via CRtspPlayer. The TRUE/FALSE
|
||||
// mapping is inverted inside (rtp_tcp = useUdp ? 0 : 1) — see rtsp_cln.cpp:3578.
|
||||
_playerClient->setRtpOverUdp(useUdp ? TRUE : FALSE);
|
||||
ANS_DBG("RTSP_Lifecycle",
|
||||
"SetRtpOverUdp: useUdp=%d url=%s — must be called before Start()",
|
||||
(int)useUdp, _url.c_str());
|
||||
}
|
||||
void ANSRTSPClient::SetJitterBufferMs(int ms) {
|
||||
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||
_playerClient->setJitterBufferMs(ms);
|
||||
ANS_DBG("RTSP_Lifecycle",
|
||||
"SetJitterBufferMs: ms=%d url=%s", ms, _url.c_str());
|
||||
}
|
||||
int ANSRTSPClient::GetJitterBufferMs() const {
|
||||
return _playerClient->getJitterBufferMs();
|
||||
}
|
||||
void ANSRTSPClient::SetNV12FastPath(bool enable) {
|
||||
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||
_useNV12FastPath = enable;
|
||||
@@ -1585,6 +1659,24 @@ extern "C" __declspec(dllexport) void SetRTSPTargetFPS(ANSCENTER::ANSRTSPClient*
|
||||
(*Handle)->SetTargetFPS(intervalMs); // 0=no limit, 100=~10FPS, 200=~5FPS
|
||||
} catch (...) { }
|
||||
}
|
||||
extern "C" __declspec(dllexport) void SetRTSPRtpOverUdp(ANSCENTER::ANSRTSPClient** Handle, int useUdp) {
|
||||
if (Handle == nullptr || *Handle == nullptr) return;
|
||||
try {
|
||||
(*Handle)->SetRtpOverUdp(useUdp != 0);
|
||||
} catch (...) { }
|
||||
}
|
||||
extern "C" __declspec(dllexport) void SetRTSPJitterBuffer(ANSCENTER::ANSRTSPClient** Handle, int jitterMs) {
|
||||
if (Handle == nullptr || *Handle == nullptr) return;
|
||||
try {
|
||||
(*Handle)->SetJitterBufferMs(jitterMs);
|
||||
} catch (...) { }
|
||||
}
|
||||
extern "C" __declspec(dllexport) int GetRTSPJitterBuffer(ANSCENTER::ANSRTSPClient** Handle) {
|
||||
if (Handle == nullptr || *Handle == nullptr) return -1;
|
||||
try {
|
||||
return (*Handle)->GetJitterBufferMs();
|
||||
} catch (...) { return -1; }
|
||||
}
|
||||
extern "C" __declspec(dllexport) void SetRTSPNV12FastPath(ANSCENTER::ANSRTSPClient** Handle, int enable) {
|
||||
if (Handle == nullptr || *Handle == nullptr) return;
|
||||
try {
|
||||
|
||||
@@ -37,6 +37,11 @@ namespace ANSCENTER
|
||||
std::string _lastJpegImage;
|
||||
int _imageWidth,_imageHeight;
|
||||
int64_t _pts;
|
||||
// Last currentPts seen from the inner getImage(). Used for duplicate
|
||||
// detection separately from _pts (which gets synthetically incremented
|
||||
// for LabVIEW's staleness check). Keeping these decoupled fixes the
|
||||
// "every other dup goes through new-frame path" inefficiency.
|
||||
int64_t _lastInnerPts = -1;
|
||||
bool _isPlaying;
|
||||
bool _useNV12FastPath = false; // false = original stable CPU path, true = NV12 GPU fast path
|
||||
std::recursive_mutex _mutex;
|
||||
@@ -104,6 +109,21 @@ namespace ANSCENTER
|
||||
void SetDisplayResolution(int width, int height); // Set display output size; 0,0 = original (no resize)
|
||||
void SetImageQuality(int mode); // 0=fast (AI), 1=quality (display)
|
||||
void SetTargetFPS(double intervalMs); // Set min interval between processed frames in ms (0 = no limit, 100 = ~10 FPS, 200 = ~5 FPS)
|
||||
// Jitter buffer for the BGR delivery path. Holds frames ms milliseconds
|
||||
// before releasing to absorb network/decoder hiccups. 0 = today's
|
||||
// getLatest semantics. Default in CVideoPlayer is 200 ms.
|
||||
// Adaptive: if the consumer is slower than the producer, drop-ahead
|
||||
// keeps latency bounded — slow callers (LabVIEW polling at 1-5 Hz)
|
||||
// effectively bypass the buffer.
|
||||
// Inference path (NV12/CUDA fast-path) is unaffected — it always
|
||||
// receives the latest decoded frame regardless of this setting.
|
||||
void SetJitterBufferMs(int ms);
|
||||
int GetJitterBufferMs() const;
|
||||
// Transport for RTP packets. true = RTP/UDP (lower latency, no TCP head-of-line
|
||||
// blocking — recommended on LAN). false = RTP/RTSP/TCP interleaved (default, more
|
||||
// firewall-friendly but bursts on packet loss / large I-frames).
|
||||
// MUST be called BEFORE Start() / StartRTSP — the SETUP request is sent inside play().
|
||||
void SetRtpOverUdp(bool useUdp);
|
||||
void SetNV12FastPath(bool enable); // true = NV12 GPU fast path (zero-copy inference), false = original CPU path (stable)
|
||||
bool IsNV12FastPath() const { return _useNV12FastPath; }
|
||||
double GetLastFrameAgeMs(); // Milliseconds since last frame from decoder (detects truly stale cameras, unaffected by SetTargetFPS)
|
||||
@@ -145,6 +165,9 @@ extern "C" __declspec(dllexport) int GetRTSPHWDecodingGpuIndex(ANSCENTER::ANSRT
|
||||
extern "C" __declspec(dllexport) void SetRTSPImageQuality(ANSCENTER::ANSRTSPClient** Handle, int mode);
|
||||
extern "C" __declspec(dllexport) void SetRTSPDisplayResolution(ANSCENTER::ANSRTSPClient** Handle, int width, int height);
|
||||
extern "C" __declspec(dllexport) void SetRTSPTargetFPS(ANSCENTER::ANSRTSPClient** Handle, double intervalMs);
|
||||
extern "C" __declspec(dllexport) void SetRTSPRtpOverUdp(ANSCENTER::ANSRTSPClient** Handle, int useUdp);
|
||||
extern "C" __declspec(dllexport) void SetRTSPJitterBuffer(ANSCENTER::ANSRTSPClient** Handle, int jitterMs);
|
||||
extern "C" __declspec(dllexport) int GetRTSPJitterBuffer(ANSCENTER::ANSRTSPClient** Handle);
|
||||
extern "C" __declspec(dllexport) void SetRTSPNV12FastPath(ANSCENTER::ANSRTSPClient** Handle, int enable);
|
||||
extern "C" __declspec(dllexport) double GetRTSPLastFrameAgeMs(ANSCENTER::ANSRTSPClient** Handle);
|
||||
#endif
|
||||
@@ -833,14 +833,36 @@ int RSTPTestCVClient() {
|
||||
// std::string rtspUrl = "rtsp://admin:admin123@192.168.1.23:554";//"rtsp://alex:admin123@192.168.1.11:554";//
|
||||
//std::string rtspUrl = "rtsp://anscamuser:SonTay2020@anssontay.tplinkdns.com:554/Streaming/Channels/401";// "rtsp://admin:JA!uV_mx@42.1.117.99:8093/1/1";
|
||||
|
||||
std::string rtspUrl = "rtsp://admin:AnsHome16%&@192.168.1.104:554";//"rtsp://alex:admin123@192.168.1.11:554";//112
|
||||
std::string rtspUrl = "rtsp://admin:Grb%401212@14.224.237.228:15541/Streaming/channels/101";//"rtsp://admin:AnsHome16%&@192.168.1.104:554";//"rtsp://alex:admin123@192.168.1.11:554";//112
|
||||
|
||||
CreateANSRTSPHandle(&rtspClient, "", "", "", rtspUrl.c_str());
|
||||
//void SetRTSPImageQuality(ANSCENTER::ANSRTSPClient** Handle, int mode)
|
||||
SetRTSPImageQuality(&rtspClient,0);
|
||||
//SetBBoxRTSP(&rtspClient, 0, 0, 1920, 1080);
|
||||
//SetCropFlagRTSP(&rtspClient, 1);
|
||||
// Hardware decode (auto-pick: CUDA if available, otherwise D3D11VA/DXVA2).
|
||||
// Without this the player uses single-threaded SW decode (default since
|
||||
// commit e134ebd "Use software decoder by default"), which on 4K HEVC
|
||||
// can take 80-120ms/frame — too slow for 25/30fps. The decoder falls
|
||||
// behind, the RTP buffer fills, and frames arrive in bursts (the
|
||||
// "smooth → 1-2s freeze → catch-up" pattern). VLC uses HW decode by
|
||||
// default, which is why it plays the same camera smoothly.
|
||||
// MUST be called before StartRTSP — decoder is constructed inside play().
|
||||
SetRTSPHWDecoding(&rtspClient, /*HW_DECODING_AUTO=*/0, /*preferredGpu=*/-1);
|
||||
|
||||
// Transport: default is RTP-over-RTSP/TCP (interleaved). On a LAN, UDP
|
||||
// avoids TCP head-of-line stalls. Over WAN/NAT it typically does NOT
|
||||
// work (port forwarding usually only covers TCP/554), so leave commented
|
||||
// out for remote cameras. MUST be called before StartRTSP.
|
||||
//SetRTSPRtpOverUdp(&rtspClient, 1);
|
||||
|
||||
StartRTSP(&rtspClient);
|
||||
// Let ANSRTSP downscale internally (INTER_LINEAR, ~5x faster than the
|
||||
// INTER_LANCZOS4 the test was doing). Without this, a 4K source plus
|
||||
// LANCZOS4 resize stretches each loop iteration to ~150ms and the
|
||||
// 20-slot decoder queue overruns — frames look "skipped" because
|
||||
// intermediate ones get evicted before getLatestFrame() sees them.
|
||||
SetRTSPDisplayResolution(&rtspClient, 1920, 1080);
|
||||
//SetRTSPImageRotation(&rtspClient,0);
|
||||
cv::namedWindow("Image", cv::WINDOW_NORMAL); // Create a resizable window.
|
||||
cv::resizeWindow("Image", 1920, 1080); // Set initial size of the window (landscape).
|
||||
@@ -854,12 +876,13 @@ int RSTPTestCVClient() {
|
||||
//if ((index == 1800) || (index == 2200) || (index == 2500)) { StartRTSP(&rtspClient); }
|
||||
//if ((index == 2000) || (index == 2300) || (index == 2700)) { StartRTSP(&rtspClient); }
|
||||
//if (index > 20000) break;
|
||||
auto start = std::chrono::system_clock::now();
|
||||
auto loopStart = std::chrono::steady_clock::now();
|
||||
auto getStart = loopStart;
|
||||
cv::Mat* image = nullptr; // ✅ Use a pointer to hold the allocated image
|
||||
GetRTSPCVImage(&rtspClient, width, height, pts,&image);
|
||||
auto end1 = std::chrono::system_clock::now();
|
||||
auto elapsed1 = std::chrono::duration_cast<std::chrono::milliseconds>(end1 - start);
|
||||
if (elapsed1.count() > 0)std::cout << "Time to get image:" << elapsed1.count() << "ms" << std::endl;
|
||||
auto getEnd = std::chrono::steady_clock::now();
|
||||
auto getMs = std::chrono::duration_cast<std::chrono::milliseconds>(getEnd - getStart).count();
|
||||
if (getMs > 0) std::cout << "Time to get image:" << getMs << "ms" << std::endl;
|
||||
|
||||
// ✅ Check if the image is valid BEFORE accessing it
|
||||
if (!image || image->empty()) {
|
||||
@@ -868,23 +891,42 @@ int RSTPTestCVClient() {
|
||||
continue; // Skip processing if the image is empty
|
||||
}
|
||||
|
||||
// High-quality downscale for display: INTER_LANCZOS4 preserves sharpness and edges
|
||||
cv::Mat displayImage;
|
||||
if (image->cols > 1920) {
|
||||
double scale = 1920.0 / image->cols;
|
||||
cv::resize(*image, displayImage, cv::Size(), scale, scale, cv::INTER_LANCZOS4);
|
||||
}
|
||||
else {
|
||||
displayImage = *image;
|
||||
}
|
||||
cv::imshow("Image", displayImage);
|
||||
// ANSRTSP already returned a 1920x1080 frame — no client-side resize.
|
||||
// Time the imshow specifically — on Windows it can occasionally block
|
||||
// for 100s of ms (window message pump, GPU sync, monitor refresh
|
||||
// alignment), and that would visibly look like a freeze even though
|
||||
// the producer pulse stays healthy.
|
||||
auto showStart = std::chrono::steady_clock::now();
|
||||
cv::imshow("Image", *image);
|
||||
auto showEnd = std::chrono::steady_clock::now();
|
||||
auto showMs = std::chrono::duration_cast<std::chrono::milliseconds>(showEnd - showStart).count();
|
||||
|
||||
ANSCV_ReleaseImage_S(&image);
|
||||
//std::cout << "Index="<<index<<". Size: " << width << "x" << height << ". Timestamp: " << pts << std::endl;
|
||||
if (cv::waitKey(30) == 27) {
|
||||
|
||||
// Time waitKey separately — also a Windows GUI pump, can block.
|
||||
auto waitStart = std::chrono::steady_clock::now();
|
||||
// 1ms is enough to pump GUI events; 30ms caps the loop at <33fps and
|
||||
// is the main reason the camera-timer appeared to skip.
|
||||
bool quit = (cv::waitKey(1) == 27);
|
||||
auto waitEnd = std::chrono::steady_clock::now();
|
||||
auto waitMs = std::chrono::duration_cast<std::chrono::milliseconds>(waitEnd - waitStart).count();
|
||||
|
||||
// Print only when something is actually slow — keeps the log readable.
|
||||
// Threshold: 30ms = noticeable stutter, 100ms = visible freeze contributor.
|
||||
if (showMs >= 30 || waitMs >= 30) {
|
||||
std::cout << "[SLOW DISPLAY] imshow=" << showMs << "ms waitKey=" << waitMs
|
||||
<< "ms (frame=" << index << ")" << std::endl;
|
||||
}
|
||||
|
||||
if (quit) {
|
||||
std::cout << "Break" << std::endl;
|
||||
break;
|
||||
}
|
||||
|
||||
// Whole-iteration timing — should now be close to 1/cameraFps.
|
||||
auto loopMs = std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::steady_clock::now() - loopStart).count();
|
||||
if (loopMs >= 50) std::cout << "Loop iteration: " << loopMs << "ms" << std::endl;
|
||||
}
|
||||
cv::destroyAllWindows(); // Destroy all OpenCV windows
|
||||
StopRTSP(&rtspClient);
|
||||
|
||||
Reference in New Issue
Block a user