#ifndef VIDEO_PLAYER_H #define VIDEO_PLAYER_H #include "sys_inc.h" #include "hqueue.h" #include "video_decoder.h" #include "audio_decoder.h" #include "audio_play.h" #include "avi_write.h" #include "media_util.h" #include #include #include #include #include #include #include typedef struct { uint32 SyncTimestamp; struct timeval SyncTime; } HTCLOCK; typedef std::list FRAMELIST; // Thread-safe queue for AVFrame class FrameQueue { private: std::queue frameQueue; std::mutex queueMutex; const size_t maxFrames = 20; // Increased buffer size for better performance uint64_t m_frameSeq = 0; // Sequence counter — increments on each new frame push public: // Push a new frame to the queue (removes oldest if full) void pushFrame(AVFrame* newFrame) { if (!newFrame) return; std::lock_guard lock(queueMutex); // Clone the frame to ensure proper ownership transfer AVFrame* frameCopy = av_frame_clone(newFrame); if (!frameCopy) { std::cerr << "Failed to clone AVFrame!" << std::endl; return; } frameQueue.push(frameCopy); m_frameSeq++; // New frame arrived // If queue exceeds max size, remove the oldest frame if (frameQueue.size() > maxFrames) { AVFrame* oldFrame = frameQueue.front(); frameQueue.pop(); av_frame_free(&oldFrame); } } // Get current sequence number (check if new frames arrived) uint64_t getSequence() { std::lock_guard lock(queueMutex); return m_frameSeq; } // Retrieve latest frame (returns a clone for thread safety) AVFrame* getLatestFrame() { std::lock_guard lock(queueMutex); if (frameQueue.empty()) { return nullptr; // No frames available } // Clone the latest frame before returning it return av_frame_clone(frameQueue.back()); } // Retrieve and remove the oldest frame from the queue AVFrame* popFrame() { std::lock_guard lock(queueMutex); if (frameQueue.empty()) { return nullptr; } AVFrame* frontFrame = frameQueue.front(); frameQueue.pop(); return frontFrame; // Caller must free the returned frame } // Check if the queue is empty bool isEmpty() { std::lock_guard lock(queueMutex); return frameQueue.empty(); } // Clear the queue (e.g., when shutting down) void clearQueue() { std::lock_guard lock(queueMutex); while (!frameQueue.empty()) { AVFrame* frame = frameQueue.front(); frameQueue.pop(); av_frame_free(&frame); } m_frameSeq = 0; } }; class CVideoPlayer { public: CVideoPlayer(); virtual ~CVideoPlayer(); virtual BOOL open(std::string fileName); virtual BOOL open(std::string username, std::string password, std::string url); virtual BOOL play() = 0; virtual void stop() = 0; virtual BOOL pause() = 0; virtual void close(); virtual BOOL seek(int pos) { return FALSE; } virtual BOOL isPlaying() { return m_bPlaying; } virtual BOOL isPaused() { return m_bPaused; } virtual void setVolume(int volume); virtual void snapshot(int videofmt); virtual BOOL record(std::string baseName); virtual void stopRecord(); virtual BOOL isRecording() { return m_bRecording; } virtual BOOL onRecord() { return FALSE; } virtual int getVideoClock() { return 1000; } virtual int getAudioClock() { return 1000; } //virtual void setWindowSize(int size); virtual int64 getElapse() { return 0; } virtual int64 getDuration() { return 0; } virtual int getVideoCodec() { return m_nVideoCodec; } virtual int getAudioCodec() { return m_nAudioCodec; } virtual void setAuthInfo(std::string acct, std::string pass) { m_acct = acct; m_pass = pass; } //virtual void setRenderMode(int mode) { m_nRenderMode = mode; } virtual void setHWDecoding(int mode, int preferredGpu = -1) { m_nHWDecoding = mode; m_nPreferredGpu = preferredGpu; } virtual bool isHWDecodingActive() const { return m_pVideoDecoder && m_pVideoDecoder->isHardwareDecoderEnabled(); } virtual int getHWDecodingGpuIndex() const { return m_pVideoDecoder ? m_pVideoDecoder->getHWGpuIndex() : -1; } // Image quality mode: 0=fast (OpenCV BT.601, ~2ms), 1=quality (sws BT.709+range, ~12ms) virtual void setImageQuality(int mode) { m_nImageQuality = mode; } void setTargetFPS(double intervalMs); // Set minimum interval between processed frames in ms (0 = no limit, 100 = ~10 FPS) virtual void setRtpMulticast(BOOL flag) {} virtual void setRtpOverUdp(BOOL flag) {} #ifdef OVER_HTTP virtual void setRtspOverHttp(int flag, int port) {} #endif #ifdef OVER_WEBSOCKET virtual void setRtspOverWs(int flag, int port) {} #endif #ifdef BACKCHANNEL virtual int getBCFlag() { return 0; } virtual void setBCFlag(int flag) {} virtual int getBCDataFlag() { return 0; } virtual void setBCDataFlag(int flag) {} virtual void setAudioDevice(int index) {} #endif #ifdef REPLAY virtual int getReplayFlag() { return 0; } virtual void setReplayFlag(int flag) {} virtual void setScale(double scale) {} virtual void setRateControlFlag(int flag) {} virtual void setImmediateFlag(int flag) {} virtual void setFramesFlag(int flag, int interval) {} virtual void setReplayRange(time_t start, time_t end) {} #endif BOOL openVideo(int codec, uint8* extradata = NULL, int extradata_size = 0); BOOL openVideo(enum AVCodecID codec, uint8* extradata = NULL, int extradata_size = 0); void closeVideo(); BOOL openAudio(int codec, int samplerate, int channels, int bitpersample = 0); BOOL openAudio(enum AVCodecID codec, int samplerate, int channels, int bitpersample = 0); void closeAudio(); void enableAudio(bool status); void playVideo(uint8* data, int len, uint32 ts, uint16 seq); void playAudio(uint8* data, int len, uint32 ts, uint16 seq); void recordVideo(uint8* data, int len, uint32 ts, uint16 seq); void recordAudio(uint8* data, int len, uint32 ts, uint16 seq); int getVideoWidth(); int getVideoHeight(); int getOriginalWidth() { return m_nv12OrigWidth; } // Full NV12 resolution (before display downscale) int getOriginalHeight() { return m_nv12OrigHeight; } // Full NV12 resolution (before display downscale) double getFrameRate(); int getSampleRate() { return m_nSampleRate; } int getChannel() { return m_nChannel; } void onVideoFrame(AVFrame* frame); void onAudioFrame(AVFrame* frame); void StartVideoDecoder(); void StopVideoDecoder(); cv::Mat getImage(int& width, int& height, int64_t& pts);// Get image std::string getJpegImage(int& width, int& height, int64_t& pts);// Get image AVFrame* getNV12Frame(); // Transfers ownership of NV12/YUV frame for GPU fast-path (caller must av_frame_free) AVFrame* getCudaHWFrame(); // Returns clone of CUDA HW frame (device ptrs); caller must av_frame_free bool isCudaHWAccel() const; void setBbox(cv::Rect bbox); void setCrop(bool crop); protected: void updateClock(HTCLOCK* clock, uint32 ts, int frequency); BOOL initFrame(AVFrame*& frame, int width, int height, AVPixelFormat pixfmt); BOOL doSnapshot(AVFrame* srcframe); AVFrame* convertFrame(AVFrame* srcframe, AVFrame* dstframe, BOOL updown); void recordVideoEx(uint8* data, int len, uint32 ts, uint16 seq); BOOL recordSwitchCheck(); void recordFileSwitch(); AVFrame* cropFrame(const AVFrame* srcFrame, cv::Rect bBox, bool cropFlag); cv::Mat avframeAnyToCvmat(const AVFrame* frame); cv::Mat avframeNV12ToCvMat(const AVFrame* frame); cv::Mat avframeYUVJ420PToCvmat(const AVFrame* frame); cv::Mat avframeToCVMat(const AVFrame* frame); // YUVJ420P to JPEG std::string avframeYUVJ420PToJpegString(const AVFrame* pFrame); std::string avframeYUVJ420PToJpegStringUsingTurboJPEG(const AVFrame* pFrame); std::string avframeYUVJ420PToJpegStringUsingFFMpeg(const AVFrame* pFrame); AVFrame* convertNV12ToYUVJ420P(const AVFrame* nv12Frame); std::string encodeYUVJ420PToJPEG(AVFrame* frame, int quality = 90); //Direct conversion from NV12 to JPEG std::string avframeToJpegString(const AVFrame* pFrame); std::string encodeNV12ToJPEG_TurboJPEG(const AVFrame* frame, int quality = 90); std::string encodeNV12ToJPEG_FFmpeg(const AVFrame* pFrame, int quality = 90); // Check if frame are identical bool areFramesIdentical(AVFrame* frame1, AVFrame* frame2); protected: // Use a constant for the maximum queue size //const int MAX_VIDEO_FRAMES = 10; //const int MAX_AUDIO_FRAMES = 2; BOOL m_bVideoInited; BOOL m_bAudioInited; tjhandle _tjInstance; std::recursive_mutex _mutex; std::unique_ptr m_pVideoDecoder = std::make_unique(); std::unique_ptr m_pAudioDecoder = std::make_unique(); std::unique_ptr m_pAudioPlay = nullptr; cv::Mat m_currentImage; AVFrame* m_currentNV12Frame = nullptr; // Preserved NV12/YUV frame for GPU fast-path AVFrame* m_currentCudaHWFrame = nullptr; // CUDA HW frame with device ptrs for zero-copy int m_Width; int m_Height; int m_nv12OrigWidth = 0; // Original NV12 frame width (before display resize) int m_nv12OrigHeight = 0; // Original NV12 frame height (before display resize) int64_t m_pts; uint64_t m_lastFrameSeq = 0; // Last processed frame sequence number bool m_bWaitingForKeyframe = true; // Skip frames until first keyframe after start/restart int m_cleanFrameCount = 0; // Count of clean frames after keyframe static const int SETTLE_FRAME_COUNT = 5; // Number of clean frames before delivering new frames // Frame rate limiting — skip post-decode processing for frames beyond target interval double m_targetIntervalMs = 100.0; // default 100ms (~10 FPS), 0 = no limit (process all frames) std::chrono::steady_clock::time_point m_lastProcessedTime; // timestamp of last processed frame bool m_targetFPSInitialized = false; // first-frame flag BOOL m_bPlaying; BOOL m_bPaused; std::string m_acct; std::string m_pass; std::string m_sFileName; std::string m_sBaseName; std::string m_jpegImage; std::string m_lastJpegImage; int m_nHWDecoding; int m_nPreferredGpu = -1; // -1=auto (least-loaded), >=0 = prefer this GPU for NVDEC int m_nImageQuality = 0; // 0=fast (default), 1=quality BT.709 //AVPixelFormat m_nDstVideoFmt; BOOL m_bUpdown; BOOL m_bSnapshot; int m_nSnapVideoFmt; H26XParamSets m_h26XParamSets; int m_nVideoCodec; int m_nAudioCodec; int m_nSampleRate; int m_nChannel; int m_nBitPerSample; AVFrame* m_pSnapFrame; FrameQueue g_frameQueue; FrameQueue a_frameQueue; BOOL m_bRecording; BOOL m_bNalFlag; AVICTX* m_pAviCtx; void* m_pRecordMutex; HTCLOCK m_audioClock; void* m_pAudioListMutex; //FRAMELIST m_audioFrameList; BOOL m_audioPlayFlag; //pthread_t m_audioPlayThread; HTCLOCK m_videoClock; void* m_pVideoListMutex; //FRAMELIST m_videoFrameList; BOOL m_videoPlayFlag; //pthread_t m_videoPlayThread; uint64 m_nLastAudioPts; time_t m_lastAudioTS; cv::Rect m_Bbox; bool m_bCrop; SwsContext* swsCtx = nullptr; // Shared SwsContext int lastWidth = 0, lastHeight = 0; AVPixelFormat lastPixFmt = AV_PIX_FMT_NONE; AVPixelFormat lastOutPixFmt = AV_PIX_FMT_NONE; // Dedicated NV12→BGR context with correct color space (BT.709 for HD/4K) SwsContext* m_nv12SwsCtx = nullptr; int m_nv12LastWidth = 0, m_nv12LastHeight = 0; int m_nv12LastColorspace = -1; int m_nv12LastRange = -1; void initSwsContext(int width, int height, AVPixelFormat pixFmt, AVPixelFormat outputPixFmt = AV_PIX_FMT_YUVJ420P); void initNV12SwsContext(const AVFrame* frame); void cropPlane(const AVFrame* srcFrame, AVFrame* croppedFrame, int planeIndex, int offsetX, int offsetY, int subsampleX, int subsampleY); bool cropFrameData(const AVFrame* srcFrame, AVFrame* croppedFrame, const cv::Rect& bBox); }; #endif // end of VIDEO_PLAYER_H