Files
ANSCustomModels/ANSCustomFireNSmokeDetection/ANSCustomFireNSmoke.cpp
T
2026-06-18 05:57:14 +10:00

1295 lines
42 KiB
C++
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#include "ANSCustomFireNSmoke.h"
bool ANSCustomFS::IsDetectedAreaValid(const cv::Rect& area) const {
return area.width > MIN_ROI_SIZE && area.height > MIN_ROI_SIZE;
}
// Reject fire/smoke boxes that are too small relative to the frame. Defined as a
// fraction of frame area so the behavior is consistent across camera resolutions.
bool ANSCustomFS::IsObjectLargeEnough(const cv::Rect& box, const cv::Size& frameSize) const {
const double frameArea = static_cast<double>(frameSize.width) * frameSize.height;
if (frameArea <= 0.0) return true; // can't judge — don't drop
const double boxArea = static_cast<double>(std::max(0, box.width)) * std::max(0, box.height);
return (boxArea / frameArea) >= static_cast<double>(_minObjectAreaRatio);
}
bool ANSCustomFS::IsFireOrSmoke(int classId, float confidence) const {
return (classId == 0) || // Fire
(classId == 2 && confidence >= SMOKE_CONFIDENCE_THRESHOLD); // Smoke
}
bool ANSCustomFS::ValidateMotionCorrelation(const std::vector<cv::Rect>& fireNSmokeRects) const {
return !fireNSmokeRects.empty() && fireNSmokeRects.size() < MAX_MOTION_TRACKING;
}
void ANSCustomFS::UpdatePositiveDetection() {
_isFireNSmokeDetected = true;
_retainDetectedArea = 0;
_isRealFireFrame = true;
}
void ANSCustomFS::ResetDetectedArea() {
_detectedArea = cv::Rect(0, 0, 0, 0);
}
void ANSCustomFS::UpdatePriorityRegion(const cv::Mat& frame)
{
// divideImage caches geometry by frame size — it's cheap when the size is stable.
std::vector<ImageSection> sections = divideImage(frame);
if (cachedSections.empty()) {
_currentPriority = 0;
return;
}
const int lowestPriority = getLowestPriorityRegion();
if (_currentPriority == 0 || _currentPriority >= lowestPriority) {
// Start of a new scan cycle — re-rank sections by warm-HSV pixel count.
// This steers the first-pass scan straight to the likely fire regions
// (reduces "late detection" compared to the old distance-to-center order).
RankCachedSectionsByWarmColor(frame);
_currentPriority = getHighestPriorityRegion();
}
else {
_currentPriority++;
}
}
void ANSCustomFS::RankCachedSectionsByWarmColor(const cv::Mat& frame)
{
if (cachedSections.empty() || frame.empty()) return;
try {
cv::Mat hsv;
cv::cvtColor(frame, hsv, cv::COLOR_BGR2HSV);
// Warm color = red / orange / yellow hues with enough saturation and brightness.
// OpenCV H is 0..179, so we cover the red wrap with two ranges.
cv::Mat mask1, mask2, warmMask;
cv::inRange(hsv,
cv::Scalar(0, 100, 150), cv::Scalar(30, 255, 255), mask1);
cv::inRange(hsv,
cv::Scalar(160, 100, 150), cv::Scalar(180, 255, 255), mask2);
warmMask = mask1 | mask2;
const cv::Rect frameRect(0, 0, frame.cols, frame.rows);
std::vector<std::pair<int, int>> scored; // (warmCount, sectionIndex)
scored.reserve(cachedSections.size());
for (size_t i = 0; i < cachedSections.size(); ++i) {
const cv::Rect safe = cachedSections[i].region & frameRect;
int count = 0;
if (safe.area() > 0) {
count = cv::countNonZero(warmMask(safe));
}
scored.emplace_back(count, static_cast<int>(i));
}
// Sort descending by warm count; tie-break by distance-to-center (original order).
std::stable_sort(scored.begin(), scored.end(),
[](const std::pair<int, int>& a, const std::pair<int, int>& b) {
return a.first > b.first;
});
// Rewrite priorities 1..N into cachedSections so getRegionByPriority picks warmest first.
for (size_t rank = 0; rank < scored.size(); ++rank) {
cachedSections[scored[rank].second].priority = static_cast<int>(rank + 1);
}
}
catch (const cv::Exception&) {
// On failure, leave existing priorities untouched (fallback to prior order).
}
}
int ANSCustomFS::CalculateOptimalCropSize(const cv::Mat& frame) const {
int maxDim = std::max(frame.cols, frame.rows);
if (maxDim > 1920) return 640;
if (maxDim > 1280) return 480;
if (maxDim > 640) return 320;
return 224;
}
void ANSCustomFS::RefineDetectedArea(const cv::Mat& frame, const ANSCENTER::Object& detection) {
int cropSize = CalculateOptimalCropSize(frame);
int xc = detection.box.x + detection.box.width / 2;
int yc = detection.box.y + detection.box.height / 2;
int x1_new = std::clamp(xc - cropSize / 2, 0, frame.cols - cropSize);
int y1_new = std::clamp(yc - cropSize / 2, 0, frame.rows - cropSize);
_detectedArea = cv::Rect(
x1_new,
y1_new,
std::min(cropSize, frame.cols - x1_new),
std::min(cropSize, frame.rows - y1_new)
);
}
void ANSCustomFS::ResetDetectionState() {
ResetDetectedArea();
_retainDetectedArea = 0;
_isFireNSmokeDetected = false;
_trackHistory.clear();
}
// --- Tracker-based voting functions (dual-path: high-conf + low-conf) ---
void ANSCustomFS::UpdateTrackHistory(int trackId, int classId, const cv::Rect& bbox,
float confidence, double vMean) {
const int isHighConf = (confidence >= HIGH_CONF_THRESHOLD) ? 1 : 0;
auto it = _trackHistory.find(trackId);
if (it == _trackHistory.end()) {
TrackRecord record;
record.trackId = trackId;
record.classId = classId;
record.bboxHistory.push_back(bbox);
record.highConfHits.push_back(isHighConf);
record.vMeanHistory.push_back(vMean);
record.detectedCount = isHighConf;
record.lowConfDetectedCount = 1;
record.totalFrames = 1;
record.confirmed = false;
_trackHistory[trackId] = std::move(record);
}
else {
auto& record = it->second;
record.bboxHistory.push_back(bbox);
record.highConfHits.push_back(isHighConf);
record.vMeanHistory.push_back(vMean);
if (isHighConf) record.detectedCount++;
record.lowConfDetectedCount++;
record.totalFrames++;
// Slide the window: keep only VOTE_WINDOW entries in parallel deques
while (static_cast<int>(record.bboxHistory.size()) > VOTE_WINDOW) {
record.bboxHistory.pop_front();
const int oldHit = record.highConfHits.front();
record.highConfHits.pop_front();
record.vMeanHistory.pop_front();
if (oldHit == 1) record.detectedCount = std::max(0, record.detectedCount - 1);
record.lowConfDetectedCount = std::max(0, record.lowConfDetectedCount - 1);
}
// Update confirmed status (high-conf path)
if (record.detectedCount >= VOTE_THRESHOLD) {
record.confirmed = true;
}
}
}
bool ANSCustomFS::IsTrackConfirmed(int trackId) const {
auto it = _trackHistory.find(trackId);
if (it == _trackHistory.end()) return false;
return it->second.detectedCount >= VOTE_THRESHOLD;
}
bool ANSCustomFS::IsTrackLowConfirmed(int trackId) const {
auto it = _trackHistory.find(trackId);
if (it == _trackHistory.end()) return false;
// Low-conf path requires more total hits than the high-conf window would need
// (since we accept weaker detections, we compensate with more votes)
return it->second.lowConfDetectedCount >= LOW_VOTE_THRESHOLD;
}
bool ANSCustomFS::HasFireFlicker(int trackId) const {
auto it = _trackHistory.find(trackId);
if (it == _trackHistory.end()) return false;
const auto& hist = it->second.vMeanHistory;
if (static_cast<int>(hist.size()) < FLICKER_MIN_SAMPLES) return false;
// Compute mean
double sum = 0.0;
for (double v : hist) sum += v;
const double mean = sum / static_cast<double>(hist.size());
// Compute std-dev
double sqSum = 0.0;
for (double v : hist) {
const double d = v - mean;
sqSum += d * d;
}
const double stddev = std::sqrt(sqSum / static_cast<double>(hist.size()));
// Real fire flickers at ~3–10 Hz, driving measurable luma variance.
// Sunlight reflections, static bright surfaces, and highlights do not.
return stddev > FLICKER_V_STD_THRESHOLD;
}
bool ANSCustomFS::HasBboxMovement(int trackId) const {
auto it = _trackHistory.find(trackId);
if (it == _trackHistory.end()) return false;
const auto& history = it->second.bboxHistory;
if (history.size() < 2) return false;
const cv::Rect& earliest = history.front();
const cv::Rect& latest = history.back();
// Calculate center positions
float cx1 = earliest.x + earliest.width / 2.0f;
float cy1 = earliest.y + earliest.height / 2.0f;
float cx2 = latest.x + latest.width / 2.0f;
float cy2 = latest.y + latest.height / 2.0f;
// Average size for normalization
float avgWidth = (earliest.width + latest.width) / 2.0f;
float avgHeight = (earliest.height + latest.height) / 2.0f;
if (avgWidth < 1.0f || avgHeight < 1.0f) return false;
// Position change relative to average size
float posChange = std::sqrt(
std::pow((cx2 - cx1) / avgWidth, 2) +
std::pow((cy2 - cy1) / avgHeight, 2)
);
// Size change relative to average area
float area1 = static_cast<float>(earliest.area());
float area2 = static_cast<float>(latest.area());
float avgArea = (area1 + area2) / 2.0f;
float sizeChange = (avgArea > 0) ? std::abs(area2 - area1) / avgArea : 0.0f;
return (posChange > BBOX_CHANGE_THRESHOLD) || (sizeChange > BBOX_CHANGE_THRESHOLD);
}
void ANSCustomFS::AgeTracks(const std::unordered_set<int>& detectedTrackIds) {
auto it = _trackHistory.begin();
while (it != _trackHistory.end()) {
if (detectedTrackIds.find(it->first) == detectedTrackIds.end()) {
// Track was NOT detected this frame
it->second.totalFrames++;
// Remove stale tracks that haven't been seen recently (both paths empty)
if (it->second.totalFrames > VOTE_WINDOW &&
it->second.detectedCount == 0 &&
it->second.lowConfDetectedCount == 0) {
it = _trackHistory.erase(it);
continue;
}
// Age out the sliding window (add a "miss" frame) — drop oldest from all parallel deques
if (static_cast<int>(it->second.bboxHistory.size()) >= VOTE_WINDOW) {
it->second.bboxHistory.pop_front();
if (!it->second.highConfHits.empty()) {
const int oldHit = it->second.highConfHits.front();
it->second.highConfHits.pop_front();
if (oldHit == 1) it->second.detectedCount = std::max(0, it->second.detectedCount - 1);
}
if (!it->second.vMeanHistory.empty()) {
it->second.vMeanHistory.pop_front();
}
it->second.lowConfDetectedCount = std::max(0, it->second.lowConfDetectedCount - 1);
}
}
++it;
}
}
// --- End voting functions ---
void ANSCustomFS::UpdateNoDetectionCondition()
{
_isRealFireFrame = false;
_realFireCheck = 0;
if (_isFireNSmokeDetected) {
_retainDetectedArea++;
if (_retainDetectedArea >= RETAINFRAMES) {
ResetDetectionState();
}
}
else {
ResetDetectionState();
}
}
std::vector<CustomObject> ANSCustomFS::ConvertToCustomObjects(const std::vector<ANSCENTER::Object>& objects) {
std::vector<CustomObject> results;
results.reserve(objects.size());
for (const auto& obj : objects) {
if (obj.confidence >= _detectionScoreThreshold) {
CustomObject customObj;
customObj.box = obj.box;
customObj.classId = obj.classId;
customObj.confidence = obj.confidence;
customObj.cameraId = obj.cameraId;
customObj.className = obj.className;
customObj.extraInfo = "Detection Score Threshold:" +
std::to_string(_detectionScoreThreshold);
results.push_back(customObj);
}
}
return results;
}
void ANSCustomFS::GetModelParameters() {
if (!_readROIs) {
cv::Rect exRoi;
for (auto& roi : _params.ROI_Values) {
if (roi.Name.find("ExclusiveROIs") != std::string::npos) {
exRoi.x = roi.ROIPoints[0].x;
exRoi.y = roi.ROIPoints[0].y;
exRoi.width = roi.ROIPoints[1].x - roi.ROIPoints[0].x;
exRoi.height = roi.ROIPoints[2].y - roi.ROIPoints[0].y;
_exclusiveROIs.push_back(exRoi);
}
}
// Get parameters
for (auto& param : _params.Parameters) {
if (param.Name.find("SmokeScore") != std::string::npos) {
_smokeDetetectionThreshold = std::stof(param.Value);
}
else if (param.Name.find("Sensitivity") != std::string::npos) {
_motionSpecificity = 1.0f - std::stof(param.Value);
_motionSpecificity = std::clamp(_motionSpecificity, 0.0f, 1.0f);
// Note: Motion detector threshold is set during LoadModelFromFolder.
// Runtime threshold update is not supported through ANSLIB API.
}
else if (param.Name.find("MinObjectSize") != std::string::npos) {
// UI value is a percentage of the frame area (e.g. 0.03 = 0.03%).
try {
const float pct = std::stof(param.Value);
_minObjectAreaRatio = std::clamp(pct / 100.0f, 0.0f, 1.0f);
}
catch (const std::exception&) {
_minObjectAreaRatio = DEFAULT_MIN_OBJECT_RATIO;
}
}
}
_readROIs = true;
}
}
std::vector<ANSCENTER::Object> ANSCustomFS::FindNewDetectedArea(
const cv::Mat& frame,
const std::string& camera_id, cv::Mat& draw)
{
std::vector<ANSCENTER::Object> output;
// Reset detection state
_isRealFireFrame = false;
_realFireCheck = 0;
// Update priority region
UpdatePriorityRegion(frame);
_detectedArea = getRegionByPriority(_currentPriority);
#ifdef FNS_DEBUG
cv::rectangle(draw, _detectedArea, cv::Scalar(0, 0, 255), 4); // Red
#endif
if (!IsDetectedAreaValid(_detectedArea)) {
ResetDetectedArea();
return output;
}
// Run detection on new area
cv::Mat detectedROI = frame(_detectedArea);
std::vector<ANSCENTER::Object> detectedObjects;
_detector->RunInference(detectedROI, camera_id.c_str(), detectedObjects);
if (detectedObjects.empty()) {
ResetDetectedArea();
return output;
}
// Find best detection and refine area.
// Two-tier candidacy:
// 1) High-conf (>= DETECTION_CONFIDENCE_THRESHOLD) — latches ROI and emits immediately.
// 2) Low-conf fire with fire-color (>= LOW_CONF_FIRE_THRESHOLD) — latches ROI only,
// so ProcessExistingDetectedArea can run voting next frame without false positives
// from initial single-frame emission.
float maxScore = 0.0f;
ANSCENTER::Object* bestDetection = nullptr;
bool bestIsHighConf = false;
for (auto& detectedObj : detectedObjects) {
detectedObj.box.x += _detectedArea.x;
detectedObj.box.y += _detectedArea.y;
detectedObj.cameraId = camera_id;
if (!IsFireOrSmoke(detectedObj.classId, detectedObj.confidence)) continue;
// Size gate — ignore boxes too small relative to the frame.
if (!IsObjectLargeEnough(detectedObj.box, frame.size())) continue;
const bool isHighConf = (detectedObj.confidence > DETECTION_CONFIDENCE_THRESHOLD);
const bool isLowConfFire =
(detectedObj.classId == 0) && (detectedObj.confidence >= LOW_CONF_FIRE_THRESHOLD);
if (isHighConf) {
// Prefer high-conf; always overrides a low-conf candidate
if (!bestIsHighConf || detectedObj.confidence > maxScore) {
maxScore = detectedObj.confidence;
bestDetection = &detectedObj;
bestIsHighConf = true;
}
}
else if (isLowConfFire && !bestIsHighConf) {
// Gate low-conf candidates on fire-color presence in the bbox
if (!IsFireDetected(frame, detectedObj.box)) continue;
if (detectedObj.confidence > maxScore) {
maxScore = detectedObj.confidence;
bestDetection = &detectedObj;
// bestIsHighConf stays false
}
}
}
if (bestDetection != nullptr) {
RefineDetectedArea(frame, *bestDetection);
if (bestIsHighConf) {
// Fast path: emit the first-frame detection as before
output.push_back(*bestDetection);
UpdatePositiveDetection();
}
// Low-conf path: commit ROI only. Voting in ProcessExistingDetectedArea
// will decide over subsequent frames whether to emit.
}
else {
ResetDetectedArea();
}
return output;
}
std::vector<ANSCENTER::Object> ANSCustomFS::FindMovementObjects(const cv::Mat& frame, const std::string& camera_id, cv::Mat& draw)
{
if (_motionSpecificity < 1) {
// 1. Extract the detected area
cv::Mat activeROI = frame(_detectedArea);
// 2. Detect movement in the detected area
std::vector<ANSCENTER::Object> movementObjects;
_motiondetector->RunInference(activeROI, camera_id.c_str(), movementObjects);
std::vector<ANSCENTER::Object> fireNSmokeRects;
float movementarea_threshold = 0;
int totalMovementObjects = static_cast<int>(movementObjects.size());
if ((totalMovementObjects > 0) && (totalMovementObjects <= 10)) {
for (const auto& rect : movementObjects) {
ANSCENTER::Object mBbj;
mBbj.box = rect.box;
mBbj.box.x += _detectedArea.x;
mBbj.box.y += _detectedArea.y;
movementarea_threshold = calculateIoU(_detectedArea, mBbj.box);
if (movementarea_threshold < 0.5) {
fireNSmokeRects.push_back(mBbj);
#ifdef FNS_DEBUG
cv::rectangle(draw, mBbj.box, cv::Scalar(123, 122, 34), 2);
#endif
}
}
}
activeROI.release();
return fireNSmokeRects;
}
else return std::vector<ANSCENTER::Object>{};
}
std::vector<ANSCENTER::Object> ANSCustomFS::FindExcludedObjects(
const cv::Mat& frame,
const std::string& camera_id, cv::Mat& draw)
{
// Final check for filters
// Get activeROI with padding to include surrounding area
cv::Rect paddedROI = _detectedArea;
paddedROI.x -= 100;
paddedROI.y -= 100;
paddedROI.width += 200;
paddedROI.height += 200;
// Ensure paddedROI is within frame bounds
paddedROI.x = std::max(0, paddedROI.x);
paddedROI.y = std::max(0, paddedROI.y);
paddedROI.width = std::min(frame.cols - paddedROI.x, paddedROI.width);
paddedROI.height = std::min(frame.rows - paddedROI.y, paddedROI.height);
// Run filter inference on padded ROI
cv::Mat paddedFrame = frame(paddedROI);
std::vector<ANSCENTER::Object> filteredObjects;
_filter->RunInference(paddedFrame, camera_id.c_str(), filteredObjects);
if (!filteredObjects.empty()) {
std::vector<ANSCENTER::Object> excludedObjects;
for (const auto& filteredObj : filteredObjects) {
if (EXCLUDED_FILTER_CLASSES.find(filteredObj.classId) == EXCLUDED_FILTER_CLASSES.end())
{
ANSCENTER::Object exOBbj;
exOBbj.box.x = paddedROI.x + filteredObj.box.x;
exOBbj.box.y = paddedROI.y + filteredObj.box.y;
exOBbj.box.width = filteredObj.box.width;
exOBbj.box.height = filteredObj.box.height;
excludedObjects.push_back(exOBbj);
#ifdef FNS_DEBUG
cv::rectangle(draw, exOBbj.box, cv::Scalar(23, 25, 0), 2);
cv::putText(draw, filteredObj.className, cv::Point(exOBbj.box.x, exOBbj.box.y - 10), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 255, 0), 2);
#endif
}
}
}
paddedFrame.release();
return filteredObjects;
}
ANSCustomFS::ANSCustomFS()
: EXCLUDED_FILTER_CLASSES({ 13, 59, 60, 68, 69, 70 })
{
_smokeDetetectionThreshold = 0;
}
bool ANSCustomFS::OptimizeModel(bool fp16)
{
if (!_detector || !_filter) return false;
int detectorResult = _detector->Optimize(fp16);
int filterResult = _filter->Optimize(fp16);
if ((detectorResult != 1) || (filterResult != 1)) return false;
return true;
}
bool ANSCustomFS::ConfigureParameters(CustomParams& param) {
param.ROI_Config.clear();
// We do need to have exclusive ROIs for the model
CustomROIConfig exclusiveROI;
if (this->_params.ROI_Config.empty()) {
exclusiveROI.Name = "ExclusiveROIs";
exclusiveROI.Polygon = false;
exclusiveROI.Rectangle = true;
exclusiveROI.Line = false;
exclusiveROI.MinItems = 0;
exclusiveROI.MaxItems = 20;
exclusiveROI.ROIMatch = "All Corners";
param.ROI_Config.push_back(exclusiveROI);
}
else {
for (auto& roi : this->_params.ROI_Config) {
param.ROI_Config.push_back(roi);
}
}
param.ROI_Options.clear();
param.Parameters.clear();
CustomParameter param1;
param1.Name = "SmokeScore";
param1.DataType = "float";
param1.NoOfDecimals = 2;
param1.MaxValue = 1;
param1.MinValue = 0;
param1.StartValue = std::to_string(SMOKE_CONFIDENCE_THRESHOLD);
param1.ListItems.clear();
param1.DefaultValue = std::to_string(SMOKE_CONFIDENCE_THRESHOLD);
param.Parameters.push_back(param1);
CustomParameter param2;
param2.Name = "Sensitivity";
param2.DataType = "float";
param2.NoOfDecimals = 2;
param2.MaxValue = 1;
param2.MinValue = 0;
param2.StartValue = std::to_string(MOTION_SENSITIVITY);
param2.ListItems.clear();
param2.DefaultValue = std::to_string(MOTION_SENSITIVITY);
param.Parameters.push_back(param2);
// Minimum detected fire/smoke size, as a percentage of the full frame area.
// Boxes below this are rejected — suppresses tiny false positives.
CustomParameter param3;
param3.Name = "MinObjectSize";
param3.DataType = "float";
param3.NoOfDecimals = 2;
param3.MaxValue = 10; // up to 10% of the frame
param3.MinValue = 0;
param3.StartValue = std::to_string(DEFAULT_MIN_OBJECT_RATIO * 100.0f);
param3.ListItems.clear();
param3.DefaultValue = std::to_string(DEFAULT_MIN_OBJECT_RATIO * 100.0f);
param.Parameters.push_back(param3);
param.ROI_Values.clear();
for (auto& roi : this->_params.ROI_Values) {
CustomROIValue roiValue;
roiValue.Name = roi.Name;
roiValue.ROIPoints = roi.ROIPoints;
roiValue.ROIMatch = roi.ROIMatch;
roiValue.Option = roi.Option;
param.ROI_Values.push_back(roiValue);
}
return true;
}
std::vector<CustomObject> ANSCustomFS::RunInference(const cv::Mat& input)
{
return RunInference(input, "CustomCam");
}
bool ANSCustomFS::Destroy()
{
try {
_detector.reset();
_filter.reset();
_motiondetector.reset();
return true;
}
catch (...) {
return false;
}
}
bool ANSCustomFS::Initialize(const std::string& modelDirectory, float detectionScoreThreshold, std::string& labelMap)
{
try {
_modelDirectory = modelDirectory;
_detectionScoreThreshold = detectionScoreThreshold;
// Create model instances using factory pattern (ABI-safe)
_detector = ANSLIBPtr(ANSCENTER::ANSLIB::Create(), &ANSCENTER::ANSLIB::Destroy);
_filter = ANSLIBPtr(ANSCENTER::ANSLIB::Create(), &ANSCENTER::ANSLIB::Destroy);
_motiondetector = ANSLIBPtr(ANSCENTER::ANSLIB::Create(), &ANSCENTER::ANSLIB::Destroy);
_detectorModelType = 31; // TENSORRT
_detectorDetectionType = 1; // DETECTION
_filterModelType = 31; // TENSORRT
_filterDetectionType = 1; // DETECTION
_motionModelType = 19; // Motion detection (generic CPU)
_motionDetectionType = 1; // DETECTION
// Check the hardware type
engineType = _detector->GetEngineType();
if (engineType == 1) {
// NVIDIA GPU: Use TensorRT
_detectorModelType = 31; // TENSORRT
_detectorDetectionType = 1; // DETECTION
_filterModelType = 31; // TENSORRT
_filterDetectionType = 1; // DETECTION
std::cout << "NVIDIA GPU detected. Using TensorRT" << std::endl;
}
else {
// CPU/Other: Use OpenVINO
_detectorModelType = 30; // OPENVINO
_detectorDetectionType = 1; // DETECTION
_filterModelType = 30; //Yolo v12
_filterDetectionType = 1; // DETECTION
std::cout << "CPU detected. Using OpenVINO and ONNX" << std::endl;
}
this->_hsvThreshold = 0.25;
this->_fire_colour.push_back(std::pair(
cv::Scalar(0, 0, 179), cv::Scalar(255, 255, 255))
);
this->_smoke_colour.push_back(std::pair(
cv::Scalar(0, 0, 51), cv::Scalar(180, 128, 204))
);
if (this->_detectionScoreThreshold < 0.25f) this->_detectionScoreThreshold = 0.25f;
// Initialize parameters
_params.ROI_Config.clear();
_params.ROI_Options.clear();
_params.Parameters.clear();
_params.ROI_Values.clear();
_smokeDetetectionThreshold = SMOKE_CONFIDENCE_THRESHOLD;
#ifdef FNS_DEBUG
this->_loadEngineOnCreate = true;
#endif
int loadEngineOnCreation = _loadEngineOnCreate ? 1 : 0;
int autoEngineDetection = 1;
std::string licenseKey = "";
std::string motionDetectorLabels;
// Load detector model (fire/smoke detector).
// Floor thresholds at LOW_CONF_FIRE_THRESHOLD so the detector emits low-confidence
// detections that the dual-path voting in ProcessExistingDetectedArea can evaluate.
// The user-facing _detectionScoreThreshold is still enforced in code.
float detScoreThreshold = LOW_CONF_FIRE_THRESHOLD;
float detConfThreshold = LOW_CONF_FIRE_THRESHOLD;
float detNMSThreshold = 0.5f;
int detResult = _detector->LoadModelFromFolder(
licenseKey.c_str(),
"detector", "detector.names",
detScoreThreshold, detConfThreshold, detNMSThreshold,
autoEngineDetection,
_detectorModelType, _detectorDetectionType,
loadEngineOnCreation,
modelDirectory.c_str(),
labelMap);
if (detResult != 1) {
std::cerr << "ANSCustomFS::Initialize: Failed to load detector model." << std::endl;
return false;
}
// Enable ByteTrack tracker on the detector for persistent track IDs
int trackerResult = _detector->SetTracker(0 /*BYTETRACK*/, 1 /*enable*/);
if (trackerResult != 1) {
std::cerr << "ANSCustomFS::Initialize: Warning - Failed to enable ByteTrack tracker." << std::endl;
}
// Load filter model (COCO general object detector for false positive filtering)
float filterScoreThreshold = 0.25f;
float filterConfThreshold = 0.5f;
float filterNMSThreshold = 0.5f;
int filterResult = _filter->LoadModelFromFolder(
licenseKey.c_str(),
"filter", "filter.names",
filterScoreThreshold, filterConfThreshold, filterNMSThreshold,
autoEngineDetection,
_filterModelType, _filterDetectionType,
loadEngineOnCreation,
modelDirectory.c_str(),
_filterLabels);
if (filterResult != 1) {
std::cerr << "ANSCustomFS::Initialize: Failed to load filter model." << std::endl;
return false;
}
// Load motion detector model (optional — not used in Stage A tracker-based pipeline)
float motionScoreThreshold = MOTION_SENSITIVITY;
float motionConfThreshold = 0.5f;
float motionNMSThreshold = 0.5f;
int motionResult = _motiondetector->LoadModelFromFolder(
licenseKey.c_str(),
"motion", "motion.names",
motionScoreThreshold, motionConfThreshold, motionNMSThreshold,
autoEngineDetection,
_motionModelType, _motionDetectionType,
loadEngineOnCreation,
"", // Empty model directory - motion detector is algorithmic
motionDetectorLabels);
if (motionResult != 1) {
std::cerr << "ANSCustomFS::Initialize: Warning - Motion detector not loaded (not required for Stage A)." << std::endl;
// Non-fatal: motion detector is not used in the tracker-based Stage A pipeline
}
return true;
}
catch (const std::exception& e) {
std::cerr << "ANSCustomFS::Initialize: Exception: " << e.what() << std::endl;
return false;
}
catch (...) {
std::cerr << "ANSCustomFS::Initialize: Unknown exception." << std::endl;
return false;
}
}
ANSCustomFS::~ANSCustomFS()
{
Destroy();
}
cv::Rect ANSCustomFS::GenerateMinimumSquareBoundingBox(const std::vector<ANSCENTER::Object>& detectedObjects, int minSize) {
try {
int adMinSize = minSize - 20;
if (adMinSize < 0) adMinSize = 0;
if (detectedObjects.empty()) return cv::Rect(0, 0, minSize, minSize);
int minX = detectedObjects[0].box.x;
int minY = detectedObjects[0].box.y;
int maxX = detectedObjects[0].box.x + detectedObjects[0].box.width;
int maxY = detectedObjects[0].box.y + detectedObjects[0].box.height;
for (const auto& rect : detectedObjects) {
minX = std::min(minX, rect.box.x);
minY = std::min(minY, rect.box.y);
maxX = std::max(maxX, rect.box.x + rect.box.width);
maxY = std::max(maxY, rect.box.y + rect.box.height);
}
int width = maxX - minX;
int height = maxY - minY;
int squareSize = std::max({ width, height, adMinSize });
int centerX = minX + width / 2;
int centerY = minY + height / 2;
int squareX = centerX - squareSize / 2;
int squareY = centerY - squareSize / 2;
return cv::Rect(squareX - 10, squareY - 10, squareSize + 20, squareSize + 20);
}
catch (std::exception& e) {
return cv::Rect(0, 0, minSize, minSize);
}
}
bool ANSCustomFS::detectScreenFlicker(const std::vector<cv::Mat>& frames) {
if (frames.size() < 10) return false;
std::vector<double> intensities;
for (const auto& frame : frames) {
cv::Mat gray;
cv::cvtColor(frame, gray, cv::COLOR_BGR2GRAY);
intensities.push_back(cv::mean(gray)[0]);
}
cv::Mat fftInput(static_cast<int>(intensities.size()), 1, CV_64F, intensities.data());
cv::Mat fftOutput;
cv::dft(fftInput, fftOutput, cv::DFT_REAL_OUTPUT);
for (int i = 1; i < fftOutput.rows / 2; i++) {
double freq = std::abs(fftOutput.at<double>(i, 0));
if (freq > 50 && freq < 70) {
return true;
}
}
return false;
}
bool ANSCustomFS::detectReflection(const cv::Mat& frame) {
cv::Mat gray, edges;
cv::cvtColor(frame, gray, cv::COLOR_BGR2GRAY);
cv::Canny(gray, edges, 50, 150);
int edgeCount = cv::countNonZero(edges);
return edgeCount > 5000;
}
bool ANSCustomFS::MajorityColourInFrame(cv::Mat frame, Range range, float area_threshold)
{
cv::Mat img_hsv;
try {
cv::Mat blur, fireMask;
cv::GaussianBlur(frame, blur, cv::Size(21, 21), 0);
cv::cvtColor(blur, img_hsv, cv::COLOR_BGR2HSV);
cv::inRange(img_hsv, range.first, range.second, fireMask);
cv::erode(fireMask, fireMask, cv::Mat(), cv::Point(-1, -1), 2);
cv::dilate(fireMask, fireMask, cv::Mat(), cv::Point(-1, -1), 2);
return (cv::countNonZero(fireMask) / (frame.rows * frame.cols)) > area_threshold;
}
catch (cv::Exception& e) {
return false;
}
}
bool ANSCustomFS::MajorityColourInFrame(cv::Mat frame, std::vector<Range> ranges, float area_threshold)
{
unsigned int current_area = 0;
cv::Mat img_hsv;
try {
cv::cvtColor(frame, img_hsv, cv::COLOR_BGR2HSV);
for (const Range& range : ranges) {
cv::inRange(img_hsv, range.first, range.second, frame);
current_area += cv::countNonZero(frame);
}
}
catch (cv::Exception& e) {
return false;
}
return (float)current_area / ((float)frame.rows * frame.cols) > area_threshold;
}
bool ANSCustomFS::DetectFireNSmokeColourInFrame(const cv::Mat& frame, const cv::Rect& bBox,
const std::vector<Range>& ranges, float area_threshold)
{
if (frame.empty()) {
return false;
}
if (ranges.empty()) {
return false;
}
if (area_threshold < 0.0f || area_threshold > 1.0f) {
return false;
}
try {
constexpr int padding = 20;
const int x1 = std::max(bBox.x - padding, 0);
const int y1 = std::max(bBox.y - padding, 0);
const int x2 = std::min(bBox.x + bBox.width + padding, frame.cols);
const int y2 = std::min(bBox.y + bBox.height + padding, frame.rows);
const int roiWidth = x2 - x1;
const int roiHeight = y2 - y1;
if (roiWidth <= 0 || roiHeight <= 0) {
return false;
}
cv::Mat roiFrame = frame(cv::Rect(x1, y1, roiWidth, roiHeight));
cv::Mat blur;
cv::GaussianBlur(roiFrame, blur, cv::Size(21, 21), 0);
cv::Mat img_hsv;
cv::cvtColor(blur, img_hsv, cv::COLOR_BGR2HSV);
cv::Mat combined_mask = cv::Mat::zeros(img_hsv.size(), CV_8U);
cv::Mat temp_mask;
for (const Range& range : ranges) {
cv::inRange(img_hsv, range.first, range.second, temp_mask);
combined_mask |= temp_mask;
}
static const cv::Mat kernel = cv::getStructuringElement(cv::MORPH_RECT, cv::Size(5, 5));
cv::morphologyEx(combined_mask, combined_mask, cv::MORPH_CLOSE, kernel);
cv::morphologyEx(combined_mask, combined_mask, cv::MORPH_OPEN, kernel);
const int matched_area = cv::countNonZero(combined_mask);
const int total_area = roiWidth * roiHeight;
const float matched_ratio = static_cast<float>(matched_area) / static_cast<float>(total_area);
return matched_ratio > area_threshold;
}
catch (const cv::Exception& e) {
return false;
}
}
bool ANSCustomFS::IsFireDetected(const cv::Mat image, const cv::Rect bBox) {
bool isFireColour = this->DetectFireNSmokeColourInFrame(image, bBox, this->_fire_colour, _hsvThreshold);
return isFireColour;
}
bool ANSCustomFS::IsSmokeDetected(const cv::Mat image, const cv::Rect bBox) {
bool isSmokeColour = this->DetectFireNSmokeColourInFrame(image, bBox, this->_smoke_colour, 0.9);
return isSmokeColour;
}
float ANSCustomFS::calculateIoU(const cv::Rect& box1, const cv::Rect& box2) {
cv::Rect intersection = box1 & box2;
int intersectionArea = intersection.area();
if (intersectionArea <= 0) return 0.0f;
int unionArea = box1.area() + box2.area() - intersectionArea;
if (unionArea <= 0) return 0.0f;
return static_cast<float>(intersectionArea) / unionArea;
}
bool ANSCustomFS::IsOverlapping(const ANSCENTER::Object& obj, const std::vector<ANSCENTER::Object>& objectList, float iouThreshold)
{
if (objectList.empty()) return false;
for (const auto& otherObj : objectList)
{
float iou = calculateIoU(obj.box, otherObj.box);
if (iou > iouThreshold)
{
return true;
}
}
return false;
}
bool ANSCustomFS::IsOverlapping(const ANSCENTER::Object& obj, const std::vector<cv::Rect>& objectList, float iouThreshold) {
if (objectList.empty()) return false;
for (const auto& otherObj : objectList)
{
float iou = calculateIoU(obj.box, otherObj);
if (iou > iouThreshold)
{
return true;
}
}
return false;
}
bool ANSCustomFS::IsROIOverlapping(const cv::Rect& obj, const std::vector<cv::Rect>& objectList, float iouThreshold) {
if (objectList.empty()) return false;
for (const auto& otherObj : objectList)
{
float iou = calculateIoU(obj, otherObj);
if (iou > iouThreshold)
{
return true;
}
}
return false;
}
// Functions for screen size division
double ANSCustomFS::calculateDistanceToCenter(const cv::Point& center, const cv::Rect& rect) {
cv::Point rectCenter(rect.x + rect.width / 2, rect.y + rect.height / 2);
return std::sqrt(std::pow(rectCenter.x - center.x, 2) + std::pow(rectCenter.y - center.y, 2));
}
std::vector<ANSCustomFS::ImageSection> ANSCustomFS::divideImage(const cv::Mat& image) {
if (image.empty()) {
std::cerr << "Error: Empty image!" << std::endl;
return cachedSections;
}
cv::Size currentSize(image.cols, image.rows);
if (currentSize == previousImageSize) {
return cachedSections;
}
previousImageSize = currentSize;
cachedSections.clear();
int width = image.cols;
int height = image.rows;
int maxDimension = std::max(width, height);
int numSections = 10;
if (maxDimension <= 2560)numSections = 8;
if (maxDimension <= 1920)numSections = 6;
if (maxDimension <= 1280)numSections = 4;
if (maxDimension <= 960)numSections = 2;
if (maxDimension <= 480)numSections = 1;
int gridRows = static_cast<int>(std::sqrt(numSections));
int gridCols = (numSections + gridRows - 1) / gridRows;
int sectionWidth = width / gridCols;
int sectionHeight = height / gridRows;
cv::Point imageCenter(width / 2, height / 2);
std::vector<std::pair<double, ImageSection>> distancePriorityList;
for (int r = 0; r < gridRows; ++r) {
for (int c = 0; c < gridCols; ++c) {
int x = c * sectionWidth;
int y = r * sectionHeight;
int w = (c == gridCols - 1) ? width - x : sectionWidth;
int h = (r == gridRows - 1) ? height - y : sectionHeight;
ImageSection section(cv::Rect(x, y, w, h));
double distance = calculateDistanceToCenter(imageCenter, section.region);
distancePriorityList.emplace_back(distance, section);
}
}
std::sort(distancePriorityList.begin(), distancePriorityList.end(),
[](const std::pair<double, ImageSection>& a, const std::pair<double, ImageSection>& b) {
if (std::abs(a.first - b.first) > 1e-5) {
return a.first < b.first;
}
return a.second.region.y == b.second.region.y
? a.second.region.x < b.second.region.x
: a.second.region.y < b.second.region.y;
});
int priority = 1;
for (auto& entry : distancePriorityList) {
entry.second.priority = priority++;
cachedSections.push_back(entry.second);
}
return cachedSections;
}
int ANSCustomFS::getHighestPriorityRegion() {
// Priorities are always assigned as 1..N (N = number of sections), so the highest
// (most important) priority is 1 whenever we have any sections. Returning a literal
// keeps this correct when RankCachedSectionsByWarmColor rewrites priorities without
// reordering cachedSections.
return cachedSections.empty() ? 0 : 1;
}
int ANSCustomFS::getLowestPriorityRegion() {
// Lowest priority (least important) is N, the total section count.
return static_cast<int>(cachedSections.size());
}
cv::Rect ANSCustomFS::getRegionByPriority(int priority) {
for (const auto& section : cachedSections) {
if (section.priority == priority) {
return section.region;
}
}
return cv::Rect();
}
std::vector<CustomObject> ANSCustomFS::RunInference(const cv::Mat& input, const std::string& camera_id) {
std::lock_guard<std::recursive_mutex> lock(_mutex);
if (input.empty() || input.cols < MIN_IMAGE_SIZE || input.rows < MIN_IMAGE_SIZE) {
return {};
}
if (!_detector) {
return {};
}
try {
GetModelParameters();
#ifdef FNS_DEBUG
cv::Mat draw = input.clone();
#else
cv::Mat draw;
#endif
std::vector<ANSCENTER::Object> output;
// A. Check if detected area is retained and valid
if (_detectedArea.width > MIN_ROI_SIZE && _detectedArea.height > MIN_ROI_SIZE) {
#ifdef FNS_DEBUG
cv::rectangle(draw, _detectedArea, cv::Scalar(0, 0, 255), 2);
for (const auto& roi : _exclusiveROIs) {
cv::rectangle(draw, roi, cv::Scalar(255, 0, 0), 2);
}
#endif
output = ProcessExistingDetectedArea(input, camera_id, draw);
}
else {
// B. Find new detected area
output = FindNewDetectedArea(input, camera_id, draw);
}
#ifdef FNS_DEBUG
DisplayDebugFrame(draw);
#endif
std::vector<CustomObject> results = ConvertToCustomObjects(output);
if (results.empty()) {
UpdateNoDetectionCondition();
}
return results;
}
catch (const std::exception& e) {
return {};
}
}
// Stage A: Process existing detected area with dual-path tracker-based voting.
//
// Dual-path design addresses the "small early fire" failure mode:
// HIGH-conf fast path: conf >= 0.5, 8-of-15 frames, gated by (movement OR flicker).
// LOW-conf slow path: conf >= 0.25, 6-of-15 frames, fire-class only,
// gated by (flicker AND fire-color present).
// The flicker gate uses temporal std-dev of HSV V inside the tracked bbox —
// real fire oscillates, sunlight reflections do not.
std::vector<ANSCENTER::Object> ANSCustomFS::ProcessExistingDetectedArea(
const cv::Mat& frame,
const std::string& camera_id,
cv::Mat& draw)
{
std::vector<ANSCENTER::Object> output;
cv::Mat activeROI = frame(_detectedArea);
// Run detector on ROI — tracker assigns persistent trackIds
std::vector<ANSCENTER::Object> detectedObjects;
_detector->RunInference(activeROI, camera_id.c_str(), detectedObjects);
if (detectedObjects.empty()) {
// Age all existing tracks (missed frame for all)
AgeTracks({});
UpdateNoDetectionCondition();
return output;
}
// Precompute full-frame HSV once so we can sample V per bbox cheaply.
cv::Mat frameHsv;
cv::cvtColor(frame, frameHsv, cv::COLOR_BGR2HSV);
// COCO filter is expensive — run it at most once per frame and reuse.
bool excludedComputed = false;
std::vector<ANSCENTER::Object> excludedObjects;
// Collect detected track IDs this frame for aging
std::unordered_set<int> detectedTrackIds;
for (auto& detectedObj : detectedObjects) {
// Adjust coordinates to full frame
detectedObj.box.x += _detectedArea.x;
detectedObj.box.y += _detectedArea.y;
detectedObj.cameraId = camera_id;
// 1. Exclusive ROI check — skip if overlapping exclusion zones
if (IsROIOverlapping(detectedObj.box, _exclusiveROIs, INCLUSIVE_IOU_THRESHOLD)) {
continue;
}
// 2. Confidence gate — dual-path entry:
// - fire (class 0) at >= LOW_CONF_FIRE_THRESHOLD enters the low-conf path
// - fire at >= _detectionScoreThreshold also enters the high-conf path
// - smoke (class 2) still requires SmokeScore threshold (high-conf only)
const bool passesLowConfFire =
(detectedObj.classId == 0) && (detectedObj.confidence >= LOW_CONF_FIRE_THRESHOLD);
const bool isValidFire =
(detectedObj.classId == 0) && (detectedObj.confidence >= _detectionScoreThreshold);
const bool isValidSmoke =
(detectedObj.classId == 2) && (detectedObj.confidence >= _smokeDetetectionThreshold);
if (!isValidFire && !isValidSmoke && !passesLowConfFire) {
continue;
}
// 2b. Minimum-size gate — reject boxes too small relative to the frame.
// Applied before voting so tiny noise never accumulates confirmation votes.
if (!IsObjectLargeEnough(detectedObj.box, frame.size())) {
continue;
}
// 3. Sample mean HSV V inside the bbox for the flicker deque
cv::Rect safeBox = detectedObj.box & cv::Rect(0, 0, frame.cols, frame.rows);
double vMean = 0.0;
if (safeBox.area() > 0) {
vMean = cv::mean(frameHsv(safeBox))[2];
}
// 4. Update track history with this detection
const int trackId = detectedObj.trackId;
detectedTrackIds.insert(trackId);
UpdateTrackHistory(trackId, detectedObj.classId, detectedObj.box, detectedObj.confidence, vMean);
#ifdef FNS_DEBUG
// Draw detection with track info
cv::Scalar color = (detectedObj.classId == 0) ? cv::Scalar(0, 255, 255) : cv::Scalar(255, 0, 255);
cv::rectangle(draw, detectedObj.box, color, 2);
auto trackIt = _trackHistory.find(trackId);
if (trackIt != _trackHistory.end()) {
std::string label = "T" + std::to_string(trackId) + " H" +
std::to_string(trackIt->second.detectedCount) + "/" +
std::to_string(VOTE_THRESHOLD) + " L" +
std::to_string(trackIt->second.lowConfDetectedCount) + "/" +
std::to_string(LOW_VOTE_THRESHOLD);
cv::putText(draw, label,
cv::Point(detectedObj.box.x, detectedObj.box.y - 10),
cv::FONT_HERSHEY_SIMPLEX, 0.5, color, 2);
}
#endif
// 5. Voting — which path (if any) confirmed this track?
const bool highConfirmed = IsTrackConfirmed(trackId);
const bool lowConfirmed = IsTrackLowConfirmed(trackId);
if (!highConfirmed && !lowConfirmed) {
continue;
}
// 6. Gate: movement / flicker / color based on path
const bool hasFlicker = HasFireFlicker(trackId);
const bool hasMovement = HasBboxMovement(trackId);
if (highConfirmed) {
// High-conf path: sunlight FP kill requires evidence of temporal variation
// — either spatial (bbox drift) OR luma flicker is enough.
// This replaces the old movement-only gate which rejected small stationary fires.
if (!hasMovement && !hasFlicker) {
continue;
}
}
else {
// Low-conf-only path: stricter gate to suppress noise
// * fire class only (smoke is too easily confused at low conf)
// * require flicker (not just drift)
// * require fire-color HSV in the bbox
if (detectedObj.classId != 0) continue;
if (!hasFlicker) continue;
if (!IsFireDetected(frame, detectedObj.box)) continue;
}
// 7. COCO filter — exclude detections that overlap with known non-fire objects
if (!excludedComputed) {
excludedObjects = FindExcludedObjects(frame, camera_id, draw);
excludedComputed = true;
}
if (!excludedObjects.empty() && IsOverlapping(detectedObj, excludedObjects, 0)) {
// Detection overlaps with a known object — not fire/smoke
_realFireCheck = std::max(0, _realFireCheck - 1);
if (_realFireCheck <= 0) {
_isRealFireFrame = false;
}
continue;
}
// All checks passed — confirmed detection
AddConfirmedDetection(detectedObj, output);
_realFireCheck++;
}
// Age out tracks not seen this frame
AgeTracks(detectedTrackIds);
if (output.empty()) {
UpdateNoDetectionCondition();
}
return output;
}
void ANSCustomFS::AddConfirmedDetection(ANSCENTER::Object& detectedObj, std::vector<ANSCENTER::Object>& output) {
// A detection that passed the dual voting + flicker + color gates is a real positive.
// ConvertToCustomObjects() downstream filters by _detectionScoreThreshold, which would
// otherwise drop low-conf fires even after they've been confirmed. Bump the confidence
// to the threshold so the confirmed detection reaches the caller.
if (detectedObj.confidence < _detectionScoreThreshold) {
detectedObj.confidence = _detectionScoreThreshold;
}
output.push_back(std::move(detectedObj));
_isFireNSmokeDetected = true;
_retainDetectedArea = 0;
_isRealFireFrame = true;
}