#include "ANSCustomFireNSmoke.h" #include #include #include #include #include using ansmind::RectF; using namespace ansmind::firesmoke; namespace { std::string trim(std::string s) { while (!s.empty() && (s.back() == '\r' || s.back() == '\n' || s.back() == ' ' || s.back() == '\t')) s.pop_back(); size_t i = 0; while (i < s.size() && (s[i] == ' ' || s[i] == '\t')) ++i; return s.substr(i); } /// ANSLIB hands the class list back as one string; comma-separated in every /// package seen so far, newline-separated in the .names file it came from. std::vector splitLabels(const std::string& s) { std::vector out; const char sep = s.find(',') != std::string::npos ? ',' : '\n'; std::stringstream ss(s); std::string item; while (std::getline(ss, item, sep)) { item = trim(item); if (!item.empty()) out.push_back(item); } return out; } float toFloat(const std::string& v, float dflt) { try { return v.empty() ? dflt : std::stof(v); } catch (...) { return dflt; } } bool toBool(const std::string& v, bool dflt) { if (v.empty()) return dflt; std::string s = v; std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return char(std::tolower(c)); }); if (s == "true" || s == "1" || s == "yes" || s == "on") return true; if (s == "false" || s == "0" || s == "no" || s == "off") return false; return dflt; } /// The value in use: what the operator set, else the parameter's default. std::string valueOf(const CustomParameter& p) { return !p.Value.empty() ? p.Value : p.DefaultValue; } void addParam(CustomParams& param, const char* name, const char* type, int decimals, int minV, int maxV, const std::string& dflt) { CustomParameter p; p.Name = name; p.DataType = type; p.NoOfDecimals = decimals; p.MaxValue = maxV; p.MinValue = minV; p.StartValue = dflt; p.DefaultValue = dflt; p.ListItems.clear(); param.Parameters.push_back(p); } } // namespace ANSCustomFS::ANSCustomFS() { char* e = nullptr; size_t n = 0; if (_dupenv_s(&e, &n, "ANSCUSTOM_FS_LOG") == 0 && e) { _fsLog = e[0] == '1'; std::free(e); } } ANSCustomFS::~ANSCustomFS() { Destroy(); } // --------------------------------------------------------------------------- // geometry // --------------------------------------------------------------------------- cv::Rect ANSCustomFS::pixelRect(const RectF& r, int W, int H) { const int x0 = std::clamp(int(std::lround(r.x * W)), 0, W); const int y0 = std::clamp(int(std::lround(r.y * H)), 0, H); const int x1 = std::clamp(int(std::lround((r.x + r.w) * W)), 0, W); const int y1 = std::clamp(int(std::lround((r.y + r.h) * H)), 0, H); return cv::Rect(x0, y0, std::max(0, x1 - x0), std::max(0, y1 - y0)); } RectF ANSCustomFS::normRect(const cv::Rect& r, int W, int H) { RectF f; f.x = float(r.x) / float(W); f.y = float(r.y) / float(H); f.w = float(r.width) / float(W); f.h = float(r.height) / float(H); return f.clamped(); } float ANSCustomFS::cover(const RectF& a, const RectF& b) { const float ix = std::max(0.f, std::min(a.x + a.w, b.x + b.w) - std::max(a.x, b.x)); const float iy = std::max(0.f, std::min(a.y + a.h, b.y + b.h) - std::max(a.y, b.y)); return a.area() > 0 ? ix * iy / a.area() : 0.f; } void ANSCustomFS::nmsClassAgnostic(std::vector& dets, float iouThr) { std::sort(dets.begin(), dets.end(), [](const Det& a, const Det& b) { return a.conf > b.conf; }); std::vector kept; for (const auto& d : dets) { bool dup = false; for (const auto& k : kept) if (k.box.iou(d.box) > iouThr) { dup = true; break; } if (!dup) kept.push_back(d); } dets.swap(kept); } // --------------------------------------------------------------------------- // the models // --------------------------------------------------------------------------- bool ANSCustomFS::runDetector(const cv::Mat& bgr, const cv::Rect& roi, std::vector& out, const std::string& camera) { if (!_detector || roi.width < 2 || roi.height < 2) return false; const bool whole = roi.x == 0 && roi.y == 0 && roi.width == bgr.cols && roi.height == bgr.rows; // A ROI view is not continuous; ANSLIB wants a plain image. cv::Mat sub = whole ? bgr : bgr(roi).clone(); std::vector objs; const int rc = _detector->RunInference(sub, camera.c_str(), objs); (void)rc; // ANSLIB reports through its own log; an empty list is the answer either way for (const auto& o : objs) { Det d; d.box = normRect(cv::Rect(o.box.x + roi.x, o.box.y + roi.y, o.box.width, o.box.height), bgr.cols, bgr.rows); d.classId = o.classId; d.conf = o.confidence; if (d.box.w > 0 && d.box.h > 0) out.push_back(d); } return true; } bool ANSCustomFS::runFilter(const cv::Mat& bgr, const cv::Rect& roi, std::vector& out, const std::string& camera) { if (!_filter || roi.width < 2 || roi.height < 2) return false; const bool whole = roi.x == 0 && roi.y == 0 && roi.width == bgr.cols && roi.height == bgr.rows; cv::Mat sub = whole ? bgr : bgr(roi).clone(); std::vector objs; const int rc = _filter->RunInference(sub, camera.c_str(), objs); (void)rc; for (const auto& o : objs) { Det d; d.box = normRect(cv::Rect(o.box.x + roi.x, o.box.y + roi.y, o.box.width, o.box.height), bgr.cols, bgr.rows); d.classId = o.classId; d.conf = o.confidence; if (d.box.w > 0 && d.box.h > 0) out.push_back(d); } return true; } Cues ANSCustomFS::cuesFor(const cv::Mat& bgr, const RectF& box) const { const cv::Rect r = pixelRect(box, bgr.cols, bgr.rows); if (r.width < 2 || r.height < 2) return Cues{}; const uint8_t* p = bgr.ptr(r.y) + size_t(r.x) * 3; return patchCues(p, r.width, r.height, int(bgr.step)); } // --------------------------------------------------------------------------- // lifecycle // --------------------------------------------------------------------------- bool ANSCustomFS::Initialize(const std::string& modelDirectory, float detectionScoreThreshold, std::string& labelMap) { std::lock_guard lock(_mutex); try { _modelDirectory = modelDirectory; _detectionScoreThreshold = detectionScoreThreshold; std::cerr << "[ANSCustomFS] Initialize: model dir " << modelDirectory << std::endl; _detector = ANSLIBPtr(ANSCENTER::ANSLIB::Create(), &ANSCENTER::ANSLIB::Destroy); _filter = ANSLIBPtr(ANSCENTER::ANSLIB::Create(), &ANSCENTER::ANSLIB::Destroy); if (!_detector || !_filter) { std::cerr << "ANSCustomFS::Initialize: ANSLIB::Create failed." << std::endl; return false; } // NVIDIA -> TensorRT, anything else -> OpenVINO; both plain detectors. const int engineType = _detector->GetEngineType(); std::cerr << "[ANSCustomFS] engine type " << engineType << " -> " << (engineType == 1 ? "TensorRT" : "OpenVINO") << std::endl; const int modelType = engineType == 1 ? 31 : 30; const int detectionType = 1; // The engines are built or loaded HERE, whatever the caller set with // SetLoadEngineOnCreate: a lazily loaded engine faulted inside // ANSLIB's inference entry on the first frame (ANSLIB.cpp:160), and // the ans-custom-host asks for load-on-create anyway. The first run // on a GPU builds TensorRT engines for minutes; they cache beside the // models. const int loadEngineOnCreation = 1; (void)_loadEngineOnCreate; const int autoEngineDetection = 1; const std::string licenseKey; // The raw pass must hear the detector's QUIET verdicts too: a small // fire is 0.2 on the whole frame long before it is 0.5, and a fake_* // at 0.2 is still a fake. The alarm threshold is the verifier's // evidence, never this floor. const float floor = _p.minConf; const int detResult = _detector->LoadModelFromFolder( licenseKey.c_str(), "detector", "detector.names", floor, floor, 0.5f, autoEngineDetection, modelType, detectionType, loadEngineOnCreation, modelDirectory.c_str(), labelMap); if (detResult != 1) { std::cerr << "ANSCustomFS::Initialize: failed to load the detector from " << modelDirectory << std::endl; return false; } std::cerr << "[ANSCustomFS] detector loaded: " << labelMap << std::endl; _labels = splitLabels(labelMap); _roles.clear(); _fireClass = _smokeClass = -1; _fireName.clear(); _smokeName.clear(); for (size_t i = 0; i < _labels.size(); ++i) { const Role r = roleOf(_labels[i]); _roles.push_back(r); if (r == Role::Fire && _fireClass < 0) { _fireClass = int(i); _fireName = _labels[i]; } if (r == Role::Smoke && _smokeClass < 0) { _smokeClass = int(i); _smokeName = _labels[i]; } } if (_fireClass < 0 && _smokeClass < 0) { std::cerr << "ANSCustomFS::Initialize: the detector's classes name neither fire nor smoke: " << labelMap << std::endl; return false; } // The look-alike filter (COCO) is optional: without it vehicles and // screens are simply not explained away. std::string filterLabels; const int filterResult = _filter->LoadModelFromFolder( licenseKey.c_str(), "filter", "filter.names", 0.25f, 0.25f, 0.5f, autoEngineDetection, modelType, detectionType, loadEngineOnCreation, modelDirectory.c_str(), filterLabels); if (filterResult != 1) { std::cerr << "ANSCustomFS::Initialize: no look-alike filter model (filter.onnx) - continuing without it." << std::endl; _filter.reset(); filterLabels.clear(); } std::cerr << "[ANSCustomFS] filter " << (_filter ? "loaded: " + filterLabels.substr(0, 60) : "absent") << std::endl; _filterLabels = splitLabels(filterLabels); // Which filter classes explain a look-alike away: by NAME when the // package ships filter.names, by COCO index when it does not. const size_t nc = _filterLabels.empty() ? 80 : _filterLabels.size(); _vehicle.assign(nc, 0); _screen.assign(nc, 0); if (_filterLabels.empty()) { for (int i : {1, 2, 3, 5, 7}) _vehicle[size_t(i)] = 1; for (int i : {9, 62, 63, 67}) _screen[size_t(i)] = 1; } else { for (size_t i = 0; i < _filterLabels.size(); ++i) { const std::string& s = _filterLabels[i]; if (s == "bicycle" || s == "car" || s == "motorcycle" || s == "bus" || s == "truck") _vehicle[i] = 1; if (s == "tv" || s == "laptop" || s == "cell phone" || s == "traffic light") _screen[i] = 1; } } _verifier = Verifier(_p); _paramSignature.clear(); _lastResults.clear(); _lastRunT = -1e9; _clockStarted = false; _frames = 0; return true; } catch (const std::exception& e) { std::cerr << "ANSCustomFS::Initialize: exception: " << e.what() << std::endl; return false; } catch (...) { std::cerr << "ANSCustomFS::Initialize: unknown exception." << std::endl; return false; } } bool ANSCustomFS::OptimizeModel(bool fp16) { std::lock_guard lock(_mutex); if (!_detector) return false; const int d = _detector->Optimize(fp16); const int f = _filter ? _filter->Optimize(fp16) : 1; return d == 1 && f == 1; } bool ANSCustomFS::Destroy() { std::lock_guard lock(_mutex); try { _detector.reset(); _filter.reset(); _lastResults.clear(); return true; } catch (...) { return false; } } // --------------------------------------------------------------------------- // parameters // --------------------------------------------------------------------------- bool ANSCustomFS::ConfigureParameters(CustomParams& param) { std::lock_guard lock(_mutex); param.ROI_Config.clear(); if (this->_params.ROI_Config.empty()) { CustomROIConfig exclusiveROI; exclusiveROI.Name = "ExclusiveROIs"; exclusiveROI.Polygon = false; exclusiveROI.Rectangle = true; exclusiveROI.Line = false; exclusiveROI.MinItems = 0; exclusiveROI.MaxItems = 20; exclusiveROI.ROIMatch = "All Corners"; param.ROI_Config.push_back(exclusiveROI); } else { for (auto& roi : this->_params.ROI_Config) param.ROI_Config.push_back(roi); } param.ROI_Options.clear(); param.Parameters.clear(); // One operator-facing knob for how much evidence an alarm needs (0 = the // fewest false alarms, 1 = the earliest alarm; the physical gates never // move), the wildfire place memory, which families to watch, and the // cost levers. addParam(param, "Sensitivity", "float", 2, 0, 1, "0.5"); addParam(param, "SmokeSeconds", "float", 0, 0, 600, "0"); addParam(param, "DetectFire", "Boolean", 0, 0, 1, "true"); addParam(param, "DetectSmoke", "Boolean", 0, 0, 1, "true"); addParam(param, "ZoomChecks", "float", 0, 0, 8, "2"); addParam(param, "MaxFps", "float", 0, 1, 30, "6"); addParam(param, "CandidateScore", "float", 2, 0, 1, "0.15"); // Minimum candidate size as a PERCENTAGE of the frame area (0.03 = 0.03 %): // resolution-independent, rejects the tiny boxes that were the dominant // small false-positive source on low-resolution cameras. addParam(param, "MinObjectSize", "float", 2, 0, 10, "0.03"); param.ROI_Values.clear(); for (auto& roi : this->_params.ROI_Values) { CustomROIValue roiValue; roiValue.Name = roi.Name; roiValue.ROIPoints = roi.ROIPoints; roiValue.ROIMatch = roi.ROIMatch; roiValue.Option = roi.Option; roiValue.OriginalImageSize = roi.OriginalImageSize; param.ROI_Values.push_back(roiValue); } return true; } void ANSCustomFS::applyParams() { // Cheap to recompute, so it runs every frame and only acts on a change: // ANSVIS and ans-custom-host both push values through SetParameters // without telling the plugin. std::string sig; for (const auto& p : _params.Parameters) sig += p.Name + "=" + valueOf(p) + ";"; sig += "rois=" + std::to_string(_params.ROI_Values.size()); for (const auto& r : _params.ROI_Values) for (const auto& pt : r.ROIPoints) sig += "," + std::to_string(pt.x) + ":" + std::to_string(pt.y); if (sig == _paramSignature) return; _paramSignature = sig; Params p; // the node's defaults float sensitivity = 0.5f; for (const auto& prm : _params.Parameters) { const std::string v = valueOf(prm); if (prm.Name == "Sensitivity") sensitivity = std::clamp(toFloat(v, 0.5f), 0.f, 1.f); else if (prm.Name == "SmokeSeconds") p.smokePersistSeconds = std::clamp(toFloat(v, 0.f), 0.f, 600.f); else if (prm.Name == "DetectFire") p.fire = toBool(v, true); else if (prm.Name == "DetectSmoke") p.smoke = toBool(v, true); else if (prm.Name == "ZoomChecks") _zoomBudget = std::clamp(int(std::lround(toFloat(v, 2.f))), 0, 8); else if (prm.Name == "MaxFps") _maxFps = std::clamp(double(toFloat(v, 6.f)), 1.0, 30.0); else if (prm.Name == "CandidateScore") p.minConf = std::clamp(toFloat(v, 0.15f), 0.05f, 0.9f); else if (prm.Name == "MinObjectSize") _minObjectAreaRatio = std::clamp(toFloat(v, 0.03f) / 100.f, 0.f, 1.f); } // The same three settings the ANSMIND node offers, on a slider. if (sensitivity > 0.66f) { // high p.confirm = 1.4f; p.minSeconds = 0.5f; p.smokeMinSeconds = 2.0f; } else if (sensitivity < 0.34f) { // low p.confirm = 3.2f; p.minSeconds = 2.0f; p.smokeMinSeconds = 5.0f; } // else balanced: the defaults if (!p.fire && !p.smoke) p.fire = p.smoke = true; // both off is nobody's intent _p = p; _verifier.configure(_p); _exclusiveROIs.clear(); for (const auto& roi : _params.ROI_Values) { if (roi.Name.find("ExclusiveROIs") == std::string::npos || roi.ROIPoints.empty()) continue; int x0 = roi.ROIPoints[0].x, y0 = roi.ROIPoints[0].y, x1 = x0, y1 = y0; for (const auto& pt : roi.ROIPoints) { x0 = std::min(x0, pt.x); y0 = std::min(y0, pt.y); x1 = std::max(x1, pt.x); y1 = std::max(y1, pt.y); } if (x1 > x0 && y1 > y0) _exclusiveROIs.emplace_back(x0, y0, x1 - x0, y1 - y0); } } // --------------------------------------------------------------------------- // a frame // --------------------------------------------------------------------------- std::vector ANSCustomFS::RunInference(const cv::Mat& input) { return RunInference(input, "CustomCam"); } std::vector ANSCustomFS::RunInference(const cv::Mat& input, const std::string& camera_id) { std::lock_guard lock(_mutex); if (input.empty() || input.cols < 16 || input.rows < 16) return {}; if (!_detector) return {}; try { applyParams(); // The verifier reads BGR8 rows; anything else is converted once. cv::Mat bgr; if (input.type() == CV_8UC3) bgr = input; else if (input.type() == CV_8UC1) cv::cvtColor(input, bgr, cv::COLOR_GRAY2BGR); else if (input.type() == CV_8UC4) cv::cvtColor(input, bgr, cv::COLOR_BGRA2BGR); else input.convertTo(bgr, CV_8UC3); const int W = bgr.cols, H = bgr.rows; // Time: the replay's own clock when a harness set one, else real time. double t; if (_externalClock >= 0) { t = _externalClock; } else { const auto now = std::chrono::steady_clock::now(); if (!_clockStarted) { _t0 = now; _clockStarted = true; } t = std::chrono::duration(now - _t0).count(); } // The detector is not run faster than MaxFps: evidence is per second // of scene, not per frame, and a camera at 25 fps shows the verifier // nothing a 6 fps sample does not. Between runs the last verdict holds. if (t - _lastRunT < 1.0 / _maxFps - 1e-6) return _lastResults; _lastRunT = t; ++_frames; const Gray gray = smallGray(bgr.data, W, H, int(bgr.step)); _verifier.beginFrame(gray, t, float(W) / float(std::max(1, H))); // 1. the whole frame, heard at a low floor, one box per object // whatever the class: fire and fake_fire on the same flame are ONE // candidate, and which of the two the model preferred is the // evidence. std::vector raw; runDetector(bgr, cv::Rect(0, 0, W, H), raw, camera_id); nmsClassAgnostic(raw, 0.5f); std::vector cands; for (const auto& d : raw) { if (d.classId < 0 || size_t(d.classId) >= _roles.size()) continue; Candidate c; c.box = d.box.clamped(); c.role = _roles[size_t(d.classId)]; c.conf = d.conf; if (c.role == Role::Other || c.box.w <= 0 || c.box.h <= 0) continue; if (c.box.area() < _minObjectAreaRatio) continue; if (!_exclusiveROIs.empty()) { const cv::Rect px = pixelRect(c.box, W, H); bool excluded = false; for (const auto& ex : _exclusiveROIs) if ((px & ex).area() > 0) { excluded = true; break; } if (excluded) continue; } cands.push_back(c); } std::vector alarms; if (cands.empty()) { _verifier.update(cands, alarms); _lastResults.clear(); return _lastResults; } // 2. look-alike objects on the whole frame - only now that there is a // candidate to explain, which on a quiet camera is almost never. std::vector objects; if (_filter) runFilter(bgr, cv::Rect(0, 0, W, H), objects, camera_id); // 3. cues for every candidate; a native-resolution re-check for the // few small fire candidates the plan picks. for (auto& c : cands) { RectF pb{ c.box.x - 0.1f * c.box.w, c.box.y - 0.1f * c.box.h, c.box.w * 1.2f, c.box.h * 1.2f }; c.cues = cuesFor(bgr, pb.clamped()); } const double nativeMin = double(std::min(W, H)); for (int i : _verifier.planZoom(cands, _zoomBudget)) { Candidate& c = cands[size_t(i)]; const double bw = c.box.w * W, bh = c.box.h * H; const double side = std::min(std::max(3.0 * std::max(bw, bh), 320.0), nativeMin); // Only a real MAGNIFICATION is evidence. A "zoom" that is most of // the frame shows the model what it has already seen, minus the // context. if (side > 0.5 * nativeMin) continue; RectF roi; roi.w = float(side / W); roi.h = float(side / H); roi.x = std::clamp(c.box.x + c.box.w / 2 - roi.w / 2, 0.f, 1.f - roi.w); roi.y = std::clamp(c.box.y + c.box.h / 2 - roi.h / 2, 0.f, 1.f - roi.h); const cv::Rect px = pixelRect(roi, W, H); std::vector z; if (!runDetector(bgr, px, z, camera_id)) continue; c.zoomValid = true; const float ccx = c.box.x + c.box.w / 2, ccy = c.box.y + c.box.h / 2; for (const auto& d : z) { if (d.classId < 0 || size_t(d.classId) >= _roles.size()) continue; const bool hit = d.box.iou(c.box) > 0.1f || (ccx >= d.box.x && ccx <= d.box.x + d.box.w && ccy >= d.box.y && ccy <= d.box.y + d.box.h); if (!hit) continue; const Role r = _roles[size_t(d.classId)]; if (r == Role::Fire) c.zoomReal = std::max(c.zoomReal, d.conf); if (r == Role::FakeFire) c.zoomFake = std::max(c.zoomFake, d.conf); } if (_filter) runFilter(bgr, px, objects, camera_id); } for (auto& c : cands) for (const auto& o : objects) { if (o.classId < 0 || size_t(o.classId) >= _vehicle.size()) continue; if (cover(c.box, o.box) <= 0.6f) continue; if (_vehicle[size_t(o.classId)]) c.onVehicle = true; // A screen that fills the view is somebody TESTING with a // replay, and they are owed their alarm; a small one is a TV. if (_screen[size_t(o.classId)] && o.box.area() < 0.4f) c.onScreen = true; } // 4. judgement _verifier.update(cands, alarms); std::vector results; results.reserve(alarms.size()); for (const auto& a : alarms) { const int cls = a.smoke ? _smokeClass : _fireClass; if (cls < 0) continue; CustomObject co; co.classId = cls; co.className = a.smoke ? _smokeName : _fireName; // A confirmed track is a real positive; the caller's score filter // must not drop it, so the reported confidence never sits under // the threshold the model was opened with. co.confidence = std::max(std::clamp(a.confidence, 0.f, 1.f), _detectionScoreThreshold); co.box = pixelRect(a.box, W, H); co.trackId = int(a.trackId & 0x7fffffff); co.cameraId = camera_id; char buf[160]; std::snprintf(buf, sizeof buf, "fs_seconds=%.1f;fs_track=%llu;evidence=%.2f;held=%d", a.seconds, static_cast(a.trackId), a.evidence, a.held ? 1 : 0); co.extraInfo = buf; results.push_back(std::move(co)); } if (_fsLog) { const Motion& m = _verifier.motion(); for (const auto& d : _verifier.debug()) std::fprintf(stderr, "[fs] t=%.2f gm=(%.3f,%+.3f,%+.3f q%.2f) #%llu %s %s ev=%.2f obs=%d dyn=%.2f " "flame=%.2f core=%.2f red=%.2f zoom=%.2f veh=%.2f scr=%.2f drift=%.2f blocked=%s\n", t, m.s, m.tx, m.ty, m.q, static_cast(d.id), d.smoke ? "S" : "F", d.confirmed ? "ALARM" : "-", d.evidence, d.obs, d.dyn, d.flame, d.core, d.red, d.zoom, d.vehicle, d.screen, d.drift, d.blockedBy); } _lastResults = results; return results; } catch (const std::exception& e) { std::cerr << "ANSCustomFS::RunInference: exception: " << e.what() << std::endl; return {}; } catch (...) { return {}; } }