#include"ANSFireNSmoke.h" namespace ANSCENTER { // Normalize GLCM void GLCM::normalizeGLCM(cv::Mat& glcm) { double sum = cv::sum(glcm)[0]; if (sum > 0) { glcm /= sum; } } std::vector GLCM::graycomatrix(const cv::Mat& image, const std::vector& distances, const std::vector& angles, int levels, bool symmetric , bool normed) { // Validate input if (image.type() != CV_8UC1) { return std::vector(); } if (levels <= 0 || levels > 256) { return std::vector(); } int rows = image.rows; int cols = image.cols; // Create a 4D GLCM tensor (levels x levels x distances x angles) std::vector glcmMatrices(distances.size() * angles.size(), cv::Mat::zeros(levels, levels, CV_32F)); // Iterate through distances and angles for (size_t d = 0; d < distances.size(); ++d) { int distance = distances[d]; for (size_t a = 0; a < angles.size(); ++a) { float angle = angles[a]; int dx = static_cast(distance * std::cos(angle)); int dy = static_cast(distance * std::sin(angle)); // Compute GLCM for the specific distance and angle cv::Mat& glcm = glcmMatrices[d * angles.size() + a]; for (int y = 0; y < rows; ++y) { for (int x = 0; x < cols; ++x) { int x2 = x + dx; int y2 = y + dy; if (x2 >= 0 && x2 < cols && y2 >= 0 && y2 < rows) { int i = image.at(y, x); int j = image.at(y2, x2); glcm.at(i, j)++; } } } // Make GLCM symmetric if required if (symmetric) { cv::Mat transposed; cv::transpose(glcm, transposed); glcm += transposed; } // Normalize GLCM if required if (normed) { normalizeGLCM(glcm); } } } return glcmMatrices; } // Function to calculate texture properties std::vector GLCM::graycoprops(const std::vector& glcmMatrices, const std::string& property) { std::vector results; for (const auto& glcm : glcmMatrices) { // Ensure the GLCM is normalized cv::Mat normalizedGLCM = glcm.clone(); normalizeGLCM(normalizedGLCM); int levels = normalizedGLCM.rows; cv::Mat weights; // Define weights based on the property if (property == "energy") { double asmValue = cv::sum(normalizedGLCM.mul(normalizedGLCM))[0]; results.push_back(std::sqrt(asmValue)); continue; } else if (property == "homogeneity") { weights = cv::Mat::zeros(levels, levels, CV_32F); for (int i = 0; i < levels; ++i) { for (int j = 0; j < levels; ++j) { weights.at(i, j) = 1.0f / (1.0f + (i - j) * (i - j)); } } } else if (property == "dissimilarity") { weights = cv::Mat::zeros(levels, levels, CV_32F); for (int i = 0; i < levels; ++i) { for (int j = 0; j < levels; ++j) { weights.at(i, j) = std::abs(i - j); } } } else if (property == "contrast") { weights = cv::Mat::zeros(levels, levels, CV_32F); for (int i = 0; i < levels; ++i) { for (int j = 0; j < levels; ++j) { weights.at(i, j) = (i - j) * (i - j); } } } else if (property == "correlation") { double meanI = 0.0, meanJ = 0.0, stdI = 0.0, stdJ = 0.0, correlation = 0.0; for (int i = 0; i < levels; ++i) { for (int j = 0; j < levels; ++j) { double value = normalizedGLCM.at(i, j); meanI += i * value; meanJ += j * value; } } for (int i = 0; i < levels; ++i) { for (int j = 0; j < levels; ++j) { double value = normalizedGLCM.at(i, j); stdI += (i - meanI) * (i - meanI) * value; stdJ += (j - meanJ) * (j - meanJ) * value; } } stdI = std::sqrt(stdI); stdJ = std::sqrt(stdJ); for (int i = 0; i < levels; ++i) { for (int j = 0; j < levels; ++j) { double value = normalizedGLCM.at(i, j); correlation += ((i - meanI) * (j - meanJ) * value); } } if (stdI > 0 && stdJ > 0) { correlation /= (stdI * stdJ); } else { correlation = 1.0; // Handle near-zero standard deviation } results.push_back(correlation); continue; } else if (property == "ASM") { double asmValue = cv::sum(normalizedGLCM.mul(normalizedGLCM))[0]; results.push_back(asmValue); continue; } else { return std::vector(); } // Compute the property based on the weights if (!weights.empty()) { double result = cv::sum(normalizedGLCM.mul(weights))[0]; results.push_back(result); } } return results; } bool ANSFIRENSMOKE::OptimizeModel(bool fp16, std::string& optimizedModelFolder) { if (_engineType == EngineType::NVIDIA_GPU) // NVIDIA CUDA { return _gpuObjectDetector.OptimizeModel(fp16, optimizedModelFolder); } else { return _cpuObjectDetector.OptimizeModel(fp16, optimizedModelFolder); } } bool ANSFIRENSMOKE::LoadModel(const std::string& modelZipFilePath, const std::string& modelZipPassword) { try { if (_engineType == EngineType::NVIDIA_GPU) // NVIDIA CUDA { return _gpuObjectDetector.LoadModel(modelZipFilePath, modelZipPassword); } else { return _cpuObjectDetector.LoadModel(modelZipFilePath, modelZipPassword); } } catch (std::exception& e) { this->_logger.LogFatal("ANSFIRENSMOKE::LoadModel", e.what(), __FILE__, __LINE__); return false; } } bool ANSFIRENSMOKE::LoadModelFromFolder(std::string licenseKey, ModelConfig modelConfig, std::string modelName, const std::string& modelFolder, std::string& labelMap) { try { if (_engineType == EngineType::NVIDIA_GPU) // NVIDIA CUDA { return _gpuObjectDetector.LoadModelFromFolder(licenseKey, modelConfig,modelName,modelFolder,labelMap); } else { return _cpuObjectDetector.LoadModelFromFolder(licenseKey, modelConfig,modelName, modelFolder, labelMap); } } catch (std::exception& e) { this->_logger.LogFatal("ANSFIRENSMOKE::LoadModel", e.what(), __FILE__, __LINE__); return false; } } bool ANSFIRENSMOKE::Initialize(std::string licenseKey, ModelConfig modelConfig, const std::string& modelZipFilePath, const std::string& modelZipPassword, std::string& labelMap) { std::lock_guard lock(_mutex); try { ANSODBase::CheckLicense(); if (!_licenseValid) return false; _modelConfig = modelConfig; _engineType = ANSLicenseHelper::CheckHardwareInformation(); if (_engineType == EngineType::NVIDIA_GPU) // NVIDIA CUDA { _modelConfig.detectionType = DetectionType::DETECTION; _modelConfig.modelType = ModelType::YOLOV10RTOD; _isInitialized = _gpuObjectDetector.Initialize(licenseKey, _modelConfig, modelZipFilePath, modelZipPassword, labelMap); } else { _modelConfig.detectionType = DetectionType::DETECTION; _modelConfig.modelType = ModelType::YOLOV10OVOD; _isInitialized = _cpuObjectDetector.Initialize(licenseKey, _modelConfig, modelZipFilePath, modelZipPassword, labelMap); } //this->_hsvThreshold = 0.05; //this->_fire_colour.push_back(std::pair( // cv::Scalar(0, 10, 200), cv::Scalar(40, 255, 255)) //); //this->_smoke_colour.push_back(std::pair( // cv::Scalar(0, 0, 51), cv::Scalar(180, 60, 204)) //(0,0,80)(0,0,51) -> (180,60,200) or (180, 128, 204) //); this->_hsvThreshold = 0.25; this->_fire_colour.push_back(std::pair( cv::Scalar(0, 0, 179), cv::Scalar(255, 255, 255)) // 0,0,179 -> 255, 255, 255 ); this->_smoke_colour.push_back(std::pair( cv::Scalar(0, 0, 51), cv::Scalar(180, 128, 204)) // 0,0,0 -> 255, 255, 127 ); this->_previousDetectedArea = cv::Rect(0, 0, 0, 0); _detectedArea.width = 0; _detectedArea.height = 0; _classifierModelPath = "C:\\ProgramData\\ANSCENTER\\Shared\\FireNSmoke.ann"; if (FileExist(_classifierModelPath)) { _annhub.Init(licenseKey, _classifierModelPath); _classifierInitialized = true; std::string colourSpace = "RGB"; std::vector properties = { "energy","homogeneity","dissimilarity","contrast","correlation"}; _extractor.Init(colourSpace, properties); } return _isInitialized; } catch (std::exception& e) { this->_logger.LogFatal("ANSFIRENSMOKE::Initialize", e.what(), __FILE__, __LINE__); return false; } } std::vector ANSFIRENSMOKE::ProcecssDetection(const std::vector& detectedObjects, const std::string& camera_id, int threshold) { // Get detection queue std::deque> detectionQueue=DequeueDetection(camera_id); // Step 1: Initialize vector to store frequently appearing objects std::vector frequentObjects; // Step 2: Check each object in detectedObjects for (const auto& detectedObj : detectedObjects) { int occurrenceCount = 0; // Check if the detectedObj appears at least 8 times in the queue for (const auto& queueDetection : detectionQueue) { for (const auto& queueObj : queueDetection) { if (isOverlayObject(detectedObj, queueObj)) { occurrenceCount++; break; // Found a match in this detection, move to the next queue element } } if (occurrenceCount >= threshold) break; // No need to check further if already counted 8 } if (occurrenceCount >= threshold) { frequentObjects.push_back(detectedObj); } } // Return the array of frequently appearing objects return frequentObjects; } cv::Rect ANSFIRENSMOKE::CreateMinimumSquareBoundingBox(const std::vector& detectedObjects, int minSize) { // Ensure there are rectangles to process int adMinSize = minSize-20; if (adMinSize < 0) adMinSize = 0; if (detectedObjects.empty()) return cv::Rect(0, 0, minSize, minSize); // Initialize min and max coordinates int minX = detectedObjects[0].box.x; int minY = detectedObjects[0].box.y; int maxX = detectedObjects[0].box.x + detectedObjects[0].box.width; int maxY = detectedObjects[0].box.y + detectedObjects[0].box.height; // Calculate bounding box that includes all rectangles for (const auto& rect : detectedObjects) { minX = std::min(minX, rect.box.x); minY = std::min(minY, rect.box.y); maxX = std::max(maxX, rect.box.x + rect.box.width); maxY = std::max(maxY, rect.box.y + rect.box.height); } // Calculate width and height of the bounding box int width = maxX - minX; int height = maxY - minY; // Determine the size of the square int squareSize = std::max({ width, height, adMinSize }); // Center the square around the bounding box int centerX = minX + width / 2; int centerY = minY + height / 2; int squareX = centerX - squareSize / 2; int squareY = centerY - squareSize / 2; return cv::Rect(squareX - 10, squareY - 10, squareSize + 20, squareSize + 20); // add 10 pixels to the square } std::vector ANSFIRENSMOKE::RunInference(const cv::Mat& input) { return RunInference(input, "SmokeCam"); } std::vector ANSFIRENSMOKE::RunInference(const cv::Mat& input, const std::string& camera_id) { std::lock_guard lock(_mutex); std::vector output; output.clear(); if (!_licenseValid) { this->_logger.LogError("ANSFIRENSMOKE::RunInference", "Invalid License", __FILE__, __LINE__); return output; } if (!_isInitialized) { this->_logger.LogError("ANSFIRENSMOKE::RunInference", "Model is not initialized", __FILE__, __LINE__); return output; } try { if (input.empty()) return output; if ((input.cols < 10) || (input.rows < 10)) return output; //1. Check movement detection on entired image std::vector movementObjects = DetectMovement(input, camera_id); if (!movementObjects.empty()) { std::vector smokeNSmokeRects; //2. Check if the moving object is a for (Object& obj : movementObjects) { //bool fcol_detected = this->MajorityColourInFrame(obj.mask, this->_fire_colour, _hsvThreshold); //bool scol_detected = this->MajorityColourInFrame(obj.mask, this->_smoke_colour, 0.85); //if (fcol_detected || scol_detected) { // smokeNSmokeRects.push_back(obj); //} smokeNSmokeRects.push_back(obj); } int detectedArea = _detectedArea.width* _detectedArea.height; if ((!smokeNSmokeRects.empty())|| (detectedArea>0)) { if ((_detectedArea.width < 10) || (_detectedArea.height <10)) { _detectedArea = CreateMinimumSquareBoundingBox(smokeNSmokeRects, 640); int x1_new = _detectedArea.x; int y1_new = _detectedArea.y; _detectedArea.x = std::max(x1_new, 0); _detectedArea.y = std::max(y1_new, 0); _detectedArea.width = std::min(640, input.cols - _detectedArea.x); _detectedArea.height = std::min(640, input.rows - _detectedArea.y); } //cv::Mat draw = input.clone(); //// draw rectangles //for (const auto& rect : smokeNSmokeRects) { // cv::rectangle(draw, rect.box, cv::Scalar(0, 255, 0), 2); //} //cv::rectangle(draw, _detectedArea, cv::Scalar(0, 0, 255), 2); //// Diplsay the frame with the combined detected areas //cv::imshow("Combined Detected Areas", draw); //cv::waitKey(1); //draw.release(); cv::Mat frame = input.clone(); std::vector detectedObjects; detectedObjects.clear(); if (_detectedArea.width* _detectedArea.height > 0) { cv::Mat detectedObj = frame(_detectedArea).clone(); // Get sub-image of the detected area if (_engineType == EngineType::NVIDIA_GPU) { // NVIDIA CUDA detectedObjects = _gpuObjectDetector.RunInference(detectedObj); } else { detectedObjects = _cpuObjectDetector.RunInference(detectedObj); } detectedObj.release(); } frame.release(); // Do detections float maxScore = 0.0; if (!detectedObjects.empty()) { for (auto& detectedObj : detectedObjects) { detectedObj.box.x += _detectedArea.x; detectedObj.box.y += _detectedArea.y; detectedObj.cameraId = camera_id; if (detectedObj.classId != 1) { if (detectedObj.classId == 2) { if (detectedObj.confidence > 0.5)output.push_back(detectedObj); } else output.push_back(detectedObj); if (maxScore < detectedObj.confidence) { int cropSize = 640; int imagegSize = std::max(input.cols, input.rows); if (imagegSize > 1920) cropSize = 640; else if (imagegSize > 1280) cropSize = 480; else if (imagegSize > 640) cropSize = 320; else cropSize = 224; maxScore = detectedObj.confidence; int x1 = detectedObj.box.x; int y1 = detectedObj.box.y; int xc = x1 + detectedObj.box.width / 2; int yc = y1 + detectedObj.box.height / 2; int x1_new = std::max(xc - cropSize/2, 0); int y1_new = std::max(yc - cropSize/2, 0); x1_new = std::min(x1_new, input.cols- cropSize); y1_new = std::min(y1_new, input.rows- cropSize); _detectedArea.x = std::max(x1_new, 0); _detectedArea.y = std::max(y1_new, 0); _detectedArea.width = std::min(cropSize, input.cols - _detectedArea.x); _detectedArea.height = std::min(cropSize, input.rows - _detectedArea.y); } _isFireNSmokeDetected = true; _retainDetectedArea = 0;// Reset the retain detected area } else { if (_isFireNSmokeDetected) { _retainDetectedArea++; if (_retainDetectedArea >= 40) {// Reset detected area after 10 frames _detectedArea.width = 0; _detectedArea.height = 0; _retainDetectedArea = 0; _isFireNSmokeDetected = false;// Reset the retain detected area } } else { _detectedArea.width = 0; _detectedArea.height = 0; _retainDetectedArea = 0; } } } } else { if (_isFireNSmokeDetected) { _retainDetectedArea++; if (_retainDetectedArea >= 40) {// Reset detected area after 10 frames _detectedArea.width = 0; _detectedArea.height = 0; _retainDetectedArea = 0; _isFireNSmokeDetected = false;// Reset the retain detected area } } else { _detectedArea.width = 0; _detectedArea.height = 0; _retainDetectedArea = 0; } } } } // Always adding the detection to the queue EnqueueDetection(output, camera_id); std::vector frequentObjects = ProcecssDetection(output, camera_id, SMOKE_THRESHOLD_SIZE); return frequentObjects; } catch (std::exception& e) { this->_logger.LogFatal("ANSFIRENSMOKE::RunInference", e.what(), __FILE__, __LINE__); return output; } } // For debug /* std::vector ANSFIRENSMOKE::RunInference(const cv::Mat& input, const std::string& camera_id) { std::lock_guard lock(_mutex); std::vector output; output.clear(); if (!_licenseValid) { this->_logger.LogError("ANSFIRENSMOKE::RunInference", "Invalid License", __FILE__, __LINE__); return output; } if (!_isInitialized) { this->_logger.LogError("ANSFIRENSMOKE::RunInference", "Model is not initialized", __FILE__, __LINE__); return output; } try { //1. Check movement detection on entired image std::vector stablizedMovementObjects = DetectMovement(input, camera_id); //std::vector stablizedMovementObjects= StablizeDetection(mObjects, camera_id); if (!stablizedMovementObjects.empty()) { if (stablizedMovementObjects.size() < 6) { // Otherwise, it could be noise //// draw rectangles cv::Mat draw = input.clone(); for (auto movedObject : stablizedMovementObjects) { cv::rectangle(draw, movedObject.box, cv::Scalar(0, 128, 128), 2); if ((_detectedArea.width == 0) || (_detectedArea.height == 0)) { // Create a 640x640 rectangle centered on the detected object if (calculateIOU(movedObject.box, _previousDetectedArea) < IOU_THRESHOLD) { int x1 = movedObject.box.x; int y1 = movedObject.box.y; int xc = x1 + movedObject.box.width / 2; int yc = y1 + movedObject.box.height / 2; int x1_new = std::max(xc - 320, 0); int y1_new = std::max(yc - 320, 0); int x2_new = std::min(xc + 320, input.cols); int y2_new = std::min(yc + 320, input.rows); _detectedArea = cv::Rect(x1_new, y1_new, x2_new - x1_new, y2_new - y1_new); _previousDetectedArea = _detectedArea; } } _detectedArea.x = std::max(_detectedArea.x, 0); _detectedArea.y = std::max(_detectedArea.y, 0); _detectedArea.width = std::min(_detectedArea.width, input.cols - _detectedArea.x); _detectedArea.height = std::min(_detectedArea.height, input.rows - _detectedArea.y); //// Diplsay the frame with the combined detected areas cv::rectangle(draw, _detectedArea, cv::Scalar(0, 128, 0), 2); cv::Mat frame = input.clone(); std::vector detectedObjects; detectedObjects.clear(); if ((_detectedArea.width > 0) && (_detectedArea.height > 0)) { cv::Mat detectedObj = frame(_detectedArea).clone(); // Get sub-image of the detected area if (_engineType == EngineType::NVIDIA_GPU) { // NVIDIA CUDA detectedObjects = _gpuObjectDetector.RunInference(detectedObj); } else { detectedObjects = _cpuObjectDetector.RunInference(detectedObj); } if (!detectedObjects.empty()) { for (Object detectedObj : detectedObjects) { detectedObj.box.x += _detectedArea.x; detectedObj.box.y += _detectedArea.y; detectedObj.cameraId = camera_id; bool detected = false; if (detectedObj.classId != 1) { output.push_back(detectedObj); cv::rectangle(draw, detectedObj.box, cv::Scalar(0, 0, 255), 2); // Create a 640x640 rectangle centered on the detected object int x1 = detectedObj.box.x; int y1 = detectedObj.box.y; int xc = x1 + detectedObj.box.width / 2; int yc = y1 + detectedObj.box.height / 2; int x1_new = std::max(xc - 160, 0); int y1_new = std::max(yc - 160, 0); int x2_new = std::min(xc + 160, input.cols); int y2_new = std::min(yc + 160, input.rows); _detectedArea = cv::Rect(x1_new, y1_new, x2_new - x1_new, y2_new - y1_new); detected = true; _focusCount = 0; } else { if (detected == false) { _focusCount++; if (_focusCount >= 15) { _detectedArea.width = 0; _detectedArea.height = 0; _focusCount = 0; } } } } } else { // we could not find any detection then we move to the next detected area _focusCount++; if (_focusCount >= 5) { // stay there fore 5 frames before move to next detected area _detectedArea.width = 0; _detectedArea.height = 0; _focusCount = 0; } } detectedObj.release(); } frame.release(); } cv::imshow("Combined Detected Areas", draw); cv::waitKey(1); draw.release(); } } // Always adding the detection to the queue EnqueueDetection(output, camera_id); std::vector frequentObjects = ProcecssDetection(output, camera_id, SMOKE_THRESHOLD_SIZE); return frequentObjects; } catch (std::exception& e) { this->_logger.LogFatal("ANSFIRENSMOKE::RunInference", e.what(), __FILE__, __LINE__); return output; } } */ ANSFIRENSMOKE::~ANSFIRENSMOKE() { try { } catch (std::exception& e) { this->_logger.LogFatal("ANSFIRENSMOKE::~ANSFIRENSMOKE()", e.what(), __FILE__, __LINE__); } } bool ANSFIRENSMOKE::Destroy() { try { return true; } catch (std::exception& e) { this->_logger.LogFatal("ANSFIRENSMOKE::Destroy()", e.what(), __FILE__, __LINE__); return false; } } cv::Point2f ANSFIRENSMOKE::CalculateCentroid(const std::vector& group) { int totalArea = 0; cv::Point2f centroid(0, 0); for (const auto& rect : group) { int area = rect.width * rect.height; centroid += cv::Point2f(rect.x + rect.width / 2.0f, rect.y + rect.height / 2.0f) * area; totalArea += area; } if (totalArea > 0) { centroid *= (1.0f / totalArea); } return centroid; } cv::Point2f ANSFIRENSMOKE::CalculateGroupCenter(const std::vector& group) { if (group.empty()) { return cv::Point2f(0, 0); } // Initialize min and max points with the first rectangle's values int minX = group[0].x; int minY = group[0].y; int maxX = group[0].x + group[0].width; int maxY = group[0].y + group[0].height; // Loop through the group to find the minimum and maximum points for (const auto& rect : group) { minX = std::min(minX, rect.x); minY = std::min(minY, rect.y); maxX = std::max(maxX, rect.x + rect.width); maxY = std::max(maxY, rect.y + rect.height); } // Calculate the center point int centerX = (minX + maxX) / 2; int centerY = (minY + maxY) / 2; return cv::Point2f(centerX, centerY); } std::vector ANSFIRENSMOKE::CreateCoveringMaskRects(const cv::Mat& image, const std::vector& maskRects, float maxDistance) { std::vector coveringRects; std::vector visited(maskRects.size(), false); // Loop through each maskRect to group nearby rectangles for (size_t i = 0; i < maskRects.size(); ++i) { if (visited[i]) continue; // Start a new group with the current rectangle std::vector group; group.push_back(maskRects[i]); visited[i] = true; // Find nearby rectangles to add to the current group for (size_t j = i + 1; j < maskRects.size(); ++j) { if (visited[j]) continue; // Calculate distance between centroids of the two rectangles cv::Point2f center1(maskRects[i].x + maskRects[i].width / 2.0f, maskRects[i].y + maskRects[i].height / 2.0f); cv::Point2f center2(maskRects[j].x + maskRects[j].width / 2.0f, maskRects[j].y + maskRects[j].height / 2.0f); float distance = cv::norm(center1 - center2); // If the distance is within the maxDistance, add it to the group if (distance < maxDistance) { group.push_back(maskRects[j]); visited[j] = true; } } // Calculate the centroid of the group cv::Point2f centroid = CalculateCentroid(group); // Create a 640x640 rectangle centered on the group's centroid cv::Point topLeft(static_cast(centroid.x) - 320, static_cast(centroid.y) - 320); // Check if the rectangle is within the image bounds topLeft.x = std::max(topLeft.x, 0); topLeft.y = std::max(topLeft.y, 0); topLeft.x = std::min(topLeft.x, image.cols - 640); topLeft.y = std::min(topLeft.y, image.rows - 640); coveringRects.emplace_back(topLeft, cv::Size(640, 640)); } return coveringRects; } cv::Mat ANSFIRENSMOKE::CreateBinaryImageWithRects(int width, int height, const std::vector& rects) { // Initialize the image with zeros (background value 0) cv::Mat image = cv::Mat::zeros(height, width, CV_8UC1); // Iterate over each rectangle for (const auto& rect : rects) { // Set pixels within each rectangle to 1 cv::rectangle(image, rect, cv::Scalar(1), cv::FILLED); } return image; } bool ANSFIRENSMOKE::MajorityColourInFrame(cv::Mat frame, Range range, float area_threshold) { cv::Mat img_hsv; try { cv::Mat blur, fireMask; cv::GaussianBlur(frame, blur, cv::Size(21, 21), 0); cv::cvtColor(blur, img_hsv, cv::COLOR_BGR2HSV); cv::inRange(img_hsv, range.first, range.second, fireMask); cv::erode(fireMask, fireMask, cv::Mat(), cv::Point(-1, -1), 2); cv::dilate(fireMask, fireMask, cv::Mat(), cv::Point(-1, -1), 2); return (cv::countNonZero(fireMask) / (frame.rows * frame.cols)) > area_threshold; } catch (cv::Exception& e) { return false; } } bool ANSFIRENSMOKE::MajorityColourInFrame(cv::Mat frame, std::vector ranges, float area_threshold) { unsigned int current_area = 0; cv::Mat img_hsv; try { cv::cvtColor(frame, img_hsv, cv::COLOR_BGR2HSV); for (const Range& range : ranges) { cv::inRange(img_hsv, range.first, range.second, frame); current_area += cv::countNonZero(frame); } } catch (cv::Exception& e) { return false; } return (float)current_area / ((float)frame.rows * frame.cols) > area_threshold; } bool ANSFIRENSMOKE::DetectFireNSmokeColourInFrame(const cv::Mat& frame, const cv::Rect bBox, const std::vector& ranges, float area_threshold) { // Validate inputs if (frame.empty()) { std::cerr << "Error: Input frame is empty." << std::endl; return false; } if (ranges.empty()) { std::cerr << "Error: No HSV ranges provided for detection." << std::endl; return false; } if (area_threshold < 0.0f || area_threshold > 1.0f) { std::cerr << "Error: Area threshold must be between 0.0 and 1.0." << std::endl; return false; } try { // Get the actual area of the bounding box int x1 = std::max(bBox.x-20, 0); int y1 = std::max(bBox.y-20, 0); int x2 = std::min(bBox.x + bBox.width + 20, frame.cols); int y2 = std::min(bBox.y + bBox.height + 20, frame.rows); cv::Rect roi(x1, y1, x2 - x1, y2 - y1); cv::Mat roiFrame = frame(roi).clone(); unsigned int total_area = roiFrame.rows * roiFrame.cols; // Total area of the frame unsigned int matched_area = 0; // Convert input frame to HSV color space // Apply guasian to remove noise cv::Mat blur; cv::GaussianBlur(roiFrame, blur, cv::Size(21, 21), 0); cv::Mat img_hsv; cv::cvtColor(blur, img_hsv, cv::COLOR_BGR2HSV); // Create an empty mask to accumulate results cv::Mat combined_mask = cv::Mat::zeros(img_hsv.size(), CV_8U); // Process each HSV range and accumulate the results in the combined mask for (const Range& range : ranges) { cv::Mat temp_mask; cv::inRange(img_hsv, range.first, range.second, temp_mask); combined_mask |= temp_mask; // Accumulate results } // Apply morphological operations to clean up the mask cv::Mat kernel = cv::getStructuringElement(cv::MORPH_RECT, cv::Size(5, 5)); cv::morphologyEx(combined_mask, combined_mask, cv::MORPH_CLOSE, kernel); cv::morphologyEx(combined_mask, combined_mask, cv::MORPH_OPEN, kernel); // Count the number of non-zero pixels in the combined mask matched_area = cv::countNonZero(combined_mask); // Check if the matched area exceeds the threshold float matched_ratio = static_cast(matched_area) / total_area; bool smoke_detected = matched_ratio > area_threshold; blur.release(); img_hsv.release(); roiFrame.release(); return smoke_detected; } catch (const cv::Exception& e) { // Handle OpenCV exceptions std::cerr << "OpenCV Exception: " << e.what() << std::endl; return false; } } bool ANSFIRENSMOKE::IsFireDetected(const cv::Mat image, const cv::Rect bBox) { bool isFireColour = this->DetectFireNSmokeColourInFrame(image, bBox,this->_fire_colour, _hsvThreshold); return isFireColour; bool fireDetected = false; auto start1 = std::chrono::system_clock::now(); if (_classifierInitialized) { int w = std::min(bBox.width,bBox.height); int paddingSize = static_cast(w * 0.2); int padding = std::max(10, paddingSize); int x1 = std::max(bBox.x - padding, 0); int y1 = std::max(bBox.y - padding, 0); int x2 = std::min(bBox.x + bBox.width + padding, image.cols); int y2 = std::min(bBox.y + bBox.height + padding, image.rows); cv::Rect roi(x1, y1, x2 - x1, y2 - y1); cv::Mat roiImage = image(roi).clone(); cv::Mat rgbImage; cv::cvtColor(roiImage, rgbImage, cv::COLOR_BGR2RGB); std::vector glcmFeatures = _extractor.createFeatureVector(rgbImage); std::vector output = _annhub.Inference(glcmFeatures); if (output.size() < 1) fireDetected = true; double fireValue = abs(output[0]); if (fireValue < 0.2) { fireDetected= true; } rgbImage.release(); roiImage.release(); } auto end1 = std::chrono::system_clock::now(); auto elapsed1 = std::chrono::duration_cast(end1 - start1); std::cout << "Elapsed time for fire detection: " << elapsed1.count() << " ms" << std::endl; bool isFire = fireDetected || isFireColour; return isFire; } bool ANSFIRENSMOKE::IsSmokeDetected(const cv::Mat image, const cv::Rect bBox) { bool isSmokeColour = this->DetectFireNSmokeColourInFrame(image, bBox,this->_smoke_colour, 0.85); return isSmokeColour; bool smokeDetected = true; if (_classifierInitialized) { int w = std::min(bBox.width, bBox.height); int paddingSize = static_cast(w * 0.2); int padding = std::max(10, paddingSize); int x1 = std::max(bBox.x - padding, 0); int y1 = std::max(bBox.y - padding, 0); int x2 = std::min(bBox.x + bBox.width + padding, image.cols); int y2 = std::min(bBox.y + bBox.height + padding, image.rows); cv::Rect roi(x1, y1, x2 - x1, y2 - y1); cv::Mat roiImage = image(roi).clone(); cv::Mat rgbImage; cv::cvtColor(roiImage, rgbImage, cv::COLOR_BGR2RGB); std::vector glcmFeatures = _extractor.createFeatureVector(rgbImage); std::vector output = _annhub.Inference(glcmFeatures); if (output.size() < 1) smokeDetected = false; double smokeValue = abs(1 - output[0]); if (smokeValue > 0.2) { smokeDetected= false; } rgbImage.release(); roiImage.release(); } bool isSmoke = isSmokeColour&& smokeDetected; return isSmoke; } bool ANSFIRENSMOKE::IsOverlap(const cv::Rect& target, const std::vector& rectArray) { // Validate inputs if (target.area() <= 0 || rectArray.empty()) { return false; // No overlap possible } // Minimum overlap percentage (30%) const float minOverlapThreshold = 0.30f; // Check if the target rectangle overlaps sufficiently with any rectangle in the array for (const auto& rect : rectArray) { // Calculate intersection area cv::Rect intersection = rect.box & target; int intersectionArea = intersection.area(); // If there's an intersection, check the overlap ratio if (intersectionArea > 0) { // Calculate the smaller rectangle's area int smallerArea = std::min(rect.box.area(), target.area()); // Calculate overlap ratio float overlapRatio = static_cast(intersectionArea) / smallerArea; // Check if overlap ratio meets the threshold if (overlapRatio >= minOverlapThreshold) { return true; // Sufficient overlap found } } } return false; // No sufficient overlap found } float ANSFIRENSMOKE::calculateIOU(const cv::Rect& box_a, const cv::Rect& box_b) { int x_a = std::max(box_a.x, box_b.x); int y_a = std::max(box_a.y, box_b.y); int x_b = std::min(box_a.x + box_a.width, box_b.x + box_b.width); int y_b = std::min(box_a.y + box_a.height, box_b.y + box_b.height); if (x_b < x_a || y_b < y_a) { return 0.0f; } float intersection = (x_b - x_a) * (y_b - y_a); float area_a = box_a.width * box_a.height; float area_b = box_b.width * box_b.height; /* NOTE: - Esentially area of overlap over area of union. */ return intersection / (area_a + area_b - intersection); } std::vector ANSFIRENSMOKE::StablizeDetection(const std::vector& detectedObjects, const std::string& camera_id) { _stablization_queue.push_back(detectedObjects); while (_stablization_queue.size() > HISTORY_SIZE) { _stablization_queue.pop_front(); } for (auto it = _persistent_detections.begin(); it != _persistent_detections.end();) { it->second--; if (it->second <= 0) { it = _persistent_detections.erase(it); /* NOTE: - Queue to stablize detection and minimize flickering. We dequeue and make sure our queue is less than or equal to the size we define (technically almost every frame). */ _stablization_queue.push_back(detectedObjects); while (_stablization_queue.size() > HISTORY_SIZE) { _stablization_queue.pop_front(); } } else { it++; } } std::vector new_stable_detections; for (const Object& obj : detectedObjects) { std::vector matching_boxes; matching_boxes.reserve(HISTORY_SIZE); matching_boxes.push_back(obj.box); int fire_count = 0, smoke_count = 0; if (obj.classId == 0) { fire_count = 1; } else if (obj.classId == 2) { smoke_count = 1; } float recent_weight = 1.0; for (auto it = _stablization_queue.rbegin(); it != _stablization_queue.rend(); ++it) { bool matched = false; for (const Object& past_obj : *it) { if (calculateIOU(obj.box, past_obj.box) > IOU_THRESHOLD) { matching_boxes.push_back(past_obj.box); if (past_obj.classId == 0) { fire_count += recent_weight; } else if (past_obj.classId == 2) { smoke_count += recent_weight; } matched = true; break; } } const float WEIGHT_DECAY = 0.9; recent_weight *= WEIGHT_DECAY; } if (matching_boxes.size() >= MIN_MATCHES) { Object stable_obj; float sum_x = 0, sum_y = 0, sum_w = 0, sum_h = 0; float weight_sum = 0; float weight = 1.0; for (const auto& box : matching_boxes) { sum_x += box.x * weight; sum_y += box.y * weight; sum_w += box.width * weight; sum_h += box.height * weight; weight_sum += weight; weight *= ALPHA; } cv::Rect smoothed_box( int(sum_x / weight_sum), int(sum_y / weight_sum), int(sum_w / weight_sum), int(sum_h / weight_sum) ); bool matched_persistent = false; for (auto& persistent : _persistent_detections) { if (calculateIOU(persistent.first.box, smoothed_box) > IOU_THRESHOLD) { float dx = smoothed_box.x - persistent.first.box.x; float dy = smoothed_box.y - persistent.first.box.y; if (std::sqrt(dx * dx + dy * dy) < MAX_MOVEMENT) { persistent.first.box.x = int(ALPHA * persistent.first.box.x + (1 - ALPHA) * smoothed_box.x); persistent.first.box.y = int(ALPHA * persistent.first.box.y + (1 - ALPHA) * smoothed_box.y); persistent.first.box.width = int(ALPHA * persistent.first.box.width + (1 - ALPHA) * smoothed_box.width); persistent.first.box.height = int(ALPHA * persistent.first.box.height + (1 - ALPHA) * smoothed_box.height); if (fire_count > smoke_count) { persistent.first.classId = 0; persistent.first.className = "Fire"; } else { persistent.first.classId = 2; persistent.first.className = "Smoke"; } } persistent.second = MINIMUM_STABLE_DURATION; matched_persistent = true; break; } } if (!matched_persistent) { stable_obj.box = smoothed_box; if (fire_count > smoke_count) { stable_obj.classId = 0; stable_obj.className = "Fire"; } else { stable_obj.classId = 2; stable_obj.className = "Smoke"; } _persistent_detections.push_back({ stable_obj, MINIMUM_STABLE_DURATION }); new_stable_detections.push_back(stable_obj); } } } std::vector final_detections; final_detections.reserve(_persistent_detections.size()); for (const auto& persistent : _persistent_detections) { final_detections.push_back(persistent.first); } return final_detections; } } // Old implementation /* * * std::vector ANSFIRENSMOKE::RunInference(const cv::Mat& input, const std::string& camera_id) { std::lock_guard lock(_mutex); std::vector output; output.clear(); if (!_licenseValid) { this->_logger.LogError("ANSFIRENSMOKE::RunInference", "Invalid License", __FILE__, __LINE__); return output; } if (!_isInitialized) { this->_logger.LogError("ANSFIRENSMOKE::RunInference", "Model is not initialized", __FILE__, __LINE__); return output; } try { //1. Check movement detection on entired image //std::vector mObjects = DetectMovement(input, camera_id); std::vector stablizedMovementObjects = DetectMovement(input, camera_id); if (!stablizedMovementObjects.empty()) { std::vector fireNSmokeRects; fireNSmokeRects.clear(); //2. Check if the moving object is a for (Object& obj : stablizedMovementObjects) { bool fcol_detected = this->IsFireDetected(input, obj.box); bool scol_detected = this->IsSmokeDetected(input, obj.box); if (fcol_detected || scol_detected) { fireNSmokeRects.push_back(obj); } } if (!fireNSmokeRects.empty()) { cv::Rect detectedArea = CreateMinimumSquareBoundingBox(fireNSmokeRects, 640); // estimate that the detected area is within the image bounds detectedArea.x = std::max(detectedArea.x, 0); detectedArea.y = std::max(detectedArea.y, 0); detectedArea.width = std::min(detectedArea.width, input.cols - detectedArea.x); detectedArea.height = std::min(detectedArea.height, input.rows - detectedArea.y); cv::Mat frame = input.clone(); std::vector detectedObjects; detectedObjects.clear(); cv::Mat detectedObj = frame(detectedArea).clone(); // Get sub-image of the detected area if (_engineType == EngineType::NVIDIA_GPU) { // NVIDIA CUDA detectedObjects = _gpuObjectDetector.RunInference(detectedObj); } else { detectedObjects = _cpuObjectDetector.RunInference(detectedObj); } if (!detectedObjects.empty()) { for (Object detectedObj : detectedObjects) { detectedObj.box.x += detectedArea.x; detectedObj.box.y += detectedArea.y; detectedObj.cameraId = camera_id; if (IsOverlap(detectedObj.box, fireNSmokeRects)) { output.push_back(detectedObj); } } } detectedObj.release(); frame.release(); } } // Always adding the detection to the queue EnqueueDetection(output, camera_id); std::vector frequentObjects = ProcecssDetection(output, camera_id, SMOKE_THRESHOLD_SIZE); return frequentObjects; } catch (std::exception& e) { this->_logger.LogFatal("ANSFIRENSMOKE::RunInference", e.what(), __FILE__, __LINE__); return output; } } std::vector ANSFIRENSMOKE::RunInference(const cv::Mat& input) { std::lock_guard lock(_mutex); std::vector output; output.clear(); if (!_licenseValid) { this->_logger.LogError("ANSFIRENSMOKE::RunInference", "Invalid License", __FILE__, __LINE__); return output; } if (!_isInitialized) { this->_logger.LogError("ANSFIRENSMOKE::RunInference", "Model is not initialized", __FILE__, __LINE__); return output; } try { std::vector movementObjects = DetectMovement(input,"1234"); std::vector fireNsmokeRects; //2. Check if the moving object is a fire or smoke for (Object& obj : movementObjects) { bool fcol_detected = this->MajorityColourInFrame(obj.mask, this->_fire_colour, _hsvThreshold); bool scol_detected = this->MajorityColourInFrame(obj.mask, this->_smoke_colour, 0.85); if (fcol_detected || scol_detected) { if (fcol_detected) { fireNsmokeRects.push_back(obj.box); } else if (scol_detected) { fireNsmokeRects.push_back(obj.box); } } } if ((fireNsmokeRects.size() > 0)) { std::vector detectedObjects; if (_engineType == EngineType::NVIDIA_GPU) // NVIDIA CUDA { detectedObjects = _gpuObjectDetector.RunInference(input); } else { detectedObjects = _cpuObjectDetector.RunInference(input); } for (const auto& detectedObj : detectedObjects) { for (const auto& rect : fireNsmokeRects) { if (rect.contains(detectedObj.box.tl()) || rect.contains(detectedObj.box.br()) || rect.contains(cv::Point(detectedObj.box.x + detectedObj.box.width, detectedObj.box.y)) || rect.contains(cv::Point(detectedObj.box.x, detectedObj.box.y + detectedObj.box.height)) || detectedObj.box.contains(rect.tl()) || detectedObj.box.contains(rect.br()) || detectedObj.box.contains(cv::Point(rect.x + rect.width, rect.y)) || detectedObj.box.contains(cv::Point(rect.x, rect.y + rect.height))) { output.push_back(detectedObj); break; } } } } return output; } catch (std::exception& e) { this->_logger.LogFatal("ANSFIRENSMOKE::RunInference", e.what(), __FILE__, __LINE__); return output; } } */