Files
ANSCustomModels/ANSCustomFireNSmokeDetection/vendor/engine/fire_smoke.hpp
T

1194 lines
56 KiB
C++
Raw Normal View History

#pragma once
// engine/fire_smoke — turning a fire/smoke detector's raw boxes into alarms an
// operator can trust, on ANY camera including one that pans, tilts and zooms.
//
// WHAT THIS REPLACES. On ANSVIS the same job was done by a Windows-only custom
// DLL (ANSCustomFireNSmokeDetection) built around an "active window": scan the
// frame in tiles, and when a tile fires, LOCK a 640 px window onto that spot
// for dozens of frames and vote. Two things were wrong with it, and both were
// measured on the field videos rather than argued:
//
// 1. A locked window is a statement about the WORLD being still. On a PTZ
// camera the window keeps looking at pixels the fire has already left.
// 2. The detector was only ever shown CROPS. This model has four classes —
// fire, fake_fire, smoke, fake_smoke — and it needs CONTEXT to choose
// between them: on the whole frame it calls a floodlight fake_fire at
// 0.74, and on a tight crop of the same floodlight it says fire 0.49.
// The crop-only pipeline threw away the model's own best defence against
// false alarms, then tried to win it back with colour thresholds.
//
// THE SHAPE HERE.
// * The WHOLE FRAME leads. Every candidate comes from a full-frame pass, so
// the model's fake_* verdict is heard. A fake verdict is negative
// evidence, weighted harder than a real one is positive.
// * A native-resolution ZOOM re-check may only BOOST a small fire candidate
// the full frame already likes (a 30 px flame in a 4K frame goes from
// 0.20 to 0.85). It never originates an alarm and never overrides a
// full-frame fake. It is not used for smoke at all: smoke is a
// low-frequency thing that dissolves into haze at native scale (measured:
// 0.62 on the frame, 0.07 on the zoom).
// * Nothing is locked to the image. Candidates are TRACKED, and the track
// is carried through camera motion by a global pan/tilt/zoom estimate
// (block matching on a 160 px grey copy — no feature library, so it runs
// identically on every platform this engine ships on).
// * Every per-track cue is BOX-RELATIVE, so a moving camera cannot fake it:
// the appearance-change cue compares the inside of the box with the
// inside of the box, not a pixel with the same pixel.
// * An alarm is evidence over TIME (seconds, not frames — the box runs
// tasks anywhere from 3 to 25 fps) plus physical gates:
// fire must not be FROZEN (a lamp — or a golden statue, which is
// flame-coloured, flame-shaped and 0.7 up close);
// must be EMISSIVE-coloured (flame orange/yellow or a burnt-out
// white core) or agree at native resolution; must not be
// frozen; must not be RED-DOMINANT (a waving flag passes every
// flicker test ever written — it moves, and it is red — and
// up close the detector itself calls it fire at 0.6);
// both must not ride on a moving vehicle (headlights, tail lamps) or
// sit on a small screen (a TV showing a fire).
//
// No OpenCV, no allocation in the steady state beyond the track list, no
// platform calls. The engine node (graph.hpp, type "fire_smoke") feeds it;
// tests/fire_smoke_test.cpp drives it directly.
#include <algorithm>
#include <array>
#include <cmath>
#include <cstdint>
#include <cstring>
#include <deque>
#include <string>
#include <vector>
#include "../core/detection.hpp"
namespace ansmind {
namespace firesmoke {
// ── what the detector's classes mean ───────────────────────────────────────
enum class Role : int { Other = -1, Fire = 0, FakeFire = 1, Smoke = 2, FakeSmoke = 3 };
/// By LABEL, not by index: a two-class fire/smoke model works as well as the
/// four-class ANS one, it simply never produces negative evidence.
inline Role roleOf(std::string label) {
for (auto& c : label) c = char(::tolower(static_cast<unsigned char>(c)));
for (auto& c : label)
if (c == '-' || c == ' ') c = '_';
if (label == "fire" || label == "flame") return Role::Fire;
if (label == "smoke") return Role::Smoke;
if (label == "fake_fire" || label == "not_fire" || label == "fakefire")
return Role::FakeFire;
if (label == "fake_smoke" || label == "not_smoke" || label == "fakesmoke")
return Role::FakeSmoke;
return Role::Other;
}
inline bool isSmokeFamily(Role r) { return r == Role::Smoke || r == Role::FakeSmoke; }
inline bool isReal(Role r) { return r == Role::Fire || r == Role::Smoke; }
// ── a small grey copy of the frame, for camera motion ──────────────────────
struct Gray {
int w = 0, h = 0;
std::vector<uint8_t> px;
/// Mean (max − min) of B,G,R over the taps; < 0 = not measured. A camera in
/// IR night mode sends a picture with none.
float chroma = -1.f;
bool empty() const { return px.empty(); }
};
constexpr int kGrayW = 160;
/// BGR8 (tight or strided) → kGrayW-wide grey, height a multiple of 16.
/// Box-sampled: each output pixel is the mean of up to 3x3 taps spread over
/// its source cell — cheap, and immune to the aliasing a nearest pick has on
/// a 4K frame.
inline Gray smallGray(const uint8_t* bgr, int W, int H, int strideBytes = 0) {
Gray g;
if (!bgr || W < 16 || H < 16) return g;
const int stride = strideBytes > 0 ? strideBytes : W * 3;
g.w = kGrayW;
g.h = std::max(32, int(std::lround(double(kGrayW) * H / W / 16.0)) * 16);
g.px.resize(size_t(g.w) * g.h);
int64_t chroma = 0, taps = 0;
for (int y = 0; y < g.h; ++y) {
const int sy0 = int(int64_t(y) * H / g.h);
const int sy1 = std::max(sy0 + 1, int(int64_t(y + 1) * H / g.h));
for (int x = 0; x < g.w; ++x) {
const int sx0 = int(int64_t(x) * W / g.w);
const int sx1 = std::max(sx0 + 1, int(int64_t(x + 1) * W / g.w));
int sum = 0, n = 0;
for (int k = 0; k < 3; ++k) {
const int sy = std::min(H - 1, sy0 + (sy1 - sy0) * (2 * k + 1) / 6);
const uint8_t* row = bgr + size_t(sy) * stride;
for (int j = 0; j < 3; ++j) {
const int sx = std::min(W - 1, sx0 + (sx1 - sx0) * (2 * j + 1) / 6);
const uint8_t* p = row + size_t(sx) * 3;
sum += (p[0] * 29 + p[1] * 150 + p[2] * 77) >> 8;
++n;
chroma += std::max({p[0], p[1], p[2]}) - std::min({p[0], p[1], p[2]});
++taps;
}
}
g.px[size_t(y) * g.w + x] = uint8_t(sum / n);
}
}
if (taps) g.chroma = float(double(chroma) / double(taps));
return g;
}
// ── global camera motion ───────────────────────────────────────────────────
/// A similarity about the frame centre: p' = s·(p − c) + c + t, with t in
/// units of frame WIDTH (both axes). q is the inlier fraction of textured
/// blocks — 0 means "could not tell" (a black night frame), and the caller
/// then assumes a still camera, which is the right prior.
struct Motion {
float s = 1, tx = 0, ty = 0, q = 0;
bool known() const { return q >= 0.5f; }
/// Pan/tilt/zoom speed in one number, frame-widths per step.
float magnitude() const { return std::sqrt(tx * tx + ty * ty) + std::fabs(s - 1); }
};
namespace detail {
inline float medianOf(std::vector<float>& v) {
if (v.empty()) return 0;
const size_t m = v.size() / 2;
std::nth_element(v.begin(), v.begin() + m, v.end());
float hi = v[m];
if (v.size() % 2) return hi;
const float lo = *std::max_element(v.begin(), v.begin() + m);
return 0.5f * (lo + hi);
}
} // namespace detail
/// Block matching at half resolution: 8x8 blocks, ±kSearch px exhaustive SAD,
/// textured blocks only, then a robust similarity fit. ~0.3 M abs-diffs a
/// frame. Search reach is ±8.75 % of the frame per step — a PTZ slewing
/// faster than that between two processed frames reads as "unknown", and an
/// unknown step simply does not carry tracks (they re-form when it settles).
inline Motion estimateMotion(const Gray& prev, const Gray& cur) {
Motion out;
if (prev.empty() || cur.empty() || prev.w != cur.w || prev.h != cur.h) return out;
constexpr int kSearch = 7, kBlk = 8, kMargin = kSearch + 1;
const int W = prev.w / 2, H = prev.h / 2;
std::vector<int16_t> a(size_t(W) * H), b(size_t(W) * H);
const auto half = [&](const Gray& g, std::vector<int16_t>& o) {
for (int y = 0; y < H; ++y)
for (int x = 0; x < W; ++x) {
const uint8_t* p = &g.px[size_t(2 * y) * g.w + 2 * x];
o[size_t(y) * W + x] = int16_t((p[0] + p[1] + p[g.w] + p[g.w + 1]) / 4);
}
};
half(prev, a);
half(cur, b);
const int ny = (H - 2 * kMargin) / kBlk, nx = (W - 2 * kMargin) / kBlk;
if (ny < 2 || nx < 3) return out;
const int nb = ny * nx;
std::vector<int> best(size_t(nb), INT32_MAX), bdx(size_t(nb), 0), bdy(size_t(nb), 0);
for (int dy = -kSearch; dy <= kSearch; ++dy)
for (int dx = -kSearch; dx <= kSearch; ++dx) {
// A small pull toward zero so a flat block does not wander.
const int bias = std::abs(dx) + std::abs(dy);
for (int by = 0; by < ny; ++by)
for (int bx = 0; bx < nx; ++bx) {
int sad = bias;
const int y0 = kMargin + by * kBlk, x0 = kMargin + bx * kBlk;
for (int y = 0; y < kBlk; ++y) {
const int16_t* pa = &a[size_t(y0 + y) * W + x0];
const int16_t* pb = &b[size_t(y0 + y + dy) * W + x0 + dx];
for (int x = 0; x < kBlk; ++x) sad += std::abs(pa[x] - pb[x]);
}
const int i = by * nx + bx;
if (sad < best[size_t(i)]) {
best[size_t(i)] = sad;
bdx[size_t(i)] = dx;
bdy[size_t(i)] = dy;
}
}
}
// Textured, well-matched blocks only.
std::vector<float> px, py, vx, vy;
for (int by = 0; by < ny; ++by)
for (int bx = 0; bx < nx; ++bx) {
const int y0 = kMargin + by * kBlk, x0 = kMargin + bx * kBlk;
double m = 0, m2 = 0;
for (int y = 0; y < kBlk; ++y)
for (int x = 0; x < kBlk; ++x) {
const double v = a[size_t(y0 + y) * W + x0 + x];
m += v;
m2 += v * v;
}
m /= kBlk * kBlk;
const double sd = std::sqrt(std::max(0.0, m2 / (kBlk * kBlk) - m * m));
const int i = by * nx + bx;
if (sd <= 4.0 || best[size_t(i)] >= kBlk * kBlk * 12) continue;
px.push_back(float(x0 + kBlk / 2.0 - W / 2.0));
py.push_back(float(y0 + kBlk / 2.0 - H / 2.0));
vx.push_back(float(bdx[size_t(i)]));
vy.push_back(float(bdy[size_t(i)]));
}
const size_t n = vx.size();
if (n < 6) return out;
std::vector<float> tmp(vx);
float tx = detail::medianOf(tmp);
tmp = vy;
float ty = detail::medianOf(tmp);
float s1 = 0;
std::vector<char> inl(n, 1);
size_t ninl = n;
const float farR2 = float((0.15 * W) * (0.15 * W));
for (int it = 0; it < 3; ++it) {
std::vector<float> ratios, ratiosAll;
for (size_t i = 0; i < n; ++i) {
const float r2 = px[i] * px[i] + py[i] * py[i];
if (r2 <= farR2) continue;
const float v = ((vx[i] - tx) * px[i] + (vy[i] - ty) * py[i]) / std::max(r2, 1.f);
ratiosAll.push_back(v);
if (inl[i]) ratios.push_back(v);
}
if (ratiosAll.size() >= 4)
s1 = detail::medianOf(ratios.size() >= 4 ? ratios : ratiosAll);
ninl = 0;
double sx = 0, sy = 0;
for (size_t i = 0; i < n; ++i) {
const float ex = vx[i] - (s1 * px[i] + tx), ey = vy[i] - (s1 * py[i] + ty);
inl[i] = (ex * ex + ey * ey) <= 1.5f * 1.5f;
if (inl[i]) {
++ninl;
sx += vx[i] - s1 * px[i];
sy += vy[i] - s1 * py[i];
}
}
if (ninl < 4) break;
tx = float(sx / double(ninl));
ty = float(sy / double(ninl));
}
out.s = 1.f + s1;
out.tx = tx / float(W);
out.ty = ty / float(W);
out.q = float(ninl) / float(n);
return out;
}
// ── per-candidate appearance cues (all box-relative) ───────────────────────
constexpr int kPatch = 24; ///< grey appearance patch, kPatch x kPatch
constexpr int kColour = 12; ///< colour grid, kColour x kColour
constexpr double kLongLag = 3.0; ///< seconds between the two looks of the long-lag cue
struct Cues {
std::array<uint8_t, kPatch * kPatch> gray{};
float flame = 0; ///< fraction of cells that are flame orange / yellow
float core = 0; ///< fraction that are a burnt-out bright core
float red = 0; ///< fraction that are plain saturated red (flag, tail lamp)
float contrast = 0; ///< std of the grey patch / 255: fog has none, a plume has an edge
bool valid = false;
};
/// Cues from a BGR crop of the candidate's box (the node pads the box 10 % a
/// side). Area-averaged down to fixed grids, so the numbers mean the same
/// thing for a 30 px flame and a 600 px one. [strideBytes] lets the crop be a
/// window INTO a larger frame rather than a copy.
inline Cues patchCues(const uint8_t* bgr, int w, int h, int strideBytes = 0) {
Cues c;
if (!bgr || w < 2 || h < 2) return c;
const size_t stride = strideBytes > 0 ? size_t(strideBytes) : size_t(w) * 3;
const auto cellMean = [&](int gx, int gy, int grid, float& B, float& G, float& R) {
const int x0 = gx * w / grid, x1 = std::max(x0 + 1, (gx + 1) * w / grid);
const int y0 = gy * h / grid, y1 = std::max(y0 + 1, (gy + 1) * h / grid);
// At most 4x4 taps a cell: the grids are coarse, exactness buys nothing.
const int sx = std::max(1, (x1 - x0) / 4), sy = std::max(1, (y1 - y0) / 4);
double b = 0, g = 0, r = 0;
int n = 0;
for (int y = y0; y < y1; y += sy)
for (int x = x0; x < x1; x += sx) {
const uint8_t* p = bgr + size_t(y) * stride + size_t(x) * 3;
b += p[0];
g += p[1];
r += p[2];
++n;
}
B = float(b / n);
G = float(g / n);
R = float(r / n);
};
for (int gy = 0; gy < kPatch; ++gy)
for (int gx = 0; gx < kPatch; ++gx) {
float B, G, R;
cellMean(gx, gy, kPatch, B, G, R);
c.gray[size_t(gy) * kPatch + gx] =
uint8_t(std::min(255.f, 0.114f * B + 0.587f * G + 0.299f * R));
}
int flame = 0, core = 0, red = 0;
for (int gy = 0; gy < kColour; ++gy)
for (int gx = 0; gx < kColour; ++gx) {
float B, G, R;
cellMean(gx, gy, kColour, B, G, R);
const float Y = 0.114f * B + 0.587f * G + 0.299f * R;
// HUE, not brightness: a flame filmed at night through smoke is a dim
// orange (R ~ 100) and still a flame, and a flag in full sun is a
// bright red and still a flag. G/R is what tells them apart — orange
// and yellow sit at 0.38..0.97, a red field under 0.38. Measured over
// the field videos: every real fire has red <= 0.01, and a red flag
// with a yellow star always has more red than flame.
const float gr = G / (R + 1.f), br = B / (R + 1.f);
const bool isFlame = R > 60 && R >= G && G >= B - 6 && gr >= 0.38f && gr <= 0.97f &&
br < 0.72f && (R - B) > 25;
const bool isCore = Y > 215 && R >= B - 6;
if (isFlame) ++flame;
if (isCore) ++core;
if (R > 60 && gr < 0.38f && br < 0.5f) ++red;
}
const float cells = float(kColour * kColour);
c.flame = flame / cells;
c.core = core / cells;
c.red = red / cells;
{
double m = 0, v = 0;
for (uint8_t g : c.gray) m += g;
m /= double(c.gray.size());
for (uint8_t g : c.gray) v += (g - m) * (g - m);
c.contrast = float(std::sqrt(v / double(c.gray.size())) / 255.0);
}
c.valid = true;
return c;
}
/// How much the INSIDE of the box changed between two looks. Each patch is
/// contrast-normalised (exposure and gain do not count) and the comparison
/// takes the best of a ±1 cell shift (detector box jitter does not count).
/// Flames: 0.2–0.6. A lamp, a sign, a painted wall: under 0.05.
inline float patchDynamics(const std::array<uint8_t, kPatch * kPatch>& pa,
const std::array<uint8_t, kPatch * kPatch>& pb) {
std::array<float, kPatch * kPatch> a, b;
const auto norm = [](const std::array<uint8_t, kPatch * kPatch>& s,
std::array<float, kPatch * kPatch>& d) {
double m = 0, m2 = 0;
for (uint8_t v : s) {
m += v;
m2 += double(v) * v;
}
m /= s.size();
const double sd = std::sqrt(std::max(0.0, m2 / s.size() - m * m));
for (size_t i = 0; i < s.size(); ++i) d[i] = float((s[i] - m) / (sd + 4.0));
};
norm(pa, a);
norm(pb, b);
float best = 1e9f;
for (int dy = -1; dy <= 1; ++dy)
for (int dx = -1; dx <= 1; ++dx) {
double sum = 0;
for (int y = 1; y < kPatch - 1; ++y)
for (int x = 1; x < kPatch - 1; ++x)
sum += std::fabs(a[size_t(y + dy) * kPatch + x + dx] - b[size_t(y) * kPatch + x]);
best = std::min(best, float(sum / double((kPatch - 2) * (kPatch - 2))));
}
return best;
}
// ── the verifier ───────────────────────────────────────────────────────────
struct Params {
// Evidence, per observation at the 6 Hz the constants were tuned at; the
// verifier rescales by the real frame interval.
float bias = 0.20f; ///< a detection must beat this to add evidence
float missCost = 0.10f; ///< evidence lost per missed step
float fakeWeight = 1.5f; ///< a fake_* verdict counts this much against
float confirm = 2.0f; ///< evidence needed to alarm
float evidenceMax = 6.0f;
float minSeconds = 0.8f; ///< never alarm on a track younger than this
int minObs = 4;
float windowSeconds = 3.0f; ///< the gates look back this far
// Gates
// fire: FROZEN. Judged on the 25th PERCENTILE of the appearance change, not
// the median, because of a golden dragon statue on a temple roof: flame
// coloured, flame shaped, 0.7 up close — and the detector's box flipped
// between two shapes, so its change was bimodal (mostly 0.00, spikes of
// 0.4) and a median over a track's first six looks could land anywhere.
// A flame never looks the same twice; a statue mostly does, however its
// box jitters. Measured: 33 real-fire tracks, blown-out night blaze
// included, p25 0.084–0.66; ten tracks of the statue 0.005–0.016 — and the
// same inside each track's first eight looks. Needs dynMinSamples looks
// before a fire may alarm at all: three samples have no percentile.
// It looks back dynSeconds, longer than the other gates: three seconds of
// that statue can be ALL box-flips (one alarm in two minutes came from
// exactly that, ten seconds into a track); ten cannot.
float dynMin = 0.05f;
int dynMinSamples = 5;
float dynSeconds = 10.0f;
float emissiveMin = 0.12f; ///< fire: flame + core colour fraction
float zoomOk = 0.45f; ///< fire: native-resolution agreement
// fire: red-dominant = a flag, a tail lamp, a sign. Measured on the engine's
// own numbers: 47 real-fire tracks never exceed red 0.02 in any window; six
// tracks of a red flag with a yellow star sit at 0.09..0.32 with red/flame
// 0.37..1.3. Both thresholds leave a 3x margin on the fire side.
float redMax = 0.06f;
float redRatio = 0.25f;
float vehicleFrac = 0.5f, vehicleDrift = 0.5f;
float screenFrac = 0.5f;
// fire: the close-up has looked several times and seen NOTHING, and the box
// barely changes — tan cardboard has flame hue and crawled to an alarm over
// 78 s on a weak "fire"; a real flame either changes or shows up close.
float zoomDeny = 0.05f, zoomDenyDyn = 0.15f;
// fire: UNBACKED. The close-up has no context — that is why the old crop
// pipeline false-alarmed — so it may carry an alarm alone only when it is
// very sure. Warm temple lights under a red roof at dusk: whole frame
// 0.15–0.30, close-up 0.5–0.7, flickering enough to pass `static`; the
// golden statue the same (0.2–0.4 / 0.64–0.77). Every real fire measured
// had the whole frame at >= 0.41 at least once in the window, or (a small
// distant one, frame 0.24–0.29) a close-up median of 0.82.
float backedRaw = 0.40f, backedZoom = 0.75f;
// ...and when it IS that sure (the statue again, 0.78 up close at the start
// of a recording, nothing to inherit), five looks are too few to see that a
// thing is frozen: one of them read 0.04, the p25 of five did not. A fire
// the whole frame cannot see is watched for this many looks first.
int unbackedSamples = 12;
// fire: RIGID. A flame's outline breathes, so the detector's box never holds
// its size: the spread of log(box area) over the window was 0.115–1.6 on 44
// real-fire tracks (the smallest single window 0.043, a 30 px fire). A dead
// rust-brown tree in a green forest — flame-coloured, swaying enough to pass
// `static`, 0.64 on the frame at its best — held 0.016–0.022 for minutes, the
// golden statue 0.005–0.018. Brightness was tried first (a flame emits light)
// and measured nothing: real fires span 71–213, the tree sits at 100.
// Over dynSeconds of history, not the 3 s window: one 1-pixel step of the
// detector's box (16 -> 17 px at model scale) reads 0.047 in a short window.
// Ten seconds: every fixed object <= 0.047, every real fire >= 0.053 (p5 0.097).
float rigidSpread = 0.05f;
// ...and the two together, for what slips between them. The statue's box
// flips by a pixel (spread up to 0.074, past `rigid`) and once in 45 minutes
// its p25 touched the `static` line — but 27–71 % of its looks are frozen
// (change < dynMin) where a real fire has none (nine recordings of ten: 0 %;
// a tiny steady flame: up to 23 %, with a box that breathes at 0.12–0.2).
float frozenFrac = 0.15f, frozenSpread = 0.09f;
int rigidSamples = 5; // first five looks: real >= 0.053, the tree and the statue <= 0.018
// smoke in IR: RIGID. A plume's outline never sits still, so the detector's
// box around it WANDERS: centre, width and height jitter by a few per cent
// of the box every look. Two IR look-alikes on the Tay Son PTZ held their
// pose spread (centre in box units + log w + log h, over dynSeconds) at
// 0.015-0.021 (a lit sign, a stack of bright rectangles) and 0.026-0.048
// (sky through pine trunks on the ridge) for minutes: the same box to three
// decimals. Real small plumes (box under 3 % of the frame) measured 0.078+
// on one daytime recording but 0.026-0.049 on a second — so by DAY the
// pose does not separate them and the gate is IR-only (chroma 0), where
// the detector has no colour to go on and `night-brief` already waits.
// Box breathing alone (`rigidSpread`) does not separate them either — a
// plume filling the view is clamped to the frame and reads 0.005. Small
// boxes only: a big box's jitter is a fraction of a per cent of its size.
// Both look-alikes were born while the PTZ was still arriving (pose 0.11-0.35
// over their first 10 s, a blurred box shrinking onto the object) and only
// settled after 12-14 s, so a small smoke box is not believed before
// rigidSmokeAge — distant smoke is not a three-second emergency — and once
// judged rigid a track STAYS rigid until its box wanders twice the line.
float rigidSmokeArea = 0.03f, rigidSmokePose = 0.06f;
float rigidSmokeSeconds = 8.0f; ///< the history must span this long...
int rigidSmokeSamples = 10; ///< ...and hold this many looks
float rigidSmokeAge = 15.f; ///< a small smoke box waits this long before it may alarm
float blobCore = 0.2f; ///< this much saturated white = a blob a close-up cannot judge
// SMOKE is a slow thing anchored to its source, and its look-alikes at
// night are headlight beams, which are neither. So smoke waits longer
// (there is no such thing as a one-second plume), looks back further, and a
// track that has DISPLACED this many box-sizes along a fairly straight line
// inside that look-back is travelling, not rising. Measured: beams cover
// 2.5–3 box-sizes at straightness >= 0.46; 174 real plumes stay under 1.4
// and meander. It delays rather than denies — a plume's first second can
// read as straight travel (smoke RISES), and that decays as it stays put.
float smokeMinSeconds = 3.0f;
float smokeWindowSeconds = 4.0f;
float travelBoxes = 1.6f, travelStraight = 0.45f;
// How many seconds of smoke must have been SEEN THERE — not by this track,
// at this place. Two things made a plain "track must be N s old" useless on
// the real PTZ recordings: a plume's box reshapes constantly, so its tracks
// die and re-form every 6–10 s; and a plume BEGINS as wisps — 12-second
// bursts, a minute apart, for eight minutes before it is continuous. So a
// track born where smoke is, or was within the last minute, inherits what
// that place has already seen, and only seconds WITH a detection count.
// What this separates a plume from is the thing that fooled every other cue
// at 4 am: lights passing behind trees lit the mist for 16 s, and for those
// 16 s it was billowing smoke to any frame-level test. It did not come back.
// 0 = off (smokeMinSeconds only); a landscape or PTZ camera wants 20–30.
float smokePersistSeconds = 0.f;
// The same for FIRE, asked for by the Kien An PTZ crews: while a PTZ tours,
// every new view is a fresh look at rust-brown rock, a dead tree, a street
// lamp or the sun, and a glimpse of a few seconds is enough to alarm before
// the camera moves on. A real fire is still there when it comes back; the
// same place memory (gaps up to a minute, fading while unseen) applies.
// 0 = off (minSeconds and the flicker gates only).
float firePersistSeconds = 0.f;
// NIGHT. In IR mode the picture has no colour and little texture, and the
// detector is at its worst: of the false smoke on the user's PTZ nearly all
// was IR — mist lit for 16 s by lights passing behind trees, a lit patch of
// foliage for 3 s. Neither lasts; a fire's smoke does. While the frame is
// monochrome, smoke must have been seen at the place this long, whatever
// smokePersistSeconds says (0 turns it off).
float nightSmokeSeconds = 20.f, monoChroma = 1.5f; // IR reads 0.0; the darkest colour night scene 2.3
float placeFade = 0.25f; ///< seconds of place memory lost per second unseen
float lineageSeconds = 60.0f; ///< a place remembers what it saw this long
// smoke: a fifth of the box saturated white AND nothing changing is a light
// shining into the lens (a car stopped facing the camera, 8 s of "smoke").
// Either half alone is wrong, and both were tried: real smoke in front of
// the sky or its own fire has a bright core but churns (0.31–0.46), and the
// glowing haze round a night blaze barely changes but its core is <= 0.13.
float glareCore = 0.2f, glareDyn = 0.12f;
// ...and a LAMP at night is the same thing smaller: floodlights and a street
// lamp blooming in IR mode were "smoke" at 0.22–0.38 for minutes. Core
// 0.13–0.35 on every one of 14 tracks, change 0.05–0.11, and no flame colour
// at all (IR is monochrome; an LED is white). Day smoke on the same camera:
// core 0.000 on 27 tracks. Smoke lit by its own fire has a core too (0.07)
// but is orange (flame 0.72), which is why the colour is part of the test.
// A bank of floodlights in a larger box dilutes to 0.07–0.10 (change
// 0.08–0.12); real white smoke in daylight measured 0.028, and the one real
// plume with a big white core and no colour (0.37, lit from behind) churns
// at 0.16.
// ...and a lit white radar dome on a hilltop, IR again: core 0.09–0.10, change
// 0.14–0.18 under sensor noise — 44 alarms in 27 minutes at the 0.14 this began
// with. One allowance now, the floodlights' 0.22.
float lampCore = 0.05f, lampFlame = 0.05f, lampDyn = 0.22f;
// The dome's box flips between two shapes like the statue's, so its median
// change jumps (0.07–0.64) while most looks are quiet: the 10 s low quartile too.
float lampDynLow = 0.18f;
// fire, in IR night mode only: the picture has no colour, so a fire and a lamp
// are the same saturated white blob and only FLICKER can vouch for a fire
// (flames 0.2–0.6). Strong lamps behind trees on a ridge were "fire" at
// 0.5–0.74: core 0.20, no flame colour, change 0.10–0.12.
float irLampDyn = 0.18f; // the dome reads 0.10–0.16
// smoke, in IR night mode only: a street lamp's star-burst inside a box of
// lit haze (20:35 on the Tay Son tour, "smoke" 0.44-0.50, box 0.22 x 0.25)
// saturates only 3-4 of the 144 colour cells (core 0.021-0.035), under the
// 0.05 a bloomed lamp fills — but by night there is no colour to lift a
// plume over it, and a plume with a point source inside it is a fire's,
// which the fire path alarms on. Day smoke keeps the 0.05 (real white
// smoke in daylight measured 0.028).
float irSmokeLampCore = 0.02f;
// Floodlights with star-shaped bloom flicker more (0.10–0.2) and have a big
// core (0.22): a colourless box that is a fifth saturated white gets a
// wider allowance.
float floodCore = 0.15f, floodDyn = 0.22f;
// smoke: the same, by AREA — the forest box shrank to 0.84 x 0.95 and got out.
float unlocalisedArea = 0.72f;
// ...unless the camera really is inside it: a room filling with smoke from a
// fire in view is a frame-wide box too (0.8, a test burn) — but it is lit
// orange (flame colour 0.6–0.78 of the box; the forest and the haze 0.00),
// and the fire beside it is already confirmed.
float unlocalisedFlame = 0.10f;
// smoke: IN HAZE. When the detector has just called the whole frame smoke, the
// camera is inside fog — and then it also calls patches of it smoke, anywhere,
// again and again, until the place memory adds up (night fog in IR: one alarm
// in 71 minutes, 3 s into a track, a second after a frame-wide box). For this
// long after a frame-wide smoke box nothing smaller is believed either, with
// the same exemptions: lit orange, or a fire confirmed.
float hazeSeconds = 30.f;
// smoke: FEATURELESS. Rain and fog closing in on one side of the picture: a
// box a quarter of the frame, 0.33, nothing in it (grey-patch contrast 0.095)
// and nothing happening (change 0.04). A plume has an edge: every real smoke
// box over a tenth of the frame measured 0.134–0.24 (26 tracks). Small boxes
// are exempt — a thin distant wisp reads 0.07–0.12 too, but churns (0.13–0.43).
float featurelessArea = 0.10f, featurelessContrast = 0.11f, featurelessDyn = 0.08f;
// smoke: GRAIN. A telecom antenna panel seen through fog on an IR night
// (22:25 on the Tay Son tour, "smoke" 0.25-0.42 for minutes): the patch has
// next to no structure (contrast 0.040, ten grey levels) yet changes by
// 0.2-0.3 every look — that change is sensor noise, not a plume moving,
// and a patch whose only motion is noise cannot be vouched for. Real day
// plumes pair contrast under 0.05 with change over 0.15 in 0.1 % of their
// looks (and in no window); the antenna in 91 %. A thin distant wisp that
// churns (0.13-0.43) still has an edge: 0.07-0.12.
// Once grain, a track stays grain until an EDGE appears (twice the contrast
// line): the change statistic drops out for a few looks whenever the box
// hops and the patch comparison restarts, and that is not structure.
float grainContrast = 0.05f, grainDyn = 0.15f;
// smoke: a box as wide as the frame has located nothing. On the user's PTZ
// it was IR grain over a forest (conf 0.83!) and afternoon haze over a
// hillside; of 202 real-plume alarms on the same camera two were this wide,
// both with a localised alarm beside them. A plume has an edge somewhere.
float unlocalisedWidth = 0.93f;
// Association: no overlap AND an area this many times different is another
// object, however close. Without it a beam's track hopped onto a tiny box
// behind it at the moment of decision and cancelled its own displacement.
float sizeRatioMax = 3.0f;
// Lifetime
float lostSeconds = 1.5f; ///< a track unseen this long is dropped
float holdSeconds = 1.0f; ///< a confirmed track still reports through a gap
float minConf = 0.15f; ///< candidates below this are ignored
bool fire = true, smoke = true;
};
struct Candidate {
RectF box; ///< normalized to the frame
Role role = Role::Other;
float conf = 0;
// Native-resolution re-check, fire family only; zoomValid is false when
// the crop was not a real magnification.
bool zoomValid = false;
float zoomReal = 0, zoomFake = 0;
Cues cues;
bool onVehicle = false; ///< box sits inside a vehicle the filter model found
bool onScreen = false; ///< box sits inside a small TV / monitor / phone
};
struct Alarm {
RectF box;
bool smoke = false;
float confidence = 0;
uint64_t trackId = 0;
float seconds = 0; ///< how long the track has existed
float evidence = 0;
bool held = false; ///< reported through a detection gap, box predicted
};
/// Why the newest decision on a track went the way it did — for the status
/// document and the offline harness, so a missed fire or a false alarm in
/// the field can be read rather than guessed at.
struct TrackDebug {
uint64_t id = 0;
bool smoke = false, confirmed = false;
RectF box;
float evidence = 0, dyn = 0, flame = 0, core = 0, red = 0, zoom = -1;
float frozen = 0; ///< fraction of the history's looks that did not change at all
float breath = 0; ///< spread of log(box area) over the window: a flame's box breathes
float pose = -1; ///< spread of the box's pose over the history (-1 = too short); a plume wanders
float dynLow = 0; ///< 25th percentile of the appearance change (dyn is the median)
float placeSeconds = 0; ///< seconds of this family SEEN at this place
float vehicle = 0, screen = 0, drift = 0, straight = 0, seconds = 0;
int obs = 0;
// The newest observation on its own (the fields above are window means) —
// what the offline harness dumps so a gate can be re-tuned on recordings
// without re-running a single model. hit=false on a frame the track missed.
bool hit = false;
float lastDynLong = -1;
float lastReal = 0, lastFake = 0, lastDyn = -1, lastFlame = 0, lastCore = 0,
lastRed = 0, lastZoom = -1, lastZoomFake = 0, lastRaw = 0, lastContrast = 0;
bool lastVehicle = false, lastScreen = false;
/// "" | evidence | young | brief | night-brief | vehicle | screen | unlocalised | in-haze | featureless | glare | lamp | travelling | static | rigid | unbacked |
/// zoom-denies | not-emissive | plain-red
const char* blockedBy = "";
};
class Verifier {
public:
explicit Verifier(Params p = {}) : p_(p) {}
void configure(const Params& p) { p_ = p; }
const Params& params() const { return p_; }
/// Once per frame, before update(): the camera's motion since the last
/// frame. [aspect] = width / height of the frame.
/// The frame's mean chroma (< 0 = unknown) and whether it reads as IR night mode.
float chroma() const { return chroma_; }
bool monochrome() const { return mono_; }
void beginFrame(const Gray& gray, double tSeconds, float aspect) {
motion_ = estimateMotion(prevGray_, gray);
chroma_ = gray.chroma;
mono_ = gray.chroma >= 0 && gray.chroma < p_.monoChroma;
prevGray_ = gray;
dt_ = haveT_ ? std::clamp(tSeconds - t_, 0.02, 1.0) : 1.0 / 6.0;
// A long gap (stream stall, task paused) is not evidence of anything.
if (haveT_ && tSeconds - t_ > 2.0) {
tracks_.clear();
ghosts_.clear();
}
t_ = tSeconds;
haveT_ = true;
aspect_ = aspect > 0 ? aspect : 16.f / 9.f;
}
const Motion& motion() const { return motion_; }
/// Which candidates are worth a native-resolution re-check this frame:
/// small fire-family boxes, best first, at most [budget].
std::vector<int> planZoom(const std::vector<Candidate>& c, int budget) const {
std::vector<int> idx;
for (int i = 0; i < int(c.size()); ++i) {
if (c[size_t(i)].conf < p_.minConf || isSmokeFamily(c[size_t(i)].role) ||
c[size_t(i)].role == Role::Other)
continue;
idx.push_back(i);
}
std::sort(idx.begin(), idx.end(),
[&](int a, int b) { return c[size_t(a)].conf > c[size_t(b)].conf; });
if (int(idx.size()) > budget) idx.resize(size_t(std::max(0, budget)));
return idx;
}
void update(const std::vector<Candidate>& cands, std::vector<Alarm>& out) {
out.clear();
const float step = float(dt_ * 6.0); // 1.0 at the tuning rate
// 1. carry every track through the camera's motion
for (auto& tr : tracks_) tr.pred = warp(tr.box);
// 2. associate — within a family, best pairs first
struct Pair { float score; int t, d; };
std::vector<Pair> pairs;
for (int ti = 0; ti < int(tracks_.size()); ++ti) {
const Track& tr = tracks_[size_t(ti)];
for (int di = 0; di < int(cands.size()); ++di) {
const Candidate& d = cands[size_t(di)];
if (!wanted(d) || isSmokeFamily(d.role) != tr.smoke) continue;
const float i = tr.pred.iou(d.box);
const float dc = centreDist(tr.pred, d.box);
const float gate = std::max(std::max(tr.pred.w, tr.pred.h / aspect_),
std::max(d.box.w, d.box.h / aspect_));
if (i <= 0.05f && p_.sizeRatioMax > 0) {
const float a1 = tr.pred.area(), a2 = d.box.area();
if (std::max(a1, a2) > p_.sizeRatioMax * std::max(1e-9f, std::min(a1, a2))) continue;
}
if (i > 0.05f || dc < gate) pairs.push_back({i - 0.5f * dc / (gate + 1e-9f), ti, di});
}
}
std::sort(pairs.begin(), pairs.end(),
[](const Pair& a, const Pair& b) { return a.score > b.score; });
std::vector<char> usedT(tracks_.size(), 0), usedD(cands.size(), 0);
for (const auto& pr : pairs) {
if (usedT[size_t(pr.t)] || usedD[size_t(pr.d)]) continue;
usedT[size_t(pr.t)] = usedD[size_t(pr.d)] = 1;
observe(tracks_[size_t(pr.t)], cands[size_t(pr.d)], step);
}
// 3. misses
for (size_t ti = 0; ti < usedT.size(); ++ti) {
if (usedT[ti]) continue;
Track& tr = tracks_[ti];
tr.box = tr.pred;
tr.evidence = std::max(0.f, tr.evidence - p_.missCost * step);
if (tr.evidence <= 0.f) tr.confirmed = false;
tr.dbg.hit = false;
tr.dbg.evidence = tr.evidence;
tr.dbg.confirmed = tr.confirmed;
tr.dbg.box = tr.box;
}
// 4. births
for (size_t di = 0; di < cands.size(); ++di) {
if (usedD[di] || !wanted(cands[di])) continue;
Track tr;
tr.id = ++nextId_;
tr.smoke = isSmokeFamily(cands[di].role);
tr.t0 = tr.lastHit = t_;
tr.box = tr.pred = cands[di].box;
// Lineage: born where the family already is, or just was.
const auto touches = [&](const RectF& o) {
const float cx = tr.box.x + tr.box.w / 2, cy = tr.box.y + tr.box.h / 2;
const float ox = o.x + o.w / 2, oy = o.y + o.h / 2;
return tr.box.iou(o) > 0.1f ||
(cx >= o.x && cx <= o.x + o.w && cy >= o.y && cy <= o.y + o.h) ||
(ox >= tr.box.x && ox <= tr.box.x + tr.box.w && oy >= tr.box.y &&
oy <= tr.box.y + tr.box.h);
};
// ...and what the place LOOKED like. A detection on a still thing is
// intermittent, so its tracks die and re-form every few seconds, and a
// newborn with an empty history can pass the frozen test on five lucky
// looks (the statue again: twelve alarms, every one within four seconds
// of a birth). The longest history touching the birthplace is inherited,
// re-dated to end now so the look-back keeps it.
const std::deque<std::pair<double, float>>* inherit = nullptr;
for (const auto& o : tracks_)
if (o.smoke == tr.smoke && touches(o.box)) {
tr.placeSeen = std::max(tr.placeSeen, o.placeSeen);
if (!inherit || o.dynHist.size() > inherit->size()) inherit = &o.dynHist;
}
for (const auto& g : ghosts_)
if (g.smoke == tr.smoke && touches(g.box)) {
// What a place remembers FADES while nothing is seen there, a quarter
// second a second. Without that, a touring PTZ coming back to a preset
// every half minute let 3–4 s glimpses of lit haze add up to the 20 s
// that `night-brief` asks for. A plume that begins as 12 s wisps half a
// minute apart still gets there, on its third wisp.
tr.placeSeen = std::max(
tr.placeSeen, std::max(0.0, g.placeSeen - p_.placeFade * (t_ - g.died)));
if (!inherit || g.dynHist.size() > inherit->size()) inherit = &g.dynHist;
}
if (inherit && !inherit->empty()) {
tr.dynHist = *inherit;
const double shift = t_ - tr.dynHist.back().first;
for (auto& h : tr.dynHist) h.first += shift;
}
tracks_.push_back(std::move(tr));
observe(tracks_.back(), cands[di], step, /*first=*/true);
}
// 5. deaths (a place remembers its age for a moment), 6. report
for (auto& g : ghosts_) g.box = warp(g.box);
for (const auto& tr : tracks_)
if (t_ - tr.lastHit > p_.lostSeconds || outOfFrame(tr.box))
ghosts_.push_back({tr.smoke, tr.box, tr.placeSeen, t_, tr.dynHist});
if (ghosts_.size() > 64) ghosts_.erase(ghosts_.begin(), ghosts_.end() - 64);
ghosts_.erase(std::remove_if(ghosts_.begin(), ghosts_.end(),
[&](const Ghost& g) {
return t_ - g.died > p_.lineageSeconds ||
outOfFrame(g.box);
}),
ghosts_.end());
tracks_.erase(std::remove_if(tracks_.begin(), tracks_.end(),
[&](const Track& tr) {
return t_ - tr.lastHit > p_.lostSeconds ||
outOfFrame(tr.box);
}),
tracks_.end());
for (const auto& tr : tracks_) {
if (!tr.confirmed) continue;
const bool held = tr.lastHit < t_;
if (held && t_ - tr.lastHit > p_.holdSeconds) continue;
Alarm a;
a.box = tr.box.clamped();
a.smoke = tr.smoke;
a.confidence = tr.meanReal;
a.trackId = tr.id;
a.seconds = float(t_ - tr.t0);
a.evidence = tr.evidence;
a.held = held;
out.push_back(a);
}
}
std::vector<TrackDebug> debug() const {
std::vector<TrackDebug> v;
for (const auto& tr : tracks_) v.push_back(tr.dbg);
return v;
}
size_t trackCount() const { return tracks_.size(); }
void reset() {
haveWide_ = false;
tracks_.clear();
ghosts_.clear();
prevGray_ = {};
haveT_ = false;
}
private:
struct Obs {
double t = 0;
float real = 0, fake = 0;
float raw = 0; ///< the whole-frame verdict alone, before the close-up is fused in
float dyn = -1; ///< -1 = no earlier patch to compare with
float dynLong = -1; ///< the same, against the look kLongLag seconds ago
float flame = 0, core = 0, red = 0, contrast = 0;
float logArea = 0; ///< log of the detector's box area on this look (0 = a miss)
float zoom = -1; ///< -1 = no valid zoom
bool vehicle = false, screen = false;
bool wide = false; ///< the box was as big as the frame on this look
float dx = 0, dy = 0; ///< world step, in box sizes
};
struct PatchAt {
double t;
std::array<uint8_t, kPatch * kPatch> gray;
};
struct Track {
uint64_t id = 0;
bool smoke = false, confirmed = false;
bool rigid = false; ///< latched: the box held its pose for a full history
bool grain = false; ///< latched: structureless patch whose change was noise
RectF box, pred;
double t0 = 0, lastHit = 0;
double placeSeen = 0; ///< seconds seen here, by this track and its ancestors
float evidence = 0, meanReal = 0;
int nobs = 0;
std::deque<Obs> win;
std::deque<std::pair<double, float>> dynHist; ///< (t, dyn), dynSeconds deep
std::deque<std::pair<double, float>> areaHist; ///< (t, log box area), the same depth
struct PoseAt { double t; float cx, cy, lw, lh; };
std::deque<PoseAt> poseHist; ///< the box's pose per look, the same depth
std::deque<PatchAt> patches;
TrackDebug dbg;
};
bool wanted(const Candidate& c) const {
if (c.role == Role::Other || c.conf < p_.minConf) return false;
return isSmokeFamily(c.role) ? p_.smoke : p_.fire;
}
static bool outOfFrame(const RectF& b) {
return b.x + b.w < -0.05f || b.y + b.h < -0.05f || b.x > 1.05f || b.y > 1.05f ||
b.w <= 0 || b.h <= 0;
}
RectF warp(const RectF& b) const {
if (!motion_.known()) return b;
RectF r;
const float s = motion_.s;
r.x = s * (b.x - 0.5f) + 0.5f + motion_.tx;
r.y = s * (b.y - 0.5f) + 0.5f + motion_.ty * aspect_;
r.w = b.w * s;
r.h = b.h * s;
return r;
}
/// Centre distance in frame WIDTHS on both axes (isotropic in pixels).
float centreDist(const RectF& a, const RectF& b) const {
const float dx = (a.x + a.w / 2) - (b.x + b.w / 2);
const float dy = ((a.y + a.h / 2) - (b.y + b.h / 2)) / aspect_;
return std::sqrt(dx * dx + dy * dy);
}
void observe(Track& tr, const Candidate& c, float step, bool first = false) {
Obs o;
o.t = t_;
o.real = isReal(c.role) ? c.conf : 0.f;
o.raw = o.real;
if (c.box.w > 0 && c.box.h > 0) o.logArea = std::log(c.box.w * c.box.h);
o.wide = c.box.w >= p_.unlocalisedWidth || c.box.w * c.box.h >= p_.unlocalisedArea;
if (o.wide && tr.smoke && isReal(c.role)) { lastWideT_ = t_; haveWide_ = true; }
o.fake = isReal(c.role) ? 0.f : c.conf;
if (c.zoomValid && !tr.smoke) {
o.zoom = c.zoomReal;
// A fake verdict at native scale counts in full only when the frame
// had no real verdict of its own to weigh it against.
o.fake = std::max(o.fake, o.real == 0 ? c.zoomFake : 0.5f * c.zoomFake);
// The close-up agrees: take the better of the two. It sees NOTHING: the
// frame's verdict counts half — unless the thing is a saturated blob, a
// blown-out blaze at night, which no close-up can judge (it is white at
// any scale). Halving those held a real fire in the rain for 36 s.
if (o.real > 0) {
if (c.zoomReal >= 0.1f) o.real = std::max(o.real, c.zoomReal);
else if (!(c.cues.valid && c.cues.core >= p_.blobCore)) o.real *= 0.5f;
}
}
if (c.cues.valid) {
o.flame = c.cues.flame;
o.core = c.cues.core;
o.contrast = c.cues.contrast;
o.red = c.cues.red;
// Compare with the look closest to 1/6 s ago: at 25 fps consecutive
// frames barely differ even for a real flame.
const PatchAt* ref = nullptr;
for (const auto& pa : tr.patches) {
if (t_ - pa.t >= 0.12) ref = &pa;
else break;
}
if (ref) o.dyn = patchDynamics(ref->gray, c.cues.gray);
// ...and with the look from seconds ago. Sensor noise, fluttering
// leaves and box jitter change a patch as much in 1/6 s as in 3 s;
// smoke's STRUCTURE moves on, so its long-lag change is much larger.
const PatchAt* old = nullptr;
for (const auto& pa : tr.patches) {
if (t_ - pa.t >= kLongLag) old = &pa;
else break;
}
if (old) o.dynLong = patchDynamics(old->gray, c.cues.gray);
tr.patches.push_back({t_, c.cues.gray});
while (tr.patches.size() > 2 && t_ - tr.patches[1].t >= kLongLag) tr.patches.pop_front();
}
o.vehicle = c.onVehicle;
o.screen = c.onScreen;
if (!first) {
const float sz = std::max(1e-4f, (tr.pred.w + c.box.w + (tr.pred.h + c.box.h) / aspect_) / 4);
o.dx = ((c.box.x + c.box.w / 2) - (tr.pred.x + tr.pred.w / 2)) / sz;
o.dy = ((c.box.y + c.box.h / 2) - (tr.pred.y + tr.pred.h / 2)) / aspect_ / sz;
}
tr.box = c.box;
if (!first) tr.placeSeen += std::min(dt_, 1.0);
tr.lastHit = t_;
tr.nobs++;
tr.evidence = std::clamp(
tr.evidence + (o.real - p_.fakeWeight * o.fake - p_.bias) * step, 0.f, p_.evidenceMax);
if (o.dyn >= 0) tr.dynHist.push_back({t_, o.dyn});
if (o.logArea != 0) tr.areaHist.push_back({t_, o.logArea});
while (!tr.areaHist.empty() && t_ - tr.areaHist.front().first > p_.dynSeconds)
tr.areaHist.pop_front();
if (c.box.w > 0 && c.box.h > 0)
tr.poseHist.push_back({t_, c.box.x + c.box.w / 2, c.box.y + c.box.h / 2,
std::log(c.box.w), std::log(c.box.h)});
while (!tr.poseHist.empty() && t_ - tr.poseHist.front().t > p_.dynSeconds)
tr.poseHist.pop_front();
while (!tr.dynHist.empty() && t_ - tr.dynHist.front().first > p_.dynSeconds)
tr.dynHist.pop_front();
tr.win.push_back(o);
const float look = tr.smoke ? p_.smokeWindowSeconds : p_.windowSeconds;
while (!tr.win.empty() && t_ - tr.win.front().t > look) tr.win.pop_front();
decide(tr);
TrackDebug& d = tr.dbg;
d.hit = true;
d.lastRaw = c.conf;
d.lastReal = o.real;
d.lastFake = o.fake;
d.lastDyn = o.dyn;
d.lastDynLong = o.dynLong;
d.lastFlame = o.flame;
d.lastCore = o.core;
d.lastContrast = o.contrast;
d.lastRed = o.red;
d.lastZoom = o.zoom;
d.lastZoomFake = c.zoomValid ? c.zoomFake : 0.f;
d.lastVehicle = o.vehicle;
d.lastScreen = o.screen;
}
void decide(Track& tr) {
std::vector<float> dyn;
double flame = 0, core = 0, red = 0, zoom = 0, veh = 0, scr = 0, sdx = 0, sdy = 0, real = 0,
path = 0;
int nz = 0, nreal = 0;
float rawMax = 0;
double contrast = 0;
int nContrast = 0;
int wide = 0;
std::vector<float> areas;
for (const auto& h : tr.areaHist) areas.push_back(h.second);
std::vector<float> zooms;
for (const auto& o : tr.win) {
rawMax = std::max(rawMax, o.raw);
if (o.contrast > 0) { contrast += o.contrast; ++nContrast; }
wide += o.wide ? 1 : 0;
if (o.zoom >= 0) zooms.push_back(o.zoom);
if (o.dyn >= 0) dyn.push_back(o.dyn);
flame += o.flame;
core += o.core;
red += o.red;
if (o.zoom >= 0) { zoom += o.zoom; ++nz; }
veh += o.vehicle ? 1 : 0;
scr += o.screen ? 1 : 0;
sdx += o.dx;
sdy += o.dy;
path += std::sqrt(double(o.dx) * o.dx + double(o.dy) * o.dy);
if (o.real > 0) { real += o.real; ++nreal; }
}
const double n = double(std::max<size_t>(1, tr.win.size()));
TrackDebug& d = tr.dbg;
d.id = tr.id;
d.smoke = tr.smoke;
d.box = tr.box;
d.evidence = tr.evidence;
d.obs = tr.nobs;
d.seconds = float(t_ - tr.t0);
d.placeSeconds = float(tr.placeSeen);
const int dynCount = int(tr.dynHist.size());
if (tr.dynHist.empty()) {
d.dynLow = 0.f;
d.frozen = 0.f;
} else {
std::vector<float> sorted;
sorted.reserve(tr.dynHist.size());
for (const auto& h : tr.dynHist) sorted.push_back(h.second);
const size_t q = sorted.size() / 4;
std::nth_element(sorted.begin(), sorted.begin() + q, sorted.end());
d.dynLow = sorted[q];
int frozen = 0;
for (float a : sorted) frozen += a < p_.dynMin ? 1 : 0;
d.frozen = float(frozen) / float(sorted.size());
}
d.dyn = dyn.empty() ? 0.f : detail::medianOf(dyn);
d.breath = 0.f;
if (areas.size() >= 2) {
double m = 0, v = 0;
for (float a : areas) m += a;
m /= double(areas.size());
for (float a : areas) v += (a - m) * (a - m);
d.breath = float(std::sqrt(v / double(areas.size())));
}
d.pose = -1.f;
if (int(tr.poseHist.size()) >= p_.rigidSmokeSamples &&
tr.poseHist.back().t - tr.poseHist.front().t >= p_.rigidSmokeSeconds) {
const double np = double(tr.poseHist.size());
double mcx = 0, mcy = 0, mlw = 0, mlh = 0, mw = 0, mh = 0;
for (const auto& q : tr.poseHist) {
mcx += q.cx; mcy += q.cy; mlw += q.lw; mlh += q.lh;
mw += std::exp(q.lw); mh += std::exp(q.lh);
}
mcx /= np; mcy /= np; mlw /= np; mlh /= np; mw /= np; mh /= np;
double vcx = 0, vcy = 0, vlw = 0, vlh = 0;
for (const auto& q : tr.poseHist) {
vcx += (q.cx - mcx) * (q.cx - mcx); vcy += (q.cy - mcy) * (q.cy - mcy);
vlw += (q.lw - mlw) * (q.lw - mlw); vlh += (q.lh - mlh) * (q.lh - mlh);
}
// Centre jitter in BOX units, so a far small plume and a near big one
// are measured by the same ruler.
d.pose = float(std::sqrt(vcx / np / std::max(1e-8, mw * mw) +
vcy / np / std::max(1e-8, mh * mh) + vlw / np + vlh / np));
if (d.pose < p_.rigidSmokePose) tr.rigid = true;
else if (d.pose >= 2.f * p_.rigidSmokePose) tr.rigid = false;
}
d.flame = float(flame / n);
d.core = float(core / n);
d.red = float(red / n);
d.zoom = nz >= 3 ? float(zoom / nz) : -1.f;
d.vehicle = float(veh / n);
d.screen = float(scr / n);
const double span = std::max(0.5, tr.win.back().t - tr.win.front().t);
const double net = std::sqrt(sdx * sdx + sdy * sdy);
d.drift = float(net / span);
d.straight = float(net / (path + 1e-6));
tr.meanReal = nreal ? float(real / nreal) : 0.f;
if (tr.smoke && nContrast > 0) {
const float meanContrast = float(contrast / nContrast);
if (meanContrast < p_.grainContrast && d.dyn >= p_.grainDyn) tr.grain = true;
else if (meanContrast >= 2.f * p_.grainContrast) tr.grain = false;
}
const char* why = "";
if (tr.evidence < p_.confirm) why = "evidence";
else if (tr.nobs < p_.minObs ||
d.seconds < (tr.smoke ? p_.smokeMinSeconds : p_.minSeconds))
why = "young";
else if (d.placeSeconds < (tr.smoke ? p_.smokePersistSeconds : p_.firePersistSeconds))
why = "brief";
else if (tr.smoke && mono_ && d.placeSeconds < p_.nightSmokeSeconds) why = "night-brief";
else if (d.vehicle >= p_.vehicleFrac && d.drift >= p_.vehicleDrift) why = "vehicle";
else if (d.screen >= p_.screenFrac) why = "screen";
else if (tr.smoke) {
// Over the window, not on the one look: the forest box shrank for a
// single frame in 84 s, and that frame was the alarm.
bool fireNow = false;
for (const auto& o : tracks_) fireNow = fireNow || (!o.smoke && o.confirmed);
if (2 * wide >= int(tr.win.size()) && d.flame < p_.unlocalisedFlame && !fireNow)
why = "unlocalised";
else if (haveWide_ && t_ - lastWideT_ < p_.hazeSeconds && d.flame < p_.unlocalisedFlame &&
!fireNow)
why = "in-haze";
else if (tr.box.w * tr.box.h >= p_.featurelessArea &&
nContrast > 0 && float(contrast / nContrast) < p_.featurelessContrast && d.dyn < p_.featurelessDyn &&
d.flame < p_.unlocalisedFlame)
why = "featureless";
else if (tr.grain) why = "grain";
else if (d.core >= p_.glareCore && d.dyn < p_.glareDyn) why = "glare";
else if (d.flame < p_.lampFlame &&
((d.core >= p_.lampCore && (d.dyn < p_.lampDyn || d.dynLow < p_.lampDynLow)) ||
(d.core >= p_.floodCore && d.dyn < p_.floodDyn) ||
(mono_ && d.core >= p_.irSmokeLampCore && d.dyn < p_.lampDyn)))
why = "lamp";
else if (net >= p_.travelBoxes && d.straight >= p_.travelStraight) why = "travelling";
else if (mono_ && tr.box.w * tr.box.h < p_.rigidSmokeArea &&
d.seconds < p_.rigidSmokeAge)
why = "young";
else if (mono_ && tr.box.w * tr.box.h < p_.rigidSmokeArea && tr.rigid) why = "rigid";
} else {
const float emissive = d.flame + d.core;
const bool zoomOk = d.zoom >= p_.zoomOk;
if (dynCount < (rawMax < p_.backedRaw ? p_.unbackedSamples : p_.dynMinSamples))
why = "young";
else if (mono_ && d.flame < p_.lampFlame && d.core >= p_.lampCore && d.dyn < p_.irLampDyn)
why = "lamp";
else if (d.dynLow < p_.dynMin) why = "static";
else if (int(areas.size()) >= p_.rigidSamples && d.breath < p_.rigidSpread) why = "rigid";
else if (int(areas.size()) >= p_.rigidSamples && d.frozen >= p_.frozenFrac &&
d.breath < p_.frozenSpread)
why = "static";
else if (nz >= 3 && d.zoom < p_.zoomDeny && d.dyn < p_.zoomDenyDyn) why = "zoom-denies";
else if (rawMax < p_.backedRaw && !zooms.empty() &&
detail::medianOf(zooms) < p_.backedZoom)
why = "unbacked";
else if (!(emissive >= p_.emissiveMin || zoomOk)) why = "not-emissive";
// No native-resolution override here, on purpose: up close a red flag
// with a yellow star DOES look like fire to the detector (0.6 measured).
else if (d.red >= p_.redMax && d.red >= p_.redRatio * d.flame)
why = "plain-red";
}
d.blockedBy = why;
if (!*why) tr.confirmed = true;
// Hysteresis: a confirmed track stays confirmed through a gate wobble and
// is only withdrawn when its evidence is gone (step 3 / the clamp above).
if (tr.evidence <= 0.f) tr.confirmed = false;
d.confirmed = tr.confirmed;
}
Params p_;
Gray prevGray_;
Motion motion_;
std::vector<Track> tracks_;
struct Ghost {
bool smoke;
RectF box;
double placeSeen, died;
std::deque<std::pair<double, float>> dynHist;
};
std::vector<Ghost> ghosts_; ///< places whose last track just ended
double t_ = 0, dt_ = 1.0 / 6.0;
bool mono_ = false;
bool haveWide_ = false;
double lastWideT_ = 0;
float chroma_ = -1.f;
bool haveT_ = false;
float aspect_ = 16.f / 9.f;
uint64_t nextId_ = 0;
};
} // namespace firesmoke
} // namespace ansmind