ANSCENTER/ANSCORE
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix ALPR Batch and memory leak

Browse Source
This commit is contained in:
Tuan Nghia Nguyen
2026-04-15 23:00:19 +10:00
parent b05c49ad93
commit 808df4656d
8 changed files with 1846 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

756
modules/ANSLPR/ANSLPR_OCR.cpp
View File

@@ -547,6 +547,471 @@ namespace ANSCENTER
return colour;
}
// ── Classical perspective rectification ─────────────────────────────
// Takes the axis-aligned LP YOLO bbox and tries to warp the plate to
// a tight rectangle whose height is fixed and whose width preserves
// the detected plate's actual aspect ratio. This removes camera
// tilt/yaw, strips background margin, and normalizes character
// spacing — which makes the recognizer see an image much closer to
// its training distribution and reduces silent character drops.
//
// Works entirely in classical OpenCV (Canny + findContours +
// approxPolyDP + getPerspectiveTransform + warpPerspective), so it
// needs no new models and no retraining. Fails gracefully (returns
// false) on plates where the border can't be isolated — caller falls
// back to the padded axis-aligned crop in that case.
std::vector<cv::Point2f>
ANSALPR_OCR::OrderQuadCorners(const std::vector<cv::Point>& pts) {
// Standard TL/TR/BR/BL ordering via x+y / y-x extrema. Robust to
// input winding order (clockwise vs counter-clockwise) and to
// approxPolyDP starting the polygon at an arbitrary corner.
std::vector<cv::Point2f> ordered(4);
if (pts.size() != 4) return ordered;
auto sum = [](const cv::Point& p) { return p.x + p.y; };
auto diff = [](const cv::Point& p) { return p.y - p.x; };
int idxMinSum = 0, idxMaxSum = 0, idxMinDiff = 0, idxMaxDiff = 0;
for (int i = 1; i < 4; ++i) {
if (sum(pts[i]) < sum(pts[idxMinSum])) idxMinSum = i;
if (sum(pts[i]) > sum(pts[idxMaxSum])) idxMaxSum = i;
if (diff(pts[i]) < diff(pts[idxMinDiff])) idxMinDiff = i;
if (diff(pts[i]) > diff(pts[idxMaxDiff])) idxMaxDiff = i;
}
ordered[0] = cv::Point2f(static_cast<float>(pts[idxMinSum].x), static_cast<float>(pts[idxMinSum].y)); // TL
ordered[1] = cv::Point2f(static_cast<float>(pts[idxMinDiff].x), static_cast<float>(pts[idxMinDiff].y)); // TR
ordered[2] = cv::Point2f(static_cast<float>(pts[idxMaxSum].x), static_cast<float>(pts[idxMaxSum].y)); // BR
ordered[3] = cv::Point2f(static_cast<float>(pts[idxMaxDiff].x), static_cast<float>(pts[idxMaxDiff].y)); // BL
return ordered;
}
bool ANSALPR_OCR::RectifyPlateROI(
const cv::Mat& source,
const cv::Rect& bbox,
cv::Mat& outRectified) const
{
if (source.empty()) return false;
cv::Rect clamped = bbox & cv::Rect(0, 0, source.cols, source.rows);
if (clamped.width <= 20 || clamped.height <= 10) return false;
const cv::Mat roi = source(clamped);
const double roiArea = static_cast<double>(roi.rows) * roi.cols;
const double minArea = roiArea * kRectifyAreaFraction;
// Step 1: grayscale + blur + Canny to find plate border edges.
cv::Mat gray;
if (roi.channels() == 3) {
cv::cvtColor(roi, gray, cv::COLOR_BGR2GRAY);
} else if (roi.channels() == 4) {
cv::cvtColor(roi, gray, cv::COLOR_BGRA2GRAY);
} else {
gray = roi;
}
cv::GaussianBlur(gray, gray, cv::Size(5, 5), 0);
cv::Mat edges;
cv::Canny(gray, edges, 50, 150);
// Close small gaps in the plate border so findContours sees it as
// one closed shape rather than several broken line segments.
cv::Mat kernel = cv::getStructuringElement(cv::MORPH_RECT, cv::Size(3, 3));
cv::morphologyEx(edges, edges, cv::MORPH_CLOSE, kernel);
// Step 2: find external contours.
std::vector<std::vector<cv::Point>> contours;
cv::findContours(edges, contours, cv::RETR_EXTERNAL, cv::CHAIN_APPROX_SIMPLE);
if (contours.empty()) return false;
// Step 3: find the largest contour whose approxPolyDP collapses
// to 4 vertices. That's most likely the plate border.
std::vector<cv::Point> bestQuad;
double bestArea = 0.0;
for (const auto& c : contours) {
const double area = cv::contourArea(c);
if (area < minArea) continue;
// Sweep epsilon — tighter approximations require more vertices,
// looser approximations collapse to fewer. We want the
// smallest epsilon at which the contour becomes a quadrilateral.
std::vector<cv::Point> approx;
const double perimeter = cv::arcLength(c, true);
for (double eps = 0.02; eps <= 0.08; eps += 0.01) {
cv::approxPolyDP(c, approx, eps * perimeter, true);
if (approx.size() == 4) break;
}
if (approx.size() == 4 && area > bestArea) {
// Verify the quadrilateral is convex — a non-convex
// 4-point contour is almost certainly not a plate
if (cv::isContourConvex(approx)) {
bestArea = area;
bestQuad = approx;
}
}
}
// Step 4: fallback — minAreaRect on the largest contour. This
// handles pure rotation but not arbitrary perspective skew.
if (bestQuad.empty()) {
auto largest = std::max_element(contours.begin(), contours.end(),
[](const std::vector<cv::Point>& a, const std::vector<cv::Point>& b) {
return cv::contourArea(a) < cv::contourArea(b);
});
if (largest == contours.end()) return false;
if (cv::contourArea(*largest) < minArea) return false;
cv::RotatedRect rr = cv::minAreaRect(*largest);
cv::Point2f pts[4];
rr.points(pts);
bestQuad.reserve(4);
for (int i = 0; i < 4; ++i) {
bestQuad.emplace_back(static_cast<int>(pts[i].x),
static_cast<int>(pts[i].y));
}
}
// Step 5: order the 4 corners as TL/TR/BR/BL.
std::vector<cv::Point2f> srcCorners = OrderQuadCorners(bestQuad);
// Measure the source quadrilateral's dimensions so the output
// rectangle preserves the real plate aspect ratio. Without this,
// a wide single-row plate would be squashed to 2:1 and a 2-row
// plate would be stretched to wrong proportions.
auto pointDist = [](const cv::Point2f& a, const cv::Point2f& b) -> float {
const float dx = a.x - b.x;
const float dy = a.y - b.y;
return std::sqrt(dx * dx + dy * dy);
};
const float topEdge = pointDist(srcCorners[0], srcCorners[1]);
const float bottomEdge = pointDist(srcCorners[3], srcCorners[2]);
const float leftEdge = pointDist(srcCorners[0], srcCorners[3]);
const float rightEdge = pointDist(srcCorners[1], srcCorners[2]);
const float srcW = std::max(topEdge, bottomEdge);
const float srcH = std::max(leftEdge, rightEdge);
if (srcW < 20.f || srcH < 10.f) return false;
const float srcAspect = srcW / srcH;
// Gate rectification on plausible plate aspect ratios. Anything
// wildly outside the range isn't a plate; fall back to the axis-
// aligned crop rather than produce a distorted warp.
if (srcAspect < kMinPlateAspect || srcAspect > kMaxPlateAspect) {
return false;
}
// Step 6: warp to a rectangle that preserves aspect ratio. Height
// is fixed (kRectifiedHeight) so downstream sizing is predictable.
const int outH = kRectifiedHeight;
const int outW = std::clamp(static_cast<int>(std::round(outH * srcAspect)),
kRectifiedHeight, // min: square
kRectifiedHeight * 6); // max: 6:1 long plates
std::vector<cv::Point2f> dstCorners = {
{ 0.f, 0.f },
{ static_cast<float>(outW - 1), 0.f },
{ static_cast<float>(outW - 1), static_cast<float>(outH - 1) },
{ 0.f, static_cast<float>(outH - 1) }
};
const cv::Mat M = cv::getPerspectiveTransform(srcCorners, dstCorners);
cv::warpPerspective(roi, outRectified, M, cv::Size(outW, outH),
cv::INTER_LINEAR, cv::BORDER_REPLICATE);
return !outRectified.empty();
}
// ── Japan-only: kana recovery on a plate where the fast path silently
// dropped the hiragana from the bottom row ────────────────────────
ANSALPR_OCR::CodepointClassCounts
ANSALPR_OCR::CountCodepointClasses(const std::string& text) {
CodepointClassCounts counts;
size_t pos = 0;
while (pos < text.size()) {
const size_t before = pos;
uint32_t cp = ANSOCRUtility::NextUTF8Codepoint(text, pos);
if (cp == 0 || pos == before) break;
if (ANSOCRUtility::IsCharClass(cp, CHAR_DIGIT)) counts.digit++;
if (ANSOCRUtility::IsCharClass(cp, CHAR_KANJI)) counts.kanji++;
if (ANSOCRUtility::IsCharClass(cp, CHAR_HIRAGANA)) counts.hiragana++;
if (ANSOCRUtility::IsCharClass(cp, CHAR_KATAKANA)) counts.katakana++;
}
return counts;
}
bool ANSALPR_OCR::IsJapaneseIncomplete(const std::string& text) {
// A valid Japanese plate has at least one kanji in the region
// zone, at least one hiragana/katakana in the kana zone, and at
// least four digits split between classification (top) and
// designation (bottom).
//
// We only consider a plate "incomplete and worth recovering"
// when it ALREADY LOOKS Japanese on the fast path — i.e. the
// kanji region was found successfully. Gating on kanji > 0
// prevents the recovery path from firing on non-Japanese plates
// (Latin-only, European, Macau, etc.) where there's no kana to
// find anyway, which previously wasted ~35 ms per plate burning
// all recovery attempts on a search that can never succeed.
//
// For non-Japanese plates the function returns false, recovery
// is skipped, and latency is identical to the pre-recovery
// baseline.
const CodepointClassCounts c = CountCodepointClasses(text);
if (c.kanji == 0) return false; // Not a Japanese plate
if (c.digit < 4) return false; // Not enough digits — probably garbage
const int kana = c.hiragana + c.katakana;
return (kana == 0); // Kanji + digits present, kana missing
}
// Strip screws/rivets/dirt that the recognizer misreads as small
// round punctuation glyphs. The blacklist is deliberately narrow:
// only characters that are never legitimate plate content on any
// country we support. Middle dots (・ and ·) are KEPT because they
// are legitimate padding on Japanese plates with <4 designation
// digits (e.g. "・274"), and they get normalised to "0" by
// ALPRPostProcessing's zone corrections anyway.
std::string ANSALPR_OCR::StripPlateArtifacts(const std::string& text) {
if (text.empty()) return text;
std::string stripped;
stripped.reserve(text.size());
size_t pos = 0;
while (pos < text.size()) {
const size_t before = pos;
uint32_t cp = ANSOCRUtility::NextUTF8Codepoint(text, pos);
if (cp == 0 || pos == before) break;
bool drop = false;
switch (cp) {
// Small round glyphs that mimic screws / rivets
case 0x00B0: // ° degree sign
case 0x02DA: // ˚ ring above
case 0x2218: // ∘ ring operator
case 0x25CB: // ○ white circle
case 0x25CF: // ● black circle
case 0x25E6: // ◦ white bullet
case 0x2022: // • bullet
case 0x2219: // ∙ bullet operator
case 0x25A0: // ■ black square
case 0x25A1: // □ white square
// Quote-like glyphs picked up from plate border / dirt
case 0x0022: // " ASCII double quote
case 0x0027: // ' ASCII apostrophe
case 0x201C: // " LEFT DOUBLE QUOTATION MARK (smart quote)
case 0x201D: // " RIGHT DOUBLE QUOTATION MARK
case 0x201E: // „ DOUBLE LOW-9 QUOTATION MARK
case 0x201F: // ‟ DOUBLE HIGH-REVERSED-9 QUOTATION MARK
case 0x2018: // ' LEFT SINGLE QUOTATION MARK
case 0x2019: // ' RIGHT SINGLE QUOTATION MARK
case 0x201A: // ‚ SINGLE LOW-9 QUOTATION MARK
case 0x201B: // ‛ SINGLE HIGH-REVERSED-9 QUOTATION MARK
case 0x00AB: // « LEFT-POINTING DOUBLE ANGLE QUOTATION
case 0x00BB: // » RIGHT-POINTING DOUBLE ANGLE QUOTATION
case 0x2039: // ‹ SINGLE LEFT-POINTING ANGLE QUOTATION
case 0x203A: // › SINGLE RIGHT-POINTING ANGLE QUOTATION
case 0x301D: // 〝 REVERSED DOUBLE PRIME QUOTATION
case 0x301E: // 〞 DOUBLE PRIME QUOTATION
case 0x301F: // 〟 LOW DOUBLE PRIME QUOTATION
case 0x300A: // 《 LEFT DOUBLE ANGLE BRACKET
case 0x300B: // 》 RIGHT DOUBLE ANGLE BRACKET
case 0x3008: // 〈 LEFT ANGLE BRACKET
case 0x3009: // 〉 RIGHT ANGLE BRACKET
// Ideographic punctuation that isn't valid plate content
case 0x3002: // 。 ideographic full stop
case 0x3001: // 、 ideographic comma
case 0x300C: // 「 left corner bracket
case 0x300D: // 」 right corner bracket
case 0x300E: // 『 left white corner bracket
case 0x300F: // 』 right white corner bracket
// ASCII punctuation noise picked up from plate borders
case 0x0060: // ` grave accent
case 0x007E: // ~ tilde
case 0x005E: // ^ caret
case 0x007C: // | vertical bar
case 0x005C: // \ backslash
case 0x002F: // / forward slash
case 0x0028: // ( left paren
case 0x0029: // ) right paren
case 0x005B: // [ left bracket
case 0x005D: // ] right bracket
case 0x007B: // { left brace
case 0x007D: // } right brace
case 0x003C: // < less than
case 0x003E: // > greater than
// Misc symbols that round glyphs can collapse to
case 0x00A9: // © copyright sign
case 0x00AE: // ® registered sign
case 0x2117: // ℗ sound recording copyright
case 0x2122: // ™ trademark
drop = true;
break;
default:
break;
}
if (!drop) {
stripped.append(text, before, pos - before);
}
}
// Collapse runs of spaces introduced by stripping, and trim.
std::string collapsed;
collapsed.reserve(stripped.size());
bool prevSpace = false;
for (char c : stripped) {
if (c == ' ') {
if (!prevSpace) collapsed.push_back(c);
prevSpace = true;
} else {
collapsed.push_back(c);
prevSpace = false;
}
}
const size_t first = collapsed.find_first_not_of(' ');
if (first == std::string::npos) return "";
const size_t last = collapsed.find_last_not_of(' ');
return collapsed.substr(first, last - first + 1);
}
std::string ANSALPR_OCR::RecoverKanaFromBottomHalf(
const cv::Mat& plateROI, int halfH) const
{
if (!_ocrEngine || plateROI.empty()) return "";
const int plateW = plateROI.cols;
const int plateH = plateROI.rows;
if (plateW < 40 || plateH < 30 || halfH <= 0 || halfH >= plateH) {
ANS_DBG("ALPR_Kana",
"Recovery SKIP: plate too small (%dx%d, halfH=%d)",
plateW, plateH, halfH);
return "";
}
ANS_DBG("ALPR_Kana",
"Recovery START: plate=%dx%d halfH=%d bottomHalf=%dx%d",
plateW, plateH, halfH, plateW, plateH - halfH);
// The kana on a Japanese plate sits in the left ~30% of the
// bottom row and is roughly square. Try 3 well-chosen crop
// positions — one center, one slightly high, one wider — and
// bail out on the first that yields a hiragana/katakana hit.
//
// 3 attempts is the sweet spot: it catches the common row-split
// variation without burning linear time on every fail-case.
// Previous versions tried 7 attempts, which added ~20 ms/plate
// of pure waste when recovery couldn't find any kana anyway.
//
// Tiles shorter than 48 px are upscaled to 48 px height before
// recognition so the recognizer sees something close to its
// training distribution. PaddleOCR's rec model expects 48 px
// height and breaks down when given very small crops.
struct TileSpec {
float widthFraction; // fraction of plateW
float yOffset; // 0.0 = top of bottom half, 1.0 = bottom
};
const TileSpec attempts[] = {
{ 0.30f, 0.50f }, // primary: 30% wide, centered vertically
{ 0.30f, 0.35f }, // row split landed too low — try higher
{ 0.35f, 0.50f }, // slightly wider crop for off-center kana
};
int attemptNo = 0;
for (const TileSpec& spec : attempts) {
attemptNo++;
int tileW = static_cast<int>(plateW * spec.widthFraction);
if (tileW < 30) tileW = 30;
if (tileW > plateW) tileW = plateW;
// Prefer square tile, but allow non-square if the bottom
// half is short. Clipped to bottom-half height.
int tileH = tileW;
const int bottomHalfH = plateH - halfH;
if (tileH > bottomHalfH) tileH = bottomHalfH;
if (tileH < 20) continue;
const int centerY = halfH + static_cast<int>(bottomHalfH * spec.yOffset);
int cy = centerY - tileH / 2;
if (cy < halfH) cy = halfH;
if (cy + tileH > plateH) cy = plateH - tileH;
if (cy < 0) cy = 0;
const int cx = 0;
int cw = tileW;
int ch = tileH;
if (cx + cw > plateW) cw = plateW - cx;
if (cy + ch > plateH) ch = plateH - cy;
if (cw <= 10 || ch <= 10) continue;
cv::Mat kanaTile = plateROI(cv::Rect(cx, cy, cw, ch));
// Upscale tiles shorter than 48 px so the recognizer sees
// something close to its training input size. Preserve
// aspect ratio; cv::INTER_CUBIC keeps character strokes
// sharper than bilinear.
cv::Mat tileForRec;
if (kanaTile.rows < 48) {
const double scale = 48.0 / kanaTile.rows;
cv::resize(kanaTile, tileForRec, cv::Size(),
scale, scale, cv::INTER_CUBIC);
} else {
tileForRec = kanaTile;
}
std::vector<cv::Mat> tileBatch{ tileForRec };
auto tileResults = _ocrEngine->RecognizeTextBatch(tileBatch);
if (tileResults.empty()) {
ANS_DBG("ALPR_Kana",
"Attempt %d: tile=%dx%d (rec=%dx%d w=%.2f y=%.2f) "
"→ recognizer returned empty batch",
attemptNo, cw, ch, tileForRec.cols, tileForRec.rows,
spec.widthFraction, spec.yOffset);
continue;
}
const std::string& text = tileResults[0].first;
const float conf = tileResults[0].second;
ANS_DBG("ALPR_Kana",
"Attempt %d: tile=%dx%d (rec=%dx%d w=%.2f y=%.2f) "
"→ '%s' conf=%.3f",
attemptNo, cw, ch, tileForRec.cols, tileForRec.rows,
spec.widthFraction, spec.yOffset, text.c_str(), conf);
if (text.empty()) continue;
// Japanese plate kana is ALWAYS exactly 1 hiragana or
// katakana character. We accept ONLY that — nothing else.
// Kanji, Latin letters, digits, punctuation, everything
// non-kana is rejected. The returned string is exactly the
// one kana codepoint or empty.
//
// Strictness is deliberate: the relaxed "any letter class"
// accept path was letting through kanji bleed from the
// region-name zone when the tile positioning was slightly
// off, producing wrong plate text like "59-V3 西 752.23" or
// "JCL 三". With strict-only accept, a miss in the recovery
// is silent and the fast-path result passes through unchanged.
std::string firstKana; // first CHAR_HIRAGANA / CHAR_KATAKANA hit
int codepointCount = 0;
size_t pos = 0;
while (pos < text.size()) {
const size_t before = pos;
uint32_t cp = ANSOCRUtility::NextUTF8Codepoint(text, pos);
if (cp == 0 || pos == before) break;
codepointCount++;
if (!firstKana.empty()) continue;
if (ANSOCRUtility::IsCharClass(cp, CHAR_HIRAGANA) ||
ANSOCRUtility::IsCharClass(cp, CHAR_KATAKANA)) {
firstKana = text.substr(before, pos - before);
}
}
if (!firstKana.empty()) {
ANS_DBG("ALPR_Kana",
"Recovery SUCCESS at attempt %d: extracted '%s' "
"from raw '%s' (%d codepoints, conf=%.3f)",
attemptNo, firstKana.c_str(), text.c_str(),
codepointCount, conf);
return firstKana;
}
}
ANS_DBG("ALPR_Kana",
"Recovery FAILED: no kana found in %d attempts",
attemptNo);
return "";
}
// ── Full-frame vs pipeline auto-detection ────────────────────────────
// Mirror of ANSALPR_OD::shouldUseALPRChecker. The auto-detection logic
// watches whether consecutive frames from a given camera have the exact
@@ -818,16 +1283,37 @@ namespace ANSCENTER
}
// Step 2: Collect crops from every valid plate. Wide plates
// (aspect >= 2.0) are treated as a single text line; narrow
// (aspect >= 2.1) are treated as a single text line; narrow
// plates (2-row layouts like Japanese) are split horizontally
// at H/2 into top and bottom rows. All crops go through a
// single batched recognizer call, bypassing the OCR text-line
// detector entirely — for ALPR the LP YOLO box already bounds
// the text region precisely.
//
// Per-plate preprocessing pipeline:
// 1. Pad the YOLO LP bbox by 5% on each side so the plate
// border is visible to the rectifier and edge characters
// aren't clipped by a tight detector output.
// 2. Try classical perspective rectification (Canny +
// findContours + approxPolyDP + warpPerspective) to
// straighten tilted / skewed plates. Falls back to the
// padded axis-aligned crop on failure — no regression.
// 3. Run the 2-row split heuristic on whichever plate image
// we ended up with, using an aspect threshold of 2.1 so
// perfect-2:1 rectified Japanese plates still split.
//
// Rectification is gated on _country == JAPAN at runtime.
// For all other countries we skip the classical-CV pipeline
// entirely and use the plain padded axis-aligned crop — this
// keeps non-Japan inference on the original fast path and
// lets SetCountry(nonJapan) take effect on the very next
// frame without a restart.
const bool useRectification = (_country == Country::JAPAN);
struct PlateInfo {
size_t origIndex; // into lprOutput
std::vector<size_t> cropIndices; // into allCrops
cv::Mat plateROI; // full (unsplit) ROI, kept for colour
cv::Mat plateROI; // full (unsplit) ROI, kept for colour + kana recovery
int halfH = 0; // row-split Y inside plateROI (0 = single row)
};
std::vector<cv::Mat> allCrops;
std::vector<PlateInfo> plateInfos;
@@ -842,30 +1328,58 @@ namespace ANSCENTER
const int y1 = std::max(0, box.y);
const int width = std::min(frameWidth - x1, box.width);
const int height = std::min(frameHeight - y1, box.height);
if (width <= 0 || height <= 0) continue;
cv::Mat plateROI = frame(cv::Rect(x1, y1, width, height));
// Pad the YOLO LP bbox by 5% on each side. Gives the
// rectifier some background for edge detection and helps
// when the detector cropped a character edge.
const int padX = std::max(2, width * 5 / 100);
const int padY = std::max(2, height * 5 / 100);
const int px = std::max(0, x1 - padX);
const int py = std::max(0, y1 - padY);
const int pw = std::min(frameWidth - px, width + 2 * padX);
const int ph = std::min(frameHeight - py, height + 2 * padY);
const cv::Rect paddedBox(px, py, pw, ph);
// Perspective rectification is Japan-only to preserve
// baseline latency on all other countries. On non-Japan
// plates we go straight to the padded axis-aligned crop.
cv::Mat plateROI;
if (useRectification) {
cv::Mat rectified;
if (RectifyPlateROI(frame, paddedBox, rectified)) {
plateROI = rectified; // owning 3-channel BGR
} else {
plateROI = frame(paddedBox); // non-owning view
}
} else {
plateROI = frame(paddedBox); // non-owning view
}
PlateInfo info;
info.origIndex = i;
info.plateROI = plateROI;
const float aspect = static_cast<float>(width) /
std::max(1, height);
const int plateW = plateROI.cols;
const int plateH = plateROI.rows;
const float aspect = static_cast<float>(plateW) /
std::max(1, plateH);
// 2-row heuristic: aspect < 2.0 → split top/bottom.
// Threshold tuned to catch Japanese square plates
// (~1.5–1.9) while leaving wide EU/VN plates (3.0+)
// untouched.
if (aspect < 2.0f && height >= 24) {
const int halfH = height / 2;
// 2-row heuristic: aspect < 2.1 → split top/bottom.
// Bumped from 2.0 so a perfectly rectified Japanese plate
// (aspect == 2.0) still splits correctly despite floating-
// point rounding. Threshold still excludes wide EU/VN
// plates (aspect 3.0+).
if (aspect < 2.1f && plateH >= 24) {
const int halfH = plateH / 2;
info.halfH = halfH;
info.cropIndices.push_back(allCrops.size());
allCrops.push_back(plateROI(cv::Rect(0, 0, width, halfH)));
allCrops.push_back(plateROI(cv::Rect(0, 0, plateW, halfH)));
info.cropIndices.push_back(allCrops.size());
allCrops.push_back(plateROI(cv::Rect(0, halfH, width, height - halfH)));
allCrops.push_back(plateROI(cv::Rect(0, halfH, plateW, plateH - halfH)));
}
else {
info.halfH = 0;
info.cropIndices.push_back(allCrops.size());
allCrops.push_back(plateROI);
}
@@ -895,14 +1409,68 @@ namespace ANSCENTER
cv::Size(frameWidth, frameHeight), cameraId);
for (const auto& info : plateInfos) {
std::string combinedText;
for (size_t cropIdx : info.cropIndices) {
if (cropIdx >= ocrResults.size()) continue;
const std::string& lineText = ocrResults[cropIdx].first;
if (lineText.empty()) continue;
if (!combinedText.empty()) combinedText += " ";
combinedText += lineText;
// Reassemble row-by-row so we can target the bottom row
// for kana recovery when the fast path silently dropped
// the hiragana on a Japanese 2-row plate.
std::string topText, bottomText;
if (info.cropIndices.size() == 2) {
if (info.cropIndices[0] < ocrResults.size())
topText = ocrResults[info.cropIndices[0]].first;
if (info.cropIndices[1] < ocrResults.size())
bottomText = ocrResults[info.cropIndices[1]].first;
} else if (!info.cropIndices.empty() &&
info.cropIndices[0] < ocrResults.size()) {
topText = ocrResults[info.cropIndices[0]].first;
}
// Strip screw/rivet artifacts (°, ○, etc.) picked up from
// plate fasteners before any downstream processing. Runs
// on every row regardless of country — these glyphs are
// never legitimate plate content anywhere.
topText = StripPlateArtifacts(topText);
bottomText = StripPlateArtifacts(bottomText);
std::string combinedText = topText;
if (!bottomText.empty()) {
if (!combinedText.empty()) combinedText += " ";
combinedText += bottomText;
}
// Japan-only kana recovery: if the fast-path output is
// missing hiragana/katakana, re-crop the kana region and
// run the recognizer on just that tile. Clean plates
// pass the IsJapaneseIncomplete check and skip this
// block entirely — zero cost.
if (_country == Country::JAPAN && info.halfH > 0 &&
IsJapaneseIncomplete(combinedText)) {
ANS_DBG("ALPR_Kana",
"RunInference: firing recovery on plate '%s' "
"(plateROI=%dx%d halfH=%d)",
combinedText.c_str(),
info.plateROI.cols, info.plateROI.rows,
info.halfH);
std::string recovered = StripPlateArtifacts(
RecoverKanaFromBottomHalf(info.plateROI, info.halfH));
if (!recovered.empty()) {
// Prepend the recovered kana to the bottom row
// text so the final combined string reads
// "region classification kana designation".
if (bottomText.empty()) {
bottomText = recovered;
} else {
bottomText = recovered + " " + bottomText;
}
combinedText = topText;
if (!bottomText.empty()) {
if (!combinedText.empty()) combinedText += " ";
combinedText += bottomText;
}
ANS_DBG("ALPR_Kana",
"RunInference: spliced result '%s'",
combinedText.c_str());
}
}
if (combinedText.empty()) continue;
Object lprObject = lprOutput[info.origIndex];
@@ -1014,16 +1582,27 @@ namespace ANSCENTER
std::vector<std::vector<Object>> lpBatch =
_lpDetector->RunInferencesBatch(vehicleCrops, cameraId);
// ── 3. Flatten plates, splitting 2-row plates into top/bot ─
// Same aspect-ratio heuristic as ANSALPR_OCR::RunInference
// (lines ~820-870): narrow plates (aspect < 2.0) are split
// horizontally into two recognizer crops, wide plates stay as
// one. The recMap lets us stitch the per-crop OCR outputs
// back into per-plate combined strings.
// ── 3. Flatten plates, applying preprocessing per plate ───
// For each detected plate we:
// 1. Pad the LP bbox by 5% so the rectifier sees the
// plate border and tight detector crops don't clip
// edge characters.
// 2. If country == JAPAN, try classical perspective
// rectification — if it succeeds the plateROI is a
// tight, straightened 2D warp of the real plate; if
// it fails we fall back to the padded axis-aligned
// crop. For non-Japan countries we skip rectification
// entirely to preserve baseline latency.
// 3. Apply the same 2-row split heuristic as RunInference
// (aspect < 2.1 → split top/bottom).
// The halfH field lets the assembly loop call the kana
// recovery helper with the correct row-split boundary.
const bool useRectification = (_country == Country::JAPAN);
struct PlateMeta {
size_t vehIdx; // index into vehicleCrops / clamped
Object lpObj; // LP detection in VEHICLE-local coords
cv::Mat plateROI; // full plate crop (kept for colour)
size_t vehIdx; // index into vehicleCrops / clamped
Object lpObj; // LP detection in VEHICLE-local coords
cv::Mat plateROI; // full plate crop (kept for colour + kana recovery)
int halfH = 0; // row-split Y inside plateROI (0 = single row)
std::vector<size_t> cropIndices; // indices into allCrops below
};
std::vector<cv::Mat> allCrops;
@@ -1036,23 +1615,49 @@ namespace ANSCENTER
for (const auto& lp : lpBatch[v]) {
cv::Rect lpBox = lp.box & vehRect;
if (lpBox.width <= 0 || lpBox.height <= 0) continue;
cv::Mat plateROI = veh(lpBox);
// Pad by 5% on each side for the rectifier.
const int padX = std::max(2, lpBox.width * 5 / 100);
const int padY = std::max(2, lpBox.height * 5 / 100);
cv::Rect paddedBox(
lpBox.x - padX, lpBox.y - padY,
lpBox.width + 2 * padX,
lpBox.height + 2 * padY);
paddedBox &= vehRect;
if (paddedBox.width <= 0 || paddedBox.height <= 0) continue;
// Perspective rectification is Japan-only to preserve
// baseline latency on all other countries.
cv::Mat plateROI;
if (useRectification) {
cv::Mat rectified;
if (RectifyPlateROI(veh, paddedBox, rectified)) {
plateROI = rectified; // owning canonical
} else {
plateROI = veh(paddedBox); // non-owning view
}
} else {
plateROI = veh(paddedBox); // non-owning view
}
PlateMeta pm;
pm.vehIdx = v;
pm.lpObj = lp;
pm.plateROI = plateROI;
const int plateW = plateROI.cols;
const int plateH = plateROI.rows;
const float aspect =
static_cast<float>(plateROI.cols) /
std::max(1, plateROI.rows);
if (aspect < 2.0f && plateROI.rows >= 24) {
const int halfH = plateROI.rows / 2;
static_cast<float>(plateW) / std::max(1, plateH);
if (aspect < 2.1f && plateH >= 24) {
const int halfH = plateH / 2;
pm.halfH = halfH;
pm.cropIndices.push_back(allCrops.size());
allCrops.push_back(plateROI(cv::Rect(0, 0, plateROI.cols, halfH)));
allCrops.push_back(plateROI(cv::Rect(0, 0, plateW, halfH)));
pm.cropIndices.push_back(allCrops.size());
allCrops.push_back(plateROI(cv::Rect(0, halfH, plateROI.cols, plateROI.rows - halfH)));
allCrops.push_back(plateROI(cv::Rect(0, halfH, plateW, plateH - halfH)));
} else {
pm.halfH = 0;
pm.cropIndices.push_back(allCrops.size());
allCrops.push_back(plateROI);
}
@@ -1070,14 +1675,59 @@ namespace ANSCENTER
std::vector<Object> output;
output.reserve(metas.size());
for (const auto& pm : metas) {
std::string combined;
for (size_t c : pm.cropIndices) {
if (c >= ocrResults.size()) continue;
const std::string& line = ocrResults[c].first;
if (line.empty()) continue;
if (!combined.empty()) combined += " ";
combined += line;
// Reassemble row-by-row so Japan kana recovery can splice
// the recovered hiragana into the bottom row specifically.
std::string topText, bottomText;
if (pm.cropIndices.size() == 2) {
if (pm.cropIndices[0] < ocrResults.size())
topText = ocrResults[pm.cropIndices[0]].first;
if (pm.cropIndices[1] < ocrResults.size())
bottomText = ocrResults[pm.cropIndices[1]].first;
} else if (!pm.cropIndices.empty() &&
pm.cropIndices[0] < ocrResults.size()) {
topText = ocrResults[pm.cropIndices[0]].first;
}
// Strip screw/rivet artifacts (°, ○, etc.) picked up from
// plate fasteners before any downstream processing.
topText = StripPlateArtifacts(topText);
bottomText = StripPlateArtifacts(bottomText);
std::string combined = topText;
if (!bottomText.empty()) {
if (!combined.empty()) combined += " ";
combined += bottomText;
}
// Japan-only kana recovery fast-path fallback. Zero cost
// on clean plates (gated by country and by UTF-8 codepoint
// class count — clean plates return early).
if (_country == Country::JAPAN && pm.halfH > 0 &&
IsJapaneseIncomplete(combined)) {
ANS_DBG("ALPR_Kana",
"RunInferencesBatch: firing recovery on plate "
"'%s' (plateROI=%dx%d halfH=%d)",
combined.c_str(),
pm.plateROI.cols, pm.plateROI.rows, pm.halfH);
std::string recovered = StripPlateArtifacts(
RecoverKanaFromBottomHalf(pm.plateROI, pm.halfH));
if (!recovered.empty()) {
if (bottomText.empty()) {
bottomText = recovered;
} else {
bottomText = recovered + " " + bottomText;
}
combined = topText;
if (!bottomText.empty()) {
if (!combined.empty()) combined += " ";
combined += bottomText;
}
ANS_DBG("ALPR_Kana",
"RunInferencesBatch: spliced result '%s'",
combined.c_str());
}
}
if (combined.empty()) continue;
Object out = pm.lpObj;
@@ -1183,10 +1833,28 @@ namespace ANSCENTER
}
void ANSALPR_OCR::SetCountry(Country country) {
const Country previous = _country;
_country = country;
if (_ocrEngine) {
_ocrEngine->SetCountry(country);
}
// Log every SetCountry call so runtime country switches are
// visible and we can confirm the update landed on the right
// handle. The recovery + rectification gates read _country on
// every frame, so this change takes effect on the very next
// RunInference / RunInferencesBatch call — no restart needed.
ANS_DBG("ALPR_SetCountry",
"country changed %d -> %d (Japan=%d, Vietnam=%d, "
"China=%d, Australia=%d, USA=%d, Indonesia=%d) — "
"rectification+recovery gates update on next frame",
static_cast<int>(previous),
static_cast<int>(country),
static_cast<int>(Country::JAPAN),
static_cast<int>(Country::VIETNAM),
static_cast<int>(Country::CHINA),
static_cast<int>(Country::AUSTRALIA),
static_cast<int>(Country::USA),
static_cast<int>(Country::INDONESIA));
}
bool ANSALPR_OCR::Destroy() {

73
modules/ANSLPR/ANSLPR_OCR.h
View File

@@ -125,6 +125,79 @@ namespace ANSCENTER
// --- OCR helper ---
[[nodiscard]] std::string RunOCROnPlate(const cv::Mat& plateROI, const std::string& cameraId);
// ----------------------------------------------------------------
// Plate preprocessing: classical perspective rectification
//
// Takes an LP YOLO bounding box and tries to find the plate's
// actual 4 corners via Canny + findContours + approxPolyDP. When
// that succeeds, the plate is warped to a rectangle whose height
// is fixed (kRectifiedHeight) and whose width preserves the
// detected plate's aspect ratio. This produces a tight,
// perspective-corrected crop that the recognizer handles more
// reliably than the tilted / skewed axis-aligned bbox.
//
// Falls back to minAreaRect on the largest contour if no 4-point
// polygon is found, and returns false outright if nothing
// plausible can be isolated. Callers must handle the false case
// by using the (padded) axis-aligned crop instead.
// ----------------------------------------------------------------
static constexpr int kRectifiedHeight = 220;
static constexpr float kMinPlateAspect = 1.3f;
static constexpr float kMaxPlateAspect = 6.0f;
static constexpr float kRectifyAreaFraction = 0.30f;
[[nodiscard]] bool RectifyPlateROI(
const cv::Mat& source,
const cv::Rect& bbox,
cv::Mat& outRectified) const;
// Order an arbitrary quadrilateral as
// [top-left, top-right, bottom-right, bottom-left] (in that order)
// using the x+y / y-x extreme trick so perspective transforms land
// right-side-up regardless of input winding.
[[nodiscard]] static std::vector<cv::Point2f>
OrderQuadCorners(const std::vector<cv::Point>& pts);
// ----------------------------------------------------------------
// Japan-only: targeted kana recovery
//
// The PaddleOCR v5 recognizer's CTC decoder silently drops a
// character when it sits next to a large blank region in the
// input image — which is exactly the layout of the bottom row
// of a Japanese plate (single small hiragana on the left, big
// gap, then 4 digits on the right). We detect this failure
// mode by counting UTF-8 codepoint classes in the fast-path
// output, and if hiragana/katakana is missing we re-run the
// recognizer on a tight crop of the kana region only. The
// recognizer handles that tight crop correctly because the
// input matches what it was trained on (a dense text-line
// image with no large blank stretches).
// ----------------------------------------------------------------
struct CodepointClassCounts {
int digit = 0;
int kanji = 0;
int hiragana = 0;
int katakana = 0;
};
[[nodiscard]] static CodepointClassCounts CountCodepointClasses(const std::string& text);
[[nodiscard]] static bool IsJapaneseIncomplete(const std::string& text);
// Strip non-text artifacts (screws, rivets, dirt, stickers) that
// the OCR recognizer occasionally picks up from plate surface
// features. These glyphs (degree sign, ring above, circles,
// ideographic punctuation, etc.) are never legitimate plate
// characters in any supported country, so we can drop them
// unconditionally. Runs of spaces resulting from stripped
// characters are collapsed and leading/trailing spaces trimmed.
[[nodiscard]] static std::string StripPlateArtifacts(const std::string& text);
// Run recognizer-only on a tight crop of the left portion of the
// bottom half, trying three vertical offsets to absorb row-split
// inaccuracies. Returns the first non-empty result that contains
// a hiragana or katakana codepoint, or empty string on failure.
[[nodiscard]] std::string RecoverKanaFromBottomHalf(
const cv::Mat& plateROI, int halfH) const;
public:
ANSALPR_OCR();
~ANSALPR_OCR();