feat(ANSODEngine): run Darknet/YOLO exports ("darknet" family) in ONNXYOLO and RTYOLO
A Darknet/YOLO model exported by hank-ai's darknet_onnx_export (v5.1+) has two outputs: confs [B,N,nc], already objectness x class probability, and boxes [B,N,1,4], corners x1,y1,x2,y2 normalized to the input. It trains and predicts on a stretched square and has no NMS in the graph. ONNXYOLO took the 4-D boxes tensor for segmentation proto masks and letterboxed the input, so such a model decoded as garbage; a sealed package named as YOLOV4 went to YOLOOD, which cannot open an .ansmodel at all. - ONNXYOLO / RTYOLO: recognise the layout once at load (package family "darknet", output names confs/boxes, or their shapes), before the seg check; stretch-resize the input (single, batch, and RTYOLO's NV12 path); new postprocessDarknet: best class per candidate, threshold, stretch map to the original image, clip, class-aware NMS. - dllmain: a sealed package passed as modelType 1 (YOLOV4) is routed as detection (3), i.e. to RTYOLO on NVIDIA and ONNXYOLO elsewhere, in CreateANSODHandle, CreateANSODHandleEx and OptimizeModelStr. - Every other model is unchanged: all new branches are gated on the layout. Verified on macOS only: ANSONNXYOLO.cpp passes a syntax check, and the new decode functions, extracted into a standalone harness running the model through onnxruntime, reproduce Darknet's own detections on dog.jpg. NOT yet built on Windows, and the TensorRT path has not run. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -169,6 +169,7 @@ namespace ANSCENTER {
|
||||
}
|
||||
m_options.maxBatchSize = m_trtEngine->getOptions().maxBatchSize;
|
||||
m_options.optBatchSize = m_trtEngine->getOptions().optBatchSize;
|
||||
DetectDarknetLayout();
|
||||
m_trtEngine->warmUp();
|
||||
}
|
||||
_modelLoadValid = true;
|
||||
@@ -273,6 +274,7 @@ namespace ANSCENTER {
|
||||
}
|
||||
m_options.maxBatchSize = m_trtEngine->getOptions().maxBatchSize;
|
||||
m_options.optBatchSize = m_trtEngine->getOptions().optBatchSize;
|
||||
DetectDarknetLayout();
|
||||
m_trtEngine->warmUp();
|
||||
}
|
||||
_modelLoadValid = true;
|
||||
@@ -388,6 +390,7 @@ namespace ANSCENTER {
|
||||
}
|
||||
m_options.maxBatchSize = m_trtEngine->getOptions().maxBatchSize;
|
||||
m_options.optBatchSize = m_trtEngine->getOptions().optBatchSize;
|
||||
DetectDarknetLayout();
|
||||
m_trtEngine->warmUp();
|
||||
}
|
||||
_modelLoadValid = true;
|
||||
@@ -511,6 +514,9 @@ namespace ANSCENTER {
|
||||
if (srcImg.rows != inputH || srcImg.cols != inputW) {
|
||||
if (isClassification) {
|
||||
cv::resize(srcImg, cpuResized, cv::Size(inputW, inputH), 0, 0, cv::INTER_LINEAR);
|
||||
} else if (_isDarknet) {
|
||||
// Darknet: stretch the whole frame (no letterbox)
|
||||
cv::resize(srcImg, cpuResized, cv::Size(inputW, inputH), 0, 0, cv::INTER_LINEAR);
|
||||
} else {
|
||||
cpuResized = Engine<float>::cpuResizeKeepAspectRatioPadRightBottom(srcImg, inputH, inputW);
|
||||
}
|
||||
@@ -921,12 +927,15 @@ namespace ANSCENTER {
|
||||
|
||||
const float scaleW = inputW / static_cast<float>(srcImg.cols);
|
||||
const float scaleH = inputH / static_cast<float>(srcImg.rows);
|
||||
outMetadata.ratios[i] = isClassification ? 1.f : 1.f / std::min(scaleW, scaleH);
|
||||
outMetadata.ratios[i] = (isClassification || _isDarknet) ? 1.f : 1.f / std::min(scaleW, scaleH);
|
||||
|
||||
cv::Mat cpuResized;
|
||||
if (srcImg.rows != inputH || srcImg.cols != inputW) {
|
||||
if (isClassification) {
|
||||
cv::resize(srcImg, cpuResized, cv::Size(inputW, inputH), 0, 0, cv::INTER_LINEAR);
|
||||
} else if (_isDarknet) {
|
||||
// Darknet: stretch the whole frame (no letterbox)
|
||||
cv::resize(srcImg, cpuResized, cv::Size(inputW, inputH), 0, 0, cv::INTER_LINEAR);
|
||||
} else {
|
||||
cpuResized = Engine<float>::cpuResizeKeepAspectRatioPadRightBottom(srcImg, inputH, inputW);
|
||||
}
|
||||
@@ -1814,6 +1823,117 @@ namespace ANSCENTER {
|
||||
}
|
||||
}
|
||||
|
||||
// ====================================================================
|
||||
// Darknet — layout detection + postprocess
|
||||
//
|
||||
// hank-ai darknet_onnx_export (v5.1+): confs [B,N,nc] is already
|
||||
// objectness x class probability; boxes [B,N,1,4] holds corners
|
||||
// x1,y1,x2,y2 normalised 0..1 to the (stretched) network input. The
|
||||
// graph has no NMS. Best class per candidate, threshold, map to the
|
||||
// original image (a stretch, so x*imgWidth / y*imgHeight), clip, then
|
||||
// class-aware NMS. TensorRT exposes no output names here, so the
|
||||
// boxes tensor is the one whose last dim is 4.
|
||||
// ====================================================================
|
||||
void ANSRTYOLO::DetectDarknetLayout() {
|
||||
_isDarknet = false;
|
||||
if (_fromPackage && _packageFamily == "darknet") {
|
||||
_isDarknet = true;
|
||||
return;
|
||||
}
|
||||
if (!m_trtEngine) return;
|
||||
const auto& outputDims = m_trtEngine->getOutputDims();
|
||||
if (outputDims.size() != 2) return;
|
||||
|
||||
auto isBoxes = [](const nvinfer1::Dims& d) {
|
||||
return (d.nbDims == 4 && d.d[2] == 1 && d.d[3] == 4) ||
|
||||
(d.nbDims == 3 && d.d[2] == 4);
|
||||
};
|
||||
const bool b0 = isBoxes(outputDims[0]);
|
||||
const bool b1 = isBoxes(outputDims[1]);
|
||||
if (b0 == b1) return;
|
||||
const auto& boxes = b0 ? outputDims[0] : outputDims[1];
|
||||
const auto& confs = b0 ? outputDims[1] : outputDims[0];
|
||||
if (confs.nbDims != 3 || confs.d[2] < 1) return;
|
||||
if (confs.d[1] <= 0 || confs.d[1] != boxes.d[1]) return;
|
||||
_isDarknet = true;
|
||||
}
|
||||
|
||||
std::vector<Object> ANSRTYOLO::PostprocessDarknet(
|
||||
std::vector<std::vector<float>>& featureVectors,
|
||||
const std::string& camera_id, const ImageMetadata& meta) {
|
||||
try {
|
||||
const auto& outputDims = m_trtEngine->getOutputDims();
|
||||
if (outputDims.size() < 2 || featureVectors.size() < 2) return {};
|
||||
|
||||
int boxesIdx = -1, confsIdx = -1;
|
||||
for (size_t t = 0; t < outputDims.size(); ++t) {
|
||||
const auto& d = outputDims[t];
|
||||
if (d.nbDims > 0 && d.d[d.nbDims - 1] == 4 && boxesIdx < 0) boxesIdx = static_cast<int>(t);
|
||||
else if (confsIdx < 0) confsIdx = static_cast<int>(t);
|
||||
}
|
||||
if (boxesIdx < 0 || confsIdx < 0) return {};
|
||||
if (outputDims[confsIdx].nbDims != 3) return {};
|
||||
|
||||
const int numClasses = outputDims[confsIdx].d[2];
|
||||
if (numClasses <= 0) return {};
|
||||
const auto& confsVec = featureVectors[confsIdx];
|
||||
const auto& boxesVec = featureVectors[boxesIdx];
|
||||
// Number of candidates from the data (dim 1 may be dynamic)
|
||||
const int numBoxes = static_cast<int>(confsVec.size() / numClasses);
|
||||
if (numBoxes <= 0 || boxesVec.size() < static_cast<size_t>(numBoxes) * 4) return {};
|
||||
|
||||
std::vector<cv::Rect> bboxes;
|
||||
std::vector<float> scores;
|
||||
std::vector<int> labels;
|
||||
std::vector<int> indices;
|
||||
|
||||
for (int i = 0; i < numBoxes; i++) {
|
||||
const float* scoresPtr = confsVec.data() + static_cast<size_t>(i) * numClasses;
|
||||
const float* maxSPtr = std::max_element(scoresPtr, scoresPtr + numClasses);
|
||||
float score = *maxSPtr;
|
||||
if (score > _modelConfig.detectionScoreThreshold) {
|
||||
const float* b = boxesVec.data() + static_cast<size_t>(i) * 4;
|
||||
float x0 = std::clamp(b[0] * meta.imgWidth, 0.f, meta.imgWidth);
|
||||
float y0 = std::clamp(b[1] * meta.imgHeight, 0.f, meta.imgHeight);
|
||||
float x1 = std::clamp(b[2] * meta.imgWidth, 0.f, meta.imgWidth);
|
||||
float y1 = std::clamp(b[3] * meta.imgHeight, 0.f, meta.imgHeight);
|
||||
if (x1 - x0 < 1.f || y1 - y0 < 1.f) continue;
|
||||
cv::Rect_<float> bbox;
|
||||
bbox.x = x0; bbox.y = y0;
|
||||
bbox.width = x1 - x0; bbox.height = y1 - y0;
|
||||
bboxes.push_back(bbox);
|
||||
labels.push_back(static_cast<int>(maxSPtr - scoresPtr));
|
||||
scores.push_back(score);
|
||||
}
|
||||
}
|
||||
|
||||
cv::dnn::NMSBoxesBatched(bboxes, scores, labels, PROBABILITY_THRESHOLD, NMS_THRESHOLD, indices);
|
||||
std::vector<Object> objects;
|
||||
int classNameSize = static_cast<int>(_classes.size());
|
||||
for (auto& chosenIdx : indices) {
|
||||
if (static_cast<int>(objects.size()) >= TOP_K) break;
|
||||
if (scores[chosenIdx] > _modelConfig.detectionScoreThreshold) {
|
||||
Object obj{};
|
||||
obj.confidence = scores[chosenIdx];
|
||||
obj.classId = labels[chosenIdx];
|
||||
obj.box = bboxes[chosenIdx];
|
||||
if (!_classes.empty()) {
|
||||
obj.className = (obj.classId < classNameSize)
|
||||
? _classes[obj.classId] : _classes[classNameSize - 1];
|
||||
}
|
||||
else { obj.className = "Unknown"; }
|
||||
obj.cameraId = camera_id;
|
||||
objects.push_back(obj);
|
||||
}
|
||||
}
|
||||
return objects;
|
||||
}
|
||||
catch (std::exception& e) {
|
||||
_logger.LogFatal("ANSRTYOLO::PostprocessDarknet", e.what(), __FILE__, __LINE__);
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
// ====================================================================
|
||||
// DetectObjects — single image with auto-detection of task type
|
||||
// ====================================================================
|
||||
@@ -1856,8 +1976,11 @@ namespace ANSCENTER {
|
||||
const int inputW = inputDims[0].d[2];
|
||||
const int inputH = inputDims[0].d[1];
|
||||
|
||||
// Darknet stretches rather than letterboxes: the direct
|
||||
// (no-padding) resize kernel is the classification one.
|
||||
auto nv12 = m_nv12Helper.tryNV12(inputImage, inferenceGpu, inputW, inputH,
|
||||
NV12PreprocessHelper::defaultYOLOLauncher(),
|
||||
_isDarknet ? NV12PreprocessHelper::classificationLauncher()
|
||||
: NV12PreprocessHelper::defaultYOLOLauncher(),
|
||||
_logger, "ANSRTYOLO");
|
||||
if (nv12.succeeded) {
|
||||
meta.imgWidth = nv12.metaWidth;
|
||||
@@ -1912,7 +2035,14 @@ namespace ANSCENTER {
|
||||
const auto& outputDims = m_trtEngine->getOutputDims();
|
||||
const size_t numOutputs = outputDims.size();
|
||||
|
||||
if (numOutputs >= 2) {
|
||||
// Darknet (confs + boxes) — checked before seg, whose test
|
||||
// (2 outputs) it would otherwise pass
|
||||
if (_isDarknet) {
|
||||
std::vector<std::vector<float>> featureVector2d;
|
||||
Engine<float>::transformOutput(featureVectors, featureVector2d);
|
||||
results = PostprocessDarknet(featureVector2d, camera_id, meta);
|
||||
}
|
||||
else if (numOutputs >= 2) {
|
||||
std::vector<std::vector<float>> featureVector2d;
|
||||
Engine<float>::transformOutput(featureVectors, featureVector2d);
|
||||
double msTransform = dbg ? elapsed() : 0;
|
||||
@@ -2212,13 +2342,16 @@ namespace ANSCENTER {
|
||||
int nc = static_cast<int>(_classes.size());
|
||||
|
||||
enum class TaskType { DetLegacy, DetE2E, OBBLegacy, OBBE2E,
|
||||
SegLegacy, SegE2E, PoseLegacy, PoseE2E, Classify };
|
||||
SegLegacy, SegE2E, PoseLegacy, PoseE2E, Classify, Darknet };
|
||||
TaskType taskType = TaskType::DetLegacy; // default
|
||||
|
||||
// E2E: dim1 > dim2 (e.g. [B,300,6]); Legacy: dim1 < dim2 (e.g. [B,84,8400])
|
||||
const bool isEndToEnd = (dim1 > dim2) || (dim2 <= 20);
|
||||
|
||||
if (numOutputs >= 2) {
|
||||
if (_isDarknet) {
|
||||
taskType = TaskType::Darknet;
|
||||
}
|
||||
else if (numOutputs >= 2) {
|
||||
taskType = isEndToEnd ? TaskType::SegE2E : TaskType::SegLegacy;
|
||||
}
|
||||
else if (outputDims[0].nbDims <= 2) {
|
||||
@@ -2299,6 +2432,14 @@ namespace ANSCENTER {
|
||||
}
|
||||
break;
|
||||
}
|
||||
case TaskType::Darknet: {
|
||||
std::vector<std::vector<float>> fv2d(batchOutput.begin(), batchOutput.end());
|
||||
postFutures.push_back(std::async(std::launch::async,
|
||||
[this, fv = std::move(fv2d), cid = camera_id, m = imgMeta]() mutable {
|
||||
return PostprocessDarknet(fv, cid, m);
|
||||
}));
|
||||
break;
|
||||
}
|
||||
case TaskType::Classify: {
|
||||
std::vector<float> fv = batchOutput.empty() ? std::vector<float>{} : batchOutput[0];
|
||||
postFutures.push_back(std::async(std::launch::async,
|
||||
|
||||
Reference in New Issue
Block a user