diff --git a/core/ANSModelPackage/ANSModelOpen.h b/core/ANSModelPackage/ANSModelOpen.h index 3f8f853..633d30c 100644 --- a/core/ANSModelPackage/ANSModelOpen.h +++ b/core/ANSModelPackage/ANSModelOpen.h @@ -217,16 +217,28 @@ inline bool OpenModelPackage(const std::string& path, OpenedModel& out, out.name = m.value("project", std::string()); if (out.name.empty()) out.name = path; - // A package written before manifests carried an input size still has to - // answer, and the model in hand is the thing that knows. It costs nothing - // here: the bytes are already decrypted. - if (out.inputWidth <= 0 || out.inputHeight <= 0) { - onnxshape::InputShape shape; + // WHAT AN OLDER PACKAGE CANNOT TELL US, and its model can. The input + // size, when the manifest predates the field; and the head, because a + // package whose family is "yolo" may hold an end-to-end export — the box + // used to take the family from the backend marker, and the marker names + // the runtime rather than the head. It costs nothing to ask here: the + // bytes are already decrypted. + { + onnxshape::GraphShape g; std::string swhy; - if (onnxshape::fromMemory(out.model.data(), out.model.size(), shape, - swhy)) { - out.inputWidth = shape.width; - out.inputHeight = shape.height; + if (onnxshape::graphFromMemory(out.model.data(), out.model.size(), g, + swhy)) { + if (out.inputWidth <= 0 || out.inputHeight <= 0) { + out.inputWidth = g.input.width; + out.inputHeight = g.input.height; + } + if (g.endToEnd() && out.family == "yolo") { + out.family = "yolo10"; + out.warning = out.warning.empty() + ? "this package says family yolo, but its head is " + "end-to-end; reading it as yolo10" + : out.warning; + } } } return true; diff --git a/core/ANSModelPackage/ANSOnnxShape.h b/core/ANSModelPackage/ANSOnnxShape.h index e7d872a..1af0866 100644 --- a/core/ANSModelPackage/ANSOnnxShape.h +++ b/core/ANSModelPackage/ANSOnnxShape.h @@ -3,18 +3,16 @@ #pragma once // ANSOnnxShape — the input geometry a model declares, read off the graph. // -// WHY A READER NEEDS THIS. An engine has to know what its model expects before -// it can preprocess a single frame, and that answer has always come from -// whatever the bundle happened to carry: model_config.json's input_size, or an -// `input` object in a package manifest. Neither is present on every model — -// most of the bundles in the field are classes.names, a marker file and the -// export, and nothing else. The graph is always there, and the graph is what -// the session is built from, so where the metadata is silent the model itself -// is asked instead of every engine falling back to 640x640. +// WHY A READER NEEDS THIS. A node has to know what its model expects before it +// can preprocess a single frame, and until now that answer came from whatever +// the bundle happened to carry: model_config.json's input_size in a folder, an +// `input` object in a package manifest. Neither is present on every model. The +// graph always is, and the graph is the thing the session is built from — so +// where the metadata is silent, ask the model itself rather than assume 640. // // TWO SOURCES, ONE PARSER. A package's model is already in memory, decrypted, -// and must never reach the disk; an inflated bundle's is a file that may be -// hundreds of megabytes. The template below reads either, and the file case +// and must never reach the disk; an imported folder's model is a file that may +// be hundreds of megabytes. The template below reads either, and the file case // SEEKS past the nodes and the initializers rather than reading them — the // fields this wants sit after all of the weight on the wire. // @@ -23,12 +21,16 @@ // ModelProto.graph field 7, length-delimited // GraphProto.input field 11, repeated (after node and // initializer on the wire) +// GraphProto.output field 12, repeated // ValueInfoProto.type field 2 // TypeProto.tensor_type field 1 // TypeProto.Tensor.shape field 2 // TensorShapeProto.dim field 1, repeated // Dimension.dim_value field 1, varint (dim_param means the // axis is dynamic: reported as 0) +// ModelProto.metadata_props field 14, repeated StringStringEntryProto +// StringStringEntryProto.key field 1, string +// StringStringEntryProto.value field 2, string // // An axis the graph leaves open is reported as zero, and a file that cannot be // parsed is reported as a failure. Both mean the same thing to a caller: this @@ -36,6 +38,7 @@ // model's input size is worse than being silent about it. #include +#include #include #include #include @@ -52,6 +55,32 @@ struct InputShape { bool known() const { return height > 0 && width > 0; } }; +// What the whole graph says about itself: what goes in, and what shape comes +// out — which is the difference between a head that needs decoding and one +// that has already done it. +struct GraphShape { + InputShape input; + std::vector output; // the FIRST output's dims; empty when unknown + bool exportSaysEndToEnd = false; // the model's own metadata, when it says + + // AN END-TO-END HEAD, [1, N, 6]: rows of (x1 y1 x2 y2 score class) with the + // NMS already inside the graph. Telling one apart from an ordinary YOLO head + // matters more than it looks: feeding [1,N,6] to a v8 decoder yields boxes + // everywhere with coordinates read as confidences — plausible-looking + // garbage rather than an error. + // + // The trap is a two-class v8 export, whose [1, 4+nc, N] is [1, 6, 8400] — + // a 6 in the shape, and not end-to-end. So the 6 must be the LAST axis, and + // the axis before it must be a detection count rather than an anchor count; + // an end-to-end head emits max_det rows, which is hundreds, never thousands. + // When the export's own metadata says end2end, that settles it outright. + bool endToEnd() const { + if (exportSaysEndToEnd) return true; + return output.size() == 3 && output[0] == 1 && output[2] == 6 && + output[1] > 0 && output[1] <= 1000; + } +}; + namespace detail { // A model already in memory: the decrypted contents of a package. @@ -158,51 +187,105 @@ std::vector dims(Src& s, int64_t end) { return out; } +// One ValueInfoProto (a graph input or output), given a stream on its payload: +// its dims, or empty when it is not a shaped tensor. template -bool read(Src& s, int64_t end, InputShape& out, std::string& why) { +std::vector valueDims(Src& s, int64_t valueEnd) { + uint64_t len = 0; + if (!seekField(s, valueEnd, 2, len)) return {}; // ValueInfoProto.type + const int64_t typeEnd = s.tell() + int64_t(len); + if (!seekField(s, typeEnd, 1, len)) return {}; // TypeProto.tensor_type + const int64_t tensorEnd = s.tell() + int64_t(len); + if (!seekField(s, tensorEnd, 2, len)) return {}; // Tensor.shape + return dims(s, s.tell() + int64_t(len)); +} + +// The graph's first input and first output, and whether the export declared +// itself end-to-end. `end` is the whole file; `graphStart` lets the output be +// found after the input, since both hang off the same graph. +template +bool readGraph(Src& s, int64_t end, GraphShape& out, std::string& why) { uint64_t len = 0; if (!seekField(s, end, 7, len)) { // ModelProto.graph why = "this does not look like an ONNX model"; return false; } - const int64_t graphEnd = s.tell() + int64_t(len); + const int64_t graphStart = s.tell(); + const int64_t graphEnd = graphStart + int64_t(len); + if (!seekField(s, graphEnd, 11, len)) { // GraphProto.input why = "the graph declares no input"; return false; } - const int64_t inputEnd = s.tell() + int64_t(len); - if (!seekField(s, inputEnd, 2, len)) { // ValueInfoProto.type - why = "the input declares no type"; - return false; + const std::vector in = valueDims(s, s.tell() + int64_t(len)); + + // The output, from the graph's start again: it sits after the input on the + // wire, but seeking from the start costs nothing and does not depend on it. + s.seek(graphStart); + if (seekField(s, graphEnd, 12, len)) // GraphProto.output + out.output = valueDims(s, s.tell() + int64_t(len)); + + // The export's own word on its head, which beats any shape rule. + s.seek(0); + while (s.ok() && s.tell() >= 0 && s.tell() < end) { + uint32_t field = 0, wire = 0; + if (!tag(s, field, wire)) break; + if (field != 14 || wire != 2) { // ModelProto.metadata_props + if (!skip(s, wire)) break; + continue; + } + uint64_t entryLen = 0; + if (!varint(s, entryLen)) break; + const int64_t entryEnd = s.tell() + int64_t(entryLen); + std::string key, value; + while (s.ok() && s.tell() >= 0 && s.tell() < entryEnd) { + uint32_t ef = 0, ew = 0; + if (!tag(s, ef, ew)) break; + if ((ef == 1 || ef == 2) && ew == 2) { + uint64_t n = 0; + if (!varint(s, n)) break; + std::string text; + for (uint64_t i = 0; i < n; ++i) { + const int c = s.get(); + if (c < 0) break; + text.push_back(char(c)); + } + (ef == 1 ? key : value) = text; + } else if (!skip(s, ew)) { + break; + } + } + s.seek(entryEnd); + if (key == "end2end") { + for (char& c : value) c = char(::tolower(static_cast(c))); + out.exportSaysEndToEnd = (value == "true" || value == "1"); + } } - const int64_t typeEnd = s.tell() + int64_t(len); - if (!seekField(s, typeEnd, 1, len)) { // TypeProto.tensor_type - why = "the input is not a tensor"; - return false; - } - const int64_t tensorEnd = s.tell() + int64_t(len); - if (!seekField(s, tensorEnd, 2, len)) { // Tensor.shape - why = "the input tensor has no shape"; - return false; - } - const std::vector d = dims(s, s.tell() + int64_t(len)); // NCHW is the only layout these engines run. - if (d.size() != 4) { + if (in.size() != 4) { why = "the input is not a four-dimensional image tensor"; return false; } auto axis = [](int64_t v) { return (v > 0 && v < 1000000) ? int(v) : 0; }; - out.channels = axis(d[1]); - out.height = axis(d[2]); - out.width = axis(d[3]); - if (!out.known()) { + out.input.channels = axis(in[1]); + out.input.height = axis(in[2]); + out.input.width = axis(in[3]); + if (!out.input.known()) { why = "the model takes a dynamic input size"; return false; } return true; } +template +bool read(Src& s, int64_t end, InputShape& out, std::string& why) { + GraphShape g; + if (!readGraph(s, end, g, why)) return false; + out = g.input; + return true; +} + } // namespace detail // What a model already in memory declares. @@ -216,6 +299,17 @@ inline bool fromMemory(const void* data, size_t size, InputShape& out, return detail::read(s, int64_t(size), out, why); } +// The whole graph, from a model already in memory. +inline bool graphFromMemory(const void* data, size_t size, GraphShape& out, + std::string& why) { + if (!data || size == 0) { + why = "there is no model to read"; + return false; + } + detail::MemSource s{static_cast(data), int64_t(size), 0}; + return detail::readGraph(s, int64_t(size), out, why); +} + // What a model on disk declares. The file is walked, not loaded. inline bool fromFile(const std::string& path, InputShape& out, std::string& why) { @@ -235,6 +329,25 @@ inline bool fromFile(const std::string& path, InputShape& out, return detail::read(s, int64_t(size), out, why); } +// The whole graph, from a model on disk. Walked, not loaded. +inline bool graphFromFile(const std::string& path, GraphShape& out, + std::string& why) { + std::error_code ec; + const auto size = std::filesystem::file_size(path, ec); + if (ec || size == 0) { + why = "the model is not there to read"; + return false; + } + detail::FileSource s; + s.f.open(path, std::ios::binary); + s.size = int64_t(size); + if (!s.f) { + why = "the model could not be opened"; + return false; + } + return detail::readGraph(s, int64_t(size), out, why); +} + } // namespace onnxshape } // namespace ANSCENTER diff --git a/tests/ANSModelPackage-UnitTest/ANSModelPackage-UnitTest.cpp b/tests/ANSModelPackage-UnitTest/ANSModelPackage-UnitTest.cpp index ac76b2e..225dfb4 100644 --- a/tests/ANSModelPackage-UnitTest/ANSModelPackage-UnitTest.cpp +++ b/tests/ANSModelPackage-UnitTest/ANSModelPackage-UnitTest.cpp @@ -328,12 +328,15 @@ static std::string pbDim(int64_t v) { // Dimension.dim_value static std::string pbDynamicDim(const std::string& name) { return pbField(1, pbField(2, name)); // dim_param: the axis is open } -static std::string fakeOnnx(const std::string& dims) { - const std::string tensor = pbField(1, pbField(2, dims)); - const std::string input = pbField(1, "images") + pbField(2, tensor); - const std::string graph = pbField(1, std::string(4096, 'n')) + - pbField(5, std::string(65536, 'w')) + - pbField(11, input); +static std::string fakeOnnx(const std::string& dims, + const std::string& outDims = std::string()) { + auto value = [](const std::string& name, const std::string& d) { + return pbField(1, name) + pbField(2, pbField(1, pbField(2, d))); + }; + std::string graph = pbField(1, std::string(4096, 'n')) + + pbField(5, std::string(65536, 'w')) + + pbField(11, value("images", dims)); + if (!outDims.empty()) graph += pbField(12, value("output0", outDims)); return pbField(7, graph); } @@ -405,6 +408,57 @@ static void testShapeFromTheGraph(const fs::path& dir) { CHECK(modelpkg::OpenModelPackage(path, opened, err), ("open: " + err).c_str()); CHECK(opened.inputWidth == 224 && opened.inputHeight == 224, "so the engine is told 224 from the graph, not left to assume 640"); + + // AN END-TO-END HEAD IN A PACKAGE THAT SAYS "yolo". Boxes used to take the + // family from the backend marker, and the marker names the runtime, not the + // head — so a yolo26 export with end2end on arrived declared as plain yolo. + // Decoding [1,N,6] as a v8 grid reads coordinates as confidences: boxes + // everywhere, and no error to notice. The graph settles it. + { + const std::string graph = + fakeOnnx(pbDim(1) + pbDim(3) + pbDim(640) + pbDim(640), + pbDim(1) + pbDim(300) + pbDim(6)); + modelpkg::PackRequest r = req; + r.manifest = Json{{"task", "detection"}, + {"family", "yolo"}, + {"classes", Json::array({Json{{"id", 1}, {"name", "Tooth"}}})}, + {"models", Json::array({Json{{"id", "m0"}, + {"payload", "p0"}, + {"entry", true}}})}}; + r.models.clear(); + r.models.push_back({"p0", std::vector(graph.begin(), graph.end())}); + const std::string p = (dir / "e2e.ansmodel").string(); + CHECK(modelpkg::pack(p, r, err), ("pack e2e: " + err).c_str()); + modelpkg::OpenedModel m; + CHECK(modelpkg::OpenModelPackage(p, m, err) && m.family == "yolo10", + "an end-to-end head is read as yolo10 whatever the package says"); + CHECK(m.warning.find("end-to-end") != std::string::npos, + "and the reason is said out loud rather than silently swapped"); + CHECK(m.legacyMarker.empty() || m.legacyMarker == "tensorrt.txt", + "while the legacy marker is left as it was"); + } + + // A two-class v8 export is [1, 6, 8400]: a 6 in the shape, not a head that + // has already decoded itself. + { + const std::string graph = + fakeOnnx(pbDim(1) + pbDim(3) + pbDim(640) + pbDim(640), + pbDim(1) + pbDim(6) + pbDim(8400)); + modelpkg::PackRequest r = req; + r.manifest = Json{{"task", "detection"}, + {"family", "yolo"}, + {"classes", Json::array({Json{{"id", 1}, {"name", "Tooth"}}})}, + {"models", Json::array({Json{{"id", "m0"}, + {"payload", "p0"}, + {"entry", true}}})}}; + r.models.clear(); + r.models.push_back({"p0", std::vector(graph.begin(), graph.end())}); + const std::string p = (dir / "twoclass.ansmodel").string(); + CHECK(modelpkg::pack(p, r, err), ("pack v8: " + err).c_str()); + modelpkg::OpenedModel m; + CHECK(modelpkg::OpenModelPackage(p, m, err) && m.family == "yolo", + "a 6 on the wrong axis with thousands of anchors stays plain yolo"); + } } int main(int argc, char** argv) {