diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 60dd80a..dc0b182 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -179,7 +179,15 @@ "Bash(cmake --build . --target ANSODEngine -j 8)", "PowerShell(@\\(Get-ChildItem -Path \"C:\\\\Program Files\\\\Microsoft Visual Studio\" -Filter \"vcvars64.bat\" -Recurse -ErrorAction SilentlyContinue\\) + @\\(Get-ChildItem -Path \"C:\\\\Program Files \\(x86\\)\\\\Microsoft Visual Studio\" -Filter \"vcvars64.bat\" -Recurse -ErrorAction SilentlyContinue\\) | ForEach-Object { $_.FullName })", "PowerShell($d = Get-Item C:\\\\Projects\\\\CLionProjects\\\\ANSCORE\\\\cmake-build-release\\\\bin\\\\ANSODEngine.dll; \"$\\($d.Name\\) | $\\($d.Length\\) bytes | $\\($d.LastWriteTime\\)\")", - "Bash(awk '{if \\($0 ~ /=0+$/\\) print \"ZERO\"; else print \"NONZERO\"}')" + "Bash(awk '{if \\($0 ~ /=0+$/\\) print \"ZERO\"; else print \"NONZERO\"}')", + "Bash(awk '{print $4}' \"C:\\\\Users\\\\nghia\\\\Downloads\\\\ANSAIBOX-Q1A01S3_memoryleak1.log\")", + "Bash(awk -F'\\\\t' 'NF>=4 && $3 ~ /^[0-9]+$/ {print $3}' \"C:\\\\Users\\\\nghia\\\\Downloads\\\\ANSAIBOX-Q1A01S3_memoryleak1.log\")", + "PowerShell($dll = \"C:\\\\Projects\\\\CLionProjects\\\\ANSCORE\\\\bin\\\\anslicensing.dll\"; if \\(Test-Path $dll\\) { Get-Process | Where-Object { try { $_.Modules | Where-Object { $_.FileName -eq $dll } } catch {} } | Select-Object Id, ProcessName } else { \"DLL not present at $dll\" })", + "PowerShell($p = \"C:\\\\Projects\\\\CLionProjects\\\\ANSCORE\\\\cmake-build-release\\\\bin\\\\anslicensing.dll\"; try { $fs = [System.IO.File]::Open\\($p, 'Open', 'ReadWrite', 'None'\\); $fs.Close\\(\\); \"OK: writable, no other handle\" } catch { \"LOCKED: $\\($_.Exception.Message\\)\" })", + "PowerShell($candidates = @\\(\"handle64.exe\",\"handle.exe\"\\); $found = $null; foreach \\($c in $candidates\\) { $p = Get-Command $c -ErrorAction SilentlyContinue; if \\($p\\) { $found = $p.Source; break } }; if \\($found\\) { Write-Output \"FOUND: $found\"; & $found -nobanner -accepteula \"anslicensing.dll\" 2>$null } else { Write-Output \"handle.exe not in PATH\" })", + "PowerShell(Get-CimInstance Win32_Process -Filter \"ProcessId=22688\")", + "PowerShell(Get-CimInstance Win32_Process -Filter \"ParentProcessId=22688\")", + "Bash(taskkill //F //PID 50352)" ] } } diff --git a/AMD_DML_FP16_Converter_Plan.md b/AMD_DML_FP16_Converter_Plan.md new file mode 100644 index 0000000..9990c38 --- /dev/null +++ b/AMD_DML_FP16_Converter_Plan.md @@ -0,0 +1,452 @@ +# AMD DirectML FP16 Converter — Implementation Plan + +**Status:** Not started. Pick this up in a future session. +**Estimated effort:** ~1 week of focused work. +**Owner of context:** This document is self-contained — you don't need the +prior conversation to execute it. + +--- + +## 1. Goal + +Recover ~30–50% of per-call inference latency on AMD Radeon iGPUs (DML EP) +by running models in FP16 instead of FP32. Pure C++ implementation — +no Python tooling at any stage. Mirror the existing TensorRT pattern +where `OrtHandlerOptions::trtFP16 = true` causes the engine layer to +build/cache an FP16 plan automatically at session-create time. + +The current AMD performance baseline (after the always-serial-on-AMD fix) +is ~5.4 fps for full-frame ALPR. Target after FP16: 7–8 fps. + +## 2. Why C++ (and not the standard Python toolchain) + +The standard tool for this is `onnxconverter-common.float16.convert_float_to_float16`. +Project rule disallows Python anywhere in the workflow, so we must +re-implement the algorithm in C++ inside ANSCORE. + +The DirectML EP itself has no `dml_fp16_enable` runtime knob (unlike TRT's +`trt_fp16_enable`). DML runs whatever precision the ONNX model declares. +So the C++ work has to produce a transformed FP16 ONNX, then hand that +to the DML EP. + +## 3. Architecture (mirrors the TRT pattern) + +``` +TRT path (existing): + trtFP16 = true → ORT TRT EP options → TRT engine builder rewrites + → engine cached on disk → inference + +DML path (new): + dmlFP16 = true → ANSCORE C++ rewrite → cached as .fp16.onnx + → DML EP loads cached FP16 model → inference +``` + +Cache invalidation, per-model granularity, and the "first run is slow" +semantics all match TRT. From the call-site perspective the TRT and DML +options are symmetric. + +## 4. Current state of the codebase (relevant files) + +These files already exist and are the integration points. Don't recreate +them; modify them. + +| File | Relevant content | What this plan adds | +|---|---|---| +| `engines/ONNXEngine/ONNXEngine.h` | `OrtHandlerOptions` struct. AMD section commented "OPEN FOR OPTIMIZATION" — empty today | Add `dmlFP16`, `dmlOptimizedModelDir`, `dmlFP16OpBlockList` fields | +| `engines/ONNXEngine/ONNXEngine.cpp` | `BasicOrtHandler::initialize_handler()` — DML EP branch (search for `EngineType::AMD_GPU`) | Insert path-resolution call before session creation | +| `engines/ONNXEngine/ONNXEngine.cpp` | Already houses `DmlSerialState` + `DmlInferenceGuard` (the always-serial-on-AMD fix from previous work) | Reuse the same TU for FP16 helpers | +| `engines/ONNXEngine/Fp16GraphRewriter.{h,cpp}` | **Does not exist yet** | Create — main implementation of this plan | +| `modules/ANSOCR/ANSONNXOCR/PaddleOCRV5Engine.cpp` | `BuildXxxOcrOptions()` helpers populating per-model `OrtHandlerOptions` | Add `o.dmlFP16 = true/false` per model | +| Caller code in `ANSONNXYOLO`, `ONNXOCRDetector`, `ONNXOCRRecognizer` | Already constructs sessions via `BasicOrtHandler` | No changes — the FP16 swap is invisible to callers | + +## 5. Step-by-step plan + +### Step 1 — Add AMD knobs to `OrtHandlerOptions` (half day) + +In `engines/ONNXEngine/ONNXEngine.h`, replace the AMD placeholder: + +```cpp +// ---------------------------------------------------------------- +// AMD (DirectML EP) — FP16 graph rewrite +// ---------------------------------------------------------------- + +// Convert FP32 weights to FP16 before session creation. Mirrors +// trtFP16 semantically: an in-process graph rewrite produces an FP16 +// ONNX, cached on disk under dmlOptimizedModelDir, and the DML EP +// loads the cached FP16 model. External inputs/outputs stay FP32 via +// Cast nodes inserted at the boundaries — caller code is unchanged. +// Ignored on every non-DML EP. Default off until per-model accuracy +// validation completes. +bool dmlFP16 = false; + +// Where to cache rewritten FP16 ONNX models. Empty → default +// %TEMP%/ANSCENTER/DMLOptimizedCache. Cache key is a hash of +// (source ONNX bytes + dmlFP16 flag + opBlockList). Stale entries +// are silently replaced when source changes. +std::string dmlOptimizedModelDir; + +// Op types whose outputs MUST remain FP32 (NMS, TopK, RoiAlign, plus +// any op observed to produce NaN/Inf in FP16 for THIS model). Empty +// = use the built-in default block list (see kDefaultFp16BlockList in +// Fp16GraphRewriter.cpp). Per-model override. +std::vector dmlFP16OpBlockList; +``` + +### Step 2 — Add ONNX protobuf to the build (half day) + +You need the `onnx.proto3` schema compiled to C++ via `protoc`. Two +sub-options: + +**Option 2A (recommended): vendor the proto file.** +1. Download `onnx.proto3` from https://github.com/onnx/onnx (current + release tag). Put it at `engines/ONNXEngine/proto/onnx.proto3`. +2. Add a CMake `add_custom_command` calling `protoc --cpp_out=...` at + configure time, generating `onnx.pb.{h,cc}` into the build directory. +3. Link `Fp16GraphRewriter` against `protobuf::libprotobuf`. + +`libprotobuf` is almost certainly already a transitive dep of +`onnxruntime` — confirm by checking +`C:\ANSLibs\onnxruntime\include` for `google/protobuf/...` headers and +`C:\ANSLibs\protobuf` (or vcpkg's protobuf install). Just expose the +include directories to your new target. + +**Option 2B: vendor the full ONNX C++ library.** +Bigger dep but provides typed helpers, validators, and a checker. Skip +unless 2A's hand-rolled approach blows up on edge cases. + +### Step 3 — Implement `Fp16GraphRewriter` (~2 days for core) + +Create `engines/ONNXEngine/Fp16GraphRewriter.h`: + +```cpp +#pragma once +#include +#include +#include + +namespace ANSCENTER { + +// Returns the path to an FP16 version of fp32Path. If cacheDir already +// contains a fresh FP16 cache for this model, returns the cached path +// directly (fast path). Otherwise loads fp32Path, runs the in-memory +// FP16 graph rewrite, writes to cacheDir, and returns the new path. +// +// Cache filename embeds a hash of (fp32 file content + opBlockList) so +// a model update or blocklist change automatically invalidates the +// cache — same pattern as TRT engine cache invalidation. +// +// On any failure (parse error, write error, etc.) returns fp32Path +// unchanged and logs a warning via ANS_DBG. The session creation +// continues with the FP32 model — this function NEVER blocks the +// inference path. +std::string ResolveFp16OnnxPath( + const std::string& fp32Path, + const std::string& cacheDir, + const std::vector& opBlockList); + +} // namespace ANSCENTER +``` + +Create `engines/ONNXEngine/Fp16GraphRewriter.cpp`. Key components: + +#### 3a. IEEE-754 binary32 → binary16 helper + +Standard 30-line implementation, round-to-nearest-even, handle NaN/Inf. +Reference: numpy's `numpy/core/src/npymath/halffloat.c` or +`Eigen/src/Core/arch/Default/Half.h`. Self-contained, no deps. + +#### 3b. Default op blocklist + +```cpp +static const std::unordered_set kDefaultFp16BlockList = { + // Object-detection post-processing — known FP16-unsafe + "NonMaxSuppression", "TopK", "RoiAlign", + // Classical ML ops — never have FP16 implementations + "ArrayFeatureExtractor", "Binarizer", "CastMap", + "CategoryMapper", "DictVectorizer", "FeatureVectorizer", + "Imputer", "LabelEncoder", "LinearClassifier", + "LinearRegressor", "Normalizer", "OneHotEncoder", + "RandomUniform", "SVMClassifier", "SVMRegressor", + "Scaler", "TreeEnsembleClassifier", "TreeEnsembleRegressor", + "ZipMap", +}; +``` + +#### 3c. Initializer conversion + +Walk `model.graph().initializer()`. For every `TensorProto` with +`data_type == FLOAT`: + +1. Pull the float data (it lives in `raw_data` OR `float_data` — + handle both). +2. Convert each float32 to uint16 binary16. +3. Set `data_type = FLOAT16`, clear `float_data`, write the new bytes + to `raw_data`. + +Watch for: external data references (when `data_location == +EXTERNAL`) — for the YOLO and OCR models in this project, weights are +embedded, but write a guard: if external, skip with a warning. + +#### 3d. Cast-node insertion at boundaries + +For each external input (entries in `model.graph().input()` that are +NOT initializer names): + +1. Note the input's tensor name `orig`. +2. Append a new `NodeProto` to the graph: `Cast(input=orig, + output=orig + "_fp16", to=FLOAT16)`. +3. Walk every existing node in the graph; for any input that equals + `orig`, replace it with `orig + "_fp16"`. Be careful not to rewire + the new Cast node itself (skip the last node added). + +For each external output: insert `Cast(FP16 → FP32)` symmetrically. + +#### 3e. value_info type updates + +For every `ValueInfoProto` in `graph().value_info()`: + +- If the tensor is the input to OR output of a block-listed op, leave + its `tensor_type.elem_type` as FP32. +- Otherwise update to FP16. + +This step is the most error-prone in practice. When in doubt, leave +elem_type alone — the runtime will accept inferred types. Test +correctness empirically with the validation harness in Step 7. + +#### 3f. Subgraphs + +If/Loop/Scan ops have nested subgraphs (`AttributeProto` of type +GRAPH). Recurse into each. The OCR recognizer doesn't have these; +neither does YOLO post-Ultralytics-export — but write the recursion +defensively, otherwise this fails silently on future models. + +#### 3g. Cache management + +Cache filename: `..fp16.onnx`, where `sha1_8` is +the first 8 hex chars of `SHA1(fp32_file_bytes || serialized +opBlockList || "v1")`. The "v1" string lets you bust the cache for +all models if you fix a converter bug later. + +Cache directory: create on demand with `std::filesystem::create_directories`. +Default: `std::filesystem::temp_directory_path() / "ANSCENTER" / +"DMLOptimizedCache"`. + +Lookup: if cache file exists AND its mtime is newer than the source +ONNX, use it. Else regenerate. + +### Step 4 — Wire into `initialize_handler` (half day) + +In `engines/ONNXEngine/ONNXEngine.cpp`, find the DML branch in +`BasicOrtHandler::initialize_handler()`. Insert path resolution before +session creation: + +```cpp +case EngineType::AMD_GPU: { + std::string onnxToLoad = _onnx_path; + if (options.dmlFP16) { + const std::string cacheDir = + options.dmlOptimizedModelDir.empty() + ? GetDefaultDmlCacheDir() + : options.dmlOptimizedModelDir; + onnxToLoad = ResolveFp16OnnxPath( + _onnx_path, cacheDir, options.dmlFP16OpBlockList); + ANS_DBG("OrtHandler", + "AMD FP16: source=%s cached=%s", + _onnx_path.c_str(), onnxToLoad.c_str()); + } + if (!TryAppendDirectML(opts)) { /* existing fallback */ } + ort_session = std::make_unique( + env, onnxToLoad.c_str(), opts); + break; +} +``` + +`GetDefaultDmlCacheDir()` is a small helper in `Fp16GraphRewriter.cpp`. + +### Step 5 — Per-model configuration (1 day) + +Update the option-builder functions to opt models into FP16: + +`modules/ANSOCR/ANSONNXOCR/PaddleOCRV5Engine.cpp` (and equivalents): + +```cpp +OrtHandlerOptions BuildLpDetectorOptions() { + OrtHandlerOptions o; + o.preferTensorRT = true; + o.trtFP16 = true; + o.dmlFP16 = true; // YOLO det validated FP16-safe + return o; +} + +OrtHandlerOptions BuildOcrDetectorOptions() { + OrtHandlerOptions o; + o.preferTensorRT = true; + o.trtFP16 = true; + o.dmlFP16 = true; // DBNet text-det validated + return o; +} + +OrtHandlerOptions BuildOcrRecognizerOptions() { + OrtHandlerOptions o; + o.preferTensorRT = true; + o.trtFP16 = true; + // OCR recognizer is FP16-sensitive on small text — keep AMD FP32 + // until per-plate accuracy validation passes (see Step 7). + o.dmlFP16 = false; + return o; +} +``` + +Helmet OD (in `ANSCUSTOMDetector` model loading) — set `dmlFP16 = true`. + +### Step 6 — DebugView logging (per `MEMORY.md` rule) + +Sprinkle `ANS_DBG` markers at every decision/error point in +`Fp16GraphRewriter.cpp`: + +- Entry: `ANS_DBG("Fp16Rewriter", "ResolveFp16OnnxPath: src=%s cache=%s", + fp32Path.c_str(), cacheDir.c_str());` +- Cache hit: `ANS_DBG("Fp16Rewriter", "cache HIT %s", cachedPath.c_str());` +- Cache miss → rewrite: `ANS_DBG("Fp16Rewriter", + "cache MISS — rewriting (initializers=%zu nodes=%zu)", ...);` +- Per phase: log initializer-conversion count, Cast-insertion count, + value_info update count. +- On error: `ANS_DBG("Fp16Rewriter", + "ERROR: %s — falling back to FP32 source", ex.what());` and return + the original FP32 path (never block inference). + +### Step 7 — Per-model accuracy validation (1–2 days) + +You can't ship FP16 without per-model accuracy validation. Build a +small C++ harness — separate executable target, runs against +labeled fixtures from your test data: + +``` +tools/AccuracyComparator/AccuracyComparator.cpp + - Loads two sessions (FP32 and FP16) on CPU EP for determinism. + - Feeds N representative frames through both. + - For YOLO det: compare box IoU, score deltas, count of boxes lost. + - For OCR det: compare text-region masks (pixel-wise IoU on the prob + map binary at threshold). + - For OCR rec: compare DECODED CTC strings — not raw logits. + A single character delta on a plate is a regression. + - Emit pass/fail against a configurable tolerance. +``` + +Acceptance criteria: + +| Model | Tolerance | What "pass" means | +|---|---|---| +| YOLO LP detector | mean IoU ≥ 0.95 vs FP32 boxes | No boxes lost, score delta < 0.05 | +| YOLO Helmet OD | mean IoU ≥ 0.95 | Same | +| OCR DBNet | mask IoU ≥ 0.92 at threshold | No new false negatives | +| OCR CTC recognizer | char accuracy within 0.5% | If fails, leave AMD FP32 | + +Run on at least 500 frames per model from production data. + +### Step 8 — Build, deploy, measure (half day) + +Build artifacts: `ANSODEngine.dll`, `ANSOCR.dll`. No new DLL — the +rewriter lives inside the existing engine layer. + +Deploy to AMD test box. First run: each session takes an extra +2–10 s during initial cache build. Look for `[Fp16Rewriter] cache +MISS — rewriting` lines. + +Subsequent runs: instant cache hit. Look for `[Fp16Rewriter] cache HIT`. + +Measure: +- Per-call DML latency in `STATS` lines (mean, p99). +- Total fps over a 60-second run. +- Compare to the always-serial-on-AMD baseline (5.4 fps full-frame ALPR). +- Target: ≥ 7 fps. + +If a model regresses accuracy in the field (rare with the validation +harness in Step 7, but possible), flip its `dmlFP16` back to `false` +and ship a hotfix. No code change needed beyond the option flag. + +## 6. Dependencies summary + +| Dep | Where it lives | New or existing | +|---|---|---| +| `libprotobuf` (C++ runtime) | Almost certainly already vendored via onnxruntime; otherwise add via vcpkg or `C:\ANSLibs\protobuf` | Existing, just expose includes | +| `onnx.proto3` schema file | Vendor at `engines/ONNXEngine/proto/onnx.proto3` from ONNX repo | New, ~1 KB file | +| `protoc` compiler | Comes with libprotobuf install | Existing, just call from CMake | +| `` for cache dir | C++17 standard | Existing | +| SHA1 helper for cache key | Pull from any tiny public-domain impl, or use libcrypto if linked | Tiny new file (~80 lines) | + +No third-party FP16 libraries. No Python. No new DLLs in deployment. + +## 7. Risks and mitigations + +| Risk | Likelihood | Mitigation | +|---|---|---| +| Conversion produces NaN/Inf in some op | Medium | Default blocklist + per-model `dmlFP16OpBlockList` override. Always validate with Step 7 harness before flipping a model on. | +| Some FP16 op has no DML kernel → falls back to CPU silently → SLOWER than FP32 | Medium | Profile each model on AMD with ORT's profiling JSON output (`SessionOptions::EnableProfiling`). If a CPU fallback shows up, blocklist the offending op for that model. | +| ONNX schema version mismatch (we vendor proto3, models use proto with newer fields) | Low | Models from this project are exported by Ultralytics + PaddleOCR — both target stable opset 12-17 ranges that proto3 covers fully. Validate by attempting parse on a known-good model during Step 3. | +| Cache directory permissions on customer machines | Low | Default to `%TEMP%` which is always writable; allow override via `dmlOptimizedModelDir`. On any cache-write failure, fall back to FP32 source path (no inference impact). | +| Subgraph (If/Loop/Scan) bug | Low for current models | Recurse defensively in Step 3f; assert if a subgraph is encountered, fail-safe to FP32. | + +## 8. Out of scope (do NOT do as part of this plan) + +- **Auto mixed precision** (per-op accuracy-driven precision selection). + That requires running a representative dataset through the model to + decide which ops can safely be FP16. Doable in C++ but adds 1+ week. + Defer to a follow-up plan. +- **Model quantization to INT8.** Different problem, different tooling + (calibration data needed). +- **Switching from DML to MIGraphX or ROCm.** Different EP, different + port. Defer. +- **Recovering depth=2 parallelism on AMD.** The always-serial fix is + in place and stable; this plan is purely about per-call speedup + while remaining serial. Combine the two for net throughput gain. + +## 9. Project rules to follow (from MEMORY.md) + +- **Edit directly in main checkout.** No worktree, no branch. +- **No commits without explicit user request.** When this plan ships, + don't auto-commit. +- **Sprinkle `ANS_DBG` on new code paths** — entry, decisions, errors. + See Step 6 for exact placement. + +## 10. Acceptance checklist + +When this plan is done, the following must all be true: + +- [ ] `OrtHandlerOptions::dmlFP16` exists and is documented inline. +- [ ] `Fp16GraphRewriter.{h,cpp}` exists with `ResolveFp16OnnxPath` exposed. +- [ ] `BasicOrtHandler::initialize_handler` calls the resolver in the + AMD branch when `dmlFP16` is set. +- [ ] At least one production model (LP YOLO det) is opted in via its + option-builder. +- [ ] Accuracy harness in `tools/AccuracyComparator/` exists and produces + pass/fail for the opted-in models. +- [ ] AMD test-box measurement shows per-call DML latency reduced by + ≥ 25% on the opted-in model vs FP32 baseline. +- [ ] No regression on Intel or NVIDIA — the FP16 path never executes + on those EPs (`if (options.dmlFP16)` is gated on `EngineType::AMD_GPU` + via the case-branch placement). +- [ ] DebugView logs cleanly: cache HIT / cache MISS / rewrite phases + / errors all visible. +- [ ] First-run cache-build latency is ≤ 10 s per model on AMD test box. +- [ ] Subsequent-run cache-load latency is ≤ 100 ms per model. + +## 11. Open questions to resolve at start + +When you start this plan in a future session, answer these first: + +1. Is `libprotobuf` already linked into ANSODEngine.dll (transitively + via onnxruntime)? Run `dumpbin /imports ANSODEngine.dll | grep + protobuf` to confirm. If yes, skip the vcpkg/manual install step. +2. Where does `C:\ANSLibs\onnxruntime\include` live exactly? Confirm + path for the protoc include directive. +3. Which ONNX opset version do your YOLO and OCR models target? + Run `Ort::Session` with profiling once and check the model + metadata. This confirms which `onnx.proto3` revision to vendor. +4. Is the initial AMD measurement still ~5.4 fps with the always-serial + fix in place? If something regressed in the meantime, fix that + first before adding FP16. + +--- + +*Last updated: when this plan was authored. No code shipped yet.* diff --git a/modules/ANSUtilities/ANSAWSS3.cpp b/modules/ANSUtilities/ANSAWSS3.cpp index 21970e5..7672572 100644 --- a/modules/ANSUtilities/ANSAWSS3.cpp +++ b/modules/ANSUtilities/ANSAWSS3.cpp @@ -264,13 +264,21 @@ namespace ANSCENTER } // TryConnect — actual connection attempt (caller must hold _configMutex) + // + // First attempts the AWS virtual-host name (s3..); for + // S3-compatible servers like MinIO/Ceph that hostname does not exist, so + // we fall back to the raw _baseDomain (path-style). The first attempt is + // therefore *expected* to fail on MinIO — we suppress its error-level log + // (logOnFailure=false) so the operator only sees errors when the fallback + // also fails. bool ANSAWSS3::TryConnect(bool& awsPath) { - auto testConn = CreateConnection(); + const bool fallbackPossible = (_fullAWSURL != _baseDomain); + auto testConn = CreateConnection(/*logOnFailure*/ !fallbackPossible); if (!testConn) { // Fallback: connect directly to baseDomain (MinIO, Ceph, etc.) std::string savedFullURL = _fullAWSURL; _fullAWSURL = _baseDomain; - testConn = CreateConnection(); + testConn = CreateConnection(/*logOnFailure*/ true); if (!testConn) { _fullAWSURL = savedFullURL; // restore on failure return false; @@ -305,11 +313,18 @@ namespace ANSCENTER attempt++; - // Quick DNS check — is internet available? + // Quick DNS check — is internet available? Try AWS virtual-host + // form first, then fall back to raw _baseDomain for MinIO/Ceph. { CkSocket dnsCheck; CkString dnsResult; - if (!dnsCheck.DnsLookup(_fullAWSURL.c_str(), 3000, dnsResult)) { + bool resolved = dnsCheck.DnsLookup(_fullAWSURL.c_str(), 3000, dnsResult); + if (!resolved && _fullAWSURL != _baseDomain) { + CkSocket dnsCheck2; + CkString dnsResult2; + resolved = dnsCheck2.DnsLookup(_baseDomain.c_str(), 3000, dnsResult2); + } + if (!resolved) { _logger.LogDebug("ANSAWSS3::RetryLoop", "Retry #" + std::to_string(attempt) + " - no internet, retrying in 3s", __FILE__, __LINE__); @@ -343,7 +358,11 @@ namespace ANSCENTER } // ── Connection pool helpers ── - std::unique_ptr ANSAWSS3::CreateConnection() { + // logOnFailure=false: caller knows this attempt may legitimately fail + // (e.g. TryConnect's first probe of the AWS virtual-host name on a MinIO + // server). Failures are still emitted to ANS_DBG; just not as error-level + // log entries that would scare ops dashboards. + std::unique_ptr ANSAWSS3::CreateConnection(bool logOnFailure) { auto conn = std::make_unique(); // Apply request timeouts BEFORE Connect() so ConnectTimeoutMs takes @@ -355,8 +374,11 @@ namespace ANSCENTER // Connect if (!conn->rest.Connect(_fullAWSURL.c_str(), _port, _bTls, _bAutoReconnect)) { const std::string summary = SummarizeChilkatError(conn->rest.lastErrorText()); - _logger.LogError("ANSAWSS3::CreateConnection", summary, __FILE__, __LINE__); - ANS_DBG("AWSS3", "Connect failed: %s", summary.c_str()); + if (logOnFailure) { + _logger.LogError("ANSAWSS3::CreateConnection", summary, __FILE__, __LINE__); + } + ANS_DBG("AWSS3", "Connect failed%s: %s", + logOnFailure ? "" : " (probe, retry expected)", summary.c_str()); return nullptr; } @@ -369,8 +391,11 @@ namespace ANSCENTER conn->socket.put_HttpProxyForHttp(_bProxy); if (!conn->rest.UseConnection(conn->socket, true)) { const std::string summary = SummarizeChilkatError(conn->rest.lastErrorText()); - _logger.LogError("ANSAWSS3::CreateConnection - Proxy error", summary, __FILE__, __LINE__); - ANS_DBG("AWSS3", "Proxy error: %s", summary.c_str()); + if (logOnFailure) { + _logger.LogError("ANSAWSS3::CreateConnection - Proxy error", summary, __FILE__, __LINE__); + } + ANS_DBG("AWSS3", "Proxy error%s: %s", + logOnFailure ? "" : " (probe, retry expected)", summary.c_str()); return nullptr; } } @@ -385,8 +410,11 @@ namespace ANSCENTER } if (!conn->rest.SetAuthAws(conn->authAws)) { const std::string summary = SummarizeChilkatError(conn->rest.lastErrorText()); - _logger.LogError("ANSAWSS3::CreateConnection - Auth error", summary, __FILE__, __LINE__); - ANS_DBG("AWSS3", "Auth error: %s", summary.c_str()); + if (logOnFailure) { + _logger.LogError("ANSAWSS3::CreateConnection - Auth error", summary, __FILE__, __LINE__); + } + ANS_DBG("AWSS3", "Auth error%s: %s", + logOnFailure ? "" : " (probe, retry expected)", summary.c_str()); return nullptr; } } @@ -678,14 +706,28 @@ namespace ANSCENTER _pool.clear(); } - // Check if internet is available (fast DNS check) + // Check if internet is available (fast DNS check). Try the AWS + // virtual-host form first; if that synthetic name doesn't exist + // (MinIO, Ceph, custom S3-compatible servers), fall back to the + // raw _baseDomain — that is the actual host the user gave us. { CkSocket dnsCheck; CkString dnsResult; - if (!dnsCheck.DnsLookup(_fullAWSURL.c_str(), 3000, dnsResult)) { + bool resolved = dnsCheck.DnsLookup(_fullAWSURL.c_str(), 3000, dnsResult); + if (!resolved && _fullAWSURL != _baseDomain) { + CkSocket dnsCheck2; + CkString dnsResult2; + resolved = dnsCheck2.DnsLookup(_baseDomain.c_str(), 3000, dnsResult2); + if (resolved) { + ANS_DBG("AWSS3", + "DNS for '%s' failed but '%s' resolved — proceeding with path-style/MinIO fallback", + _fullAWSURL.c_str(), _baseDomain.c_str()); + } + } + if (!resolved) { // No internet — start background retry, return immediately _logger.LogDebug("ANSAWSS3::Connect", - "No internet available, starting background retry for '" + _fullAWSURL + "'...", + "No internet available, starting background retry for '" + _fullAWSURL + "' / '" + _baseDomain + "'...", __FILE__, __LINE__); _stopRetry = false; @@ -1705,7 +1747,7 @@ namespace ANSCENTER int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode == 200) { std::string scheme = _bTls ? "https://" : "http://"; - uploadedFilePath = scheme + _fullAWSURL + "/" + bucketName + "/" + fileName; + uploadedFilePath = scheme + (_bAwsPath ? _fullAWSURL : _baseDomain) + "/" + bucketName + "/" + fileName; _logger.LogDebug(kOp, "Successfully uploaded: " + fileName + " | URL: " + uploadedFilePath, __FILE__, __LINE__); @@ -1786,7 +1828,7 @@ namespace ANSCENTER int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode == 200) { std::string scheme = _bTls ? "https://" : "http://"; - uploadedFilePath = scheme + _fullAWSURL + "/" + bucketName + "/" + fileName; + uploadedFilePath = scheme + (_bAwsPath ? _fullAWSURL : _baseDomain) + "/" + bucketName + "/" + fileName; _logger.LogDebug(kOp, "Successfully uploaded: " + fileName + " | URL: " + uploadedFilePath, __FILE__, __LINE__); @@ -1890,7 +1932,7 @@ namespace ANSCENTER // Build canonical path-style URL from parts — independent // of the addressing style we actually used. std::string scheme = _bTls ? "https://" : "http://"; - uploadedFilePath = scheme + _fullAWSURL + "/" + bucketName + "/" + objectKey; + uploadedFilePath = scheme + (_bAwsPath ? _fullAWSURL : _baseDomain) + "/" + bucketName + "/" + objectKey; _logger.LogDebug(kOp, "Successfully uploaded to: " + objectKey + " | URL: " + uploadedFilePath, __FILE__, __LINE__); @@ -1959,7 +2001,7 @@ namespace ANSCENTER int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode == 200) { std::string scheme = _bTls ? "https://" : "http://"; - uploadedFilePath = scheme + _fullAWSURL + "/" + bucketName + "/" + fileName; + uploadedFilePath = scheme + (_bAwsPath ? _fullAWSURL : _baseDomain) + "/" + bucketName + "/" + fileName; _logger.LogDebug(kOp, "Successfully uploaded file: " + fileName + " | URL: " + uploadedFilePath, __FILE__, __LINE__); @@ -2170,7 +2212,7 @@ namespace ANSCENTER } std::string scheme = _bTls ? "https://" : "http://"; - uploadedFilePath = scheme + _fullAWSURL + "/" + bucketName + "/" + fileName; + uploadedFilePath = scheme + (_bAwsPath ? _fullAWSURL : _baseDomain) + "/" + bucketName + "/" + fileName; _logger.LogDebug(kOp, "Multipart upload completed for: " + fileName + " | URL: " + uploadedFilePath, __FILE__, __LINE__); @@ -2387,7 +2429,7 @@ namespace ANSCENTER // Build canonical path-style URL from parts — independent // of the addressing style we actually used. std::string scheme = _bTls ? "https://" : "http://"; - uploadedFilePath = scheme + _fullAWSURL + "/" + bucketName + "/" + objectKey; + uploadedFilePath = scheme + (_bAwsPath ? _fullAWSURL : _baseDomain) + "/" + bucketName + "/" + objectKey; _logger.LogDebug(kOp, "Multipart upload completed. S3 key: " + objectKey + " | URL: " + uploadedFilePath, __FILE__, __LINE__); @@ -2451,7 +2493,7 @@ namespace ANSCENTER int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode == 200) { std::string scheme = _bTls ? "https://" : "http://"; - uploadedFilePath = scheme + _fullAWSURL + "/" + bucketName + "/" + fileName; + uploadedFilePath = scheme + (_bAwsPath ? _fullAWSURL : _baseDomain) + "/" + bucketName + "/" + fileName; _logger.LogDebug(kOp, "Successfully uploaded: " + fileName + " | URL: " + uploadedFilePath, __FILE__, __LINE__); @@ -2541,7 +2583,7 @@ namespace ANSCENTER // Build the returned URL from parts (independent of // addressing style) — always the canonical path-style URL. std::string scheme = _bTls ? "https://" : "http://"; - uploadedFilePath = scheme + _fullAWSURL + "/" + bucketName + "/" + objectKey; + uploadedFilePath = scheme + (_bAwsPath ? _fullAWSURL : _baseDomain) + "/" + bucketName + "/" + objectKey; _logger.LogDebug(kOp, "Successfully uploaded: " + objectKey + " (" + std::to_string(bufferLength) + " bytes) | URL: " + uploadedFilePath, __FILE__, __LINE__); diff --git a/modules/ANSUtilities/ANSUtilities.h b/modules/ANSUtilities/ANSUtilities.h index 3316be5..d00882e 100644 --- a/modules/ANSUtilities/ANSUtilities.h +++ b/modules/ANSUtilities/ANSUtilities.h @@ -197,7 +197,9 @@ namespace ANSCENTER { SPDLogger& _logger = SPDLogger::GetInstance("ANSAWSS3", false); // Pool helpers - std::unique_ptr CreateConnection(); + // logOnFailure=false suppresses error-level logging on failure (still emits ANS_DBG). + // Used by TryConnect's first attempt, which is allowed to fail for path-style/MinIO servers. + std::unique_ptr CreateConnection(bool logOnFailure = true); std::unique_ptr AcquireConnection(); void ReleaseConnection(std::unique_ptr conn); diff --git a/tests/ANSUtilities-UnitTest/ANSUtilities-UnitTest.cpp b/tests/ANSUtilities-UnitTest/ANSUtilities-UnitTest.cpp index b630a42..95a4d37 100644 --- a/tests/ANSUtilities-UnitTest/ANSUtilities-UnitTest.cpp +++ b/tests/ANSUtilities-UnitTest/ANSUtilities-UnitTest.cpp @@ -267,7 +267,7 @@ int ANSFTechMinIOTest() { std::string baseDomain = "https://minio.dev.ftech.ai"; std::string bucketRegion = "vn1"; std::string serviceName = "s3"; - std::string bucketName = "aicamera"; + std::string bucketName = "funzy"; int port = 443; bool bTls = true; @@ -281,8 +281,8 @@ int ANSFTechMinIOTest() { } // Set authentication - std::string accessKey = "aicamerapush"; - std::string secretKey = "88fO8Y3EpMFMG1HFCCcfCE4cWh0k5KKZ"; + std::string accessKey = "tuanpm"; + std::string secretKey = "BRFbQujGhkKrI3xqSUE9ywnpXHt27c"; if (SetAuthenticationANSAWSHandle(&awsHandle, accessKey.c_str(), secretKey.c_str()) != 1) { std::cerr << "Failed to set AWS S3 authentication." << std::endl; ReleaseANSAWSHandle(&awsHandle); @@ -389,9 +389,12 @@ int ANSLLMOllamaTest() { int main() { //ANSLLMOllamaTest(); + + std::cout << "===== MinIO (path-style) =====" << std::endl; + ANSFTechMinIOTest(); + + std::cout << "\n===== AWS S3 (virtual-host) =====" << std::endl; ANSAWSTest(); - //ANSFTechMinIOTest(); - //ANSAWSTest(); //ANSLLMTest(); //SendEmailExample(); //uploadFileToGoogleStorage();