#include "ANSUtilities.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static bool ansawss3LicenceValid = false; static std::once_flag ansawss3LicenseOnceFlag; namespace ANSCENTER { static void VerifyGlobalANSAWSS3License(const std::string& licenseKey) { try { static const std::vector> licenseChecks = { {1000, "ANNHUB-LV"}, {1001, "DLHUB-LV"}, {1002, "ODHUB-LV"}, {1003, "ANSVIS"}, {1004, "ANSFR"}, {1005, "ANSOCR"}, {1006, "ANSALPR"}, {1007, "ANSCV"}, {1008, "ANSSRT"} }; ansawss3LicenceValid = false; for (const auto& [productId, productName] : licenseChecks) { if (ANSCENTER::ANSLicenseHelper::LicenseVerification(licenseKey, productId, productName)) { ansawss3LicenceValid = true; break; // Stop at the first valid license } } } catch (const std::exception& e) { ansawss3LicenceValid = false; } } // Builds a per-call unique suffix for scratch file names used by // multipart uploads. Combining thread id, instance address, and a // monotonically-increasing counter guarantees no collision between // parallel uploads — including two threads uploading the same source // file at the same time (Fix 2). static std::string MakeUniqueMultipartToken(const void* instancePtr) { static std::atomic counter{0}; std::ostringstream oss; oss << std::hex << std::hash{}(std::this_thread::get_id()) << "_" << reinterpret_cast(instancePtr) << "_" << counter.fetch_add(1, std::memory_order_relaxed); return oss.str(); } // ──────────────────────────────────────────────────────────────────────── // SummarizeChilkatError: turn Chilkat's verbose multi-line lastErrorText() // into a single diagnostic line suitable for _logger.LogError(). // // Chilkat's log looks like: // ChilkatLog: Connect: DllDate: ... ChilkatVersion: ... UnlockStatus: ... // restConnect: domain_or_ip: s3.us-east-1.amazonaws.com // socket2Connect: connect2: connectImplicitSsl: connectSocket_v2: // connect_domain: ckDnsResolveDomainIPv4_n: ... // clientHandshake2: readHandshakeMessages: // Failed to read beginning of SSL/TLS record. // readTlsRecord: Socket operation timeout. // See https://cknotes.com/... // ConnectFailReason: 103 // Failed. // // Diagnostics we care about: // • the deepest concrete failure reason (the non-label leaf line) // • ConnectFailReason: if present // • the target (domain_or_ip / ip_or_domain + port) // Rest (DllDate, ChilkatVersion, Architecture, VerboseLogging, ...) is noise. // // Returns e.g.: // "Connect failed [reason=103] s3.us-east-1.amazonaws.com:443 — Socket operation timeout (TLS handshake). See https://cknotes.com/failed-to-read-beginning-of-ssl-tls-record-..." // ──────────────────────────────────────────────────────────────────────── static std::string SummarizeChilkatError(const char* raw) { if (!raw || !*raw) return "(empty Chilkat log)"; std::string s(raw); // Split into trimmed lines. std::vector lines; { std::istringstream iss(s); std::string ln; while (std::getline(iss, ln)) { // Trim leading whitespace (Chilkat indents by level). size_t a = ln.find_first_not_of(" \t\r\n"); if (a == std::string::npos) continue; size_t b = ln.find_last_not_of(" \t\r\n"); lines.push_back(ln.substr(a, b - a + 1)); } } if (lines.empty()) return "(empty Chilkat log)"; // Lines that are pure labels ("foo:" or "--foo") are scaffolding. auto isLabelLine = [](const std::string& ln) { if (ln.empty()) return true; if (ln.rfind("--", 0) == 0) return true; // "--foo" close-marker // "label: value" — keep only if value is non-empty and line doesn't // look like pure call-stack breadcrumb (trailing ':'). if (!ln.empty() && ln.back() == ':') return true; return false; }; // Fields we want to extract. std::string target, failReason, tlsHint, leafError, seeUrl; for (auto& ln : lines) { if (ln.rfind("domain_or_ip:", 0) == 0 || ln.rfind("ip_or_domain:", 0) == 0) { auto p = ln.find(':'); if (p != std::string::npos) target = ln.substr(p + 1); } else if (ln.rfind("ConnectFailReason:", 0) == 0) { auto p = ln.find(':'); if (p != std::string::npos) failReason = ln.substr(p + 1); } else if (ln.rfind("port:", 0) == 0) { auto p = ln.find(':'); if (p != std::string::npos) { // Append port to target if we have one. std::string port = ln.substr(p + 1); // Trim leading space. size_t a = port.find_first_not_of(" \t"); if (a != std::string::npos) port = port.substr(a); if (!target.empty()) target += ":" + port; } } else if (ln.rfind("See http", 0) == 0 || ln.rfind("See https", 0) == 0) { seeUrl = ln.substr(4); // drop "See " } else if (ln.rfind("Failed to read beginning of SSL/TLS record", 0) == 0) { tlsHint = "TLS handshake — server sent nothing (firewall/AV TLS inspection, proxy, or packet loss)"; } else if (!isLabelLine(ln)) { // Candidate concrete message. Keep the LATEST one that's not a // stack label; Chilkat's deepest concrete error is usually the // most specific explanation. leafError = ln; } } // Trim leading space on captured fields. auto trim = [](std::string& v) { size_t a = v.find_first_not_of(" \t"); if (a == std::string::npos) { v.clear(); return; } size_t b = v.find_last_not_of(" \t"); v = v.substr(a, b - a + 1); }; trim(target); trim(failReason); trim(leafError); trim(seeUrl); // Compose. std::ostringstream out; out << "Chilkat Connect failed"; if (!failReason.empty()) out << " [reason=" << failReason << "]"; if (!target.empty()) out << " " << target; if (!leafError.empty()) out << " — " << leafError; if (!tlsHint.empty()) out << " (" << tlsHint << ")"; if (!seeUrl.empty()) out << " (" << seeUrl << ")"; return out.str(); } // ── Per-thread call status ─────────────────────────────────────────────── // Written by every upload entry point, read by the C exports in dllmain so // the caller learns WHY a call failed (and therefore whether to retry) // without any change to the exported signatures. Thread-local because a // single ANSAWSS3 handle is routinely shared by many worker threads — a // per-instance field would report another worker's outcome. static thread_local int t_callStatus = ANSAWSS3::kCallPermanentError; void ANSAWSS3::SetCallStatus(int status) { t_callStatus = status; } int ANSAWSS3::LastCallStatus() { return t_callStatus; } // ──────────────────────────────────────────────────────────────────────── // SummarizeChilkatRequestError: one line out of a request-level Chilkat log. // // The raw log is ~2 KB of call-stack scaffolding per failure. Written once // per retry, per worker, it buries the one thing an operator needs: what S3 // actually said. S3 returns its reason as XML in the response body, which // Chilkat echoes into the log: // NoSuchBucketThe specified bucket does // not exist... // Pull that out, prefix the status code, and drop the rest. // ──────────────────────────────────────────────────────────────────────── static std::string SummarizeChilkatRequestError(const char* raw, int statusCode) { std::ostringstream out; if (statusCode > 0) out << "HTTP " << statusCode; auto tagValue = [](const std::string& s, const char* tag) -> std::string { const std::string open = std::string("<") + tag + ">"; const std::string close = std::string(""; const size_t a = s.find(open); if (a == std::string::npos) return {}; const size_t b = s.find(close, a + open.size()); if (b == std::string::npos) return {}; return s.substr(a + open.size(), b - a - open.size()); }; if (raw && *raw) { const std::string s(raw); const std::string code = tagValue(s, "Code"); const std::string msg = tagValue(s, "Message"); if (!code.empty()) { if (statusCode > 0) out << " "; out << code; if (!msg.empty()) out << ": " << msg; } else { // No S3 error document — fall back to the deepest concrete line // (same idea as SummarizeChilkatError, without the connect // specific fields). std::istringstream iss(s); std::string ln, leaf; while (std::getline(iss, ln)) { const size_t a = ln.find_first_not_of(" \t\r\n"); if (a == std::string::npos) continue; const size_t b = ln.find_last_not_of(" \t\r\n"); ln = ln.substr(a, b - a + 1); if (ln.rfind("--", 0) == 0) continue; // close marker if (!ln.empty() && ln.back() == ':') continue; // stack label if (ln == "Failed.") continue; leaf = ln; } if (!leaf.empty()) { if (statusCode > 0) out << " — "; out << leaf; } } } std::string result = out.str(); return result.empty() ? "(no detail in Chilkat log)" : result; } // Private helper function to extract file name from a path // Helper function to extract filename from path std::string ANSAWSS3::ExtractFileName(const std::string& filePath) { size_t pos = filePath.find_last_of("/\\"); if (pos != std::string::npos) { return filePath.substr(pos + 1); } return filePath; } // Helper function to determine content type from file extension std::string ANSAWSS3::GetContentType(const std::string& filePath) { // Default to application/octet-stream for unknown types std::string contentType = "application/octet-stream"; size_t extPos = filePath.find_last_of('.'); if (extPos == std::string::npos) { return contentType; } // Extract and convert extension to lowercase std::string fileExt = filePath.substr(extPos + 1); std::transform(fileExt.begin(), fileExt.end(), fileExt.begin(), ::tolower); // Image types if (fileExt == "jpg" || fileExt == "jpeg") { contentType = "image/jpeg"; } else if (fileExt == "png") { contentType = "image/png"; } else if (fileExt == "gif") { contentType = "image/gif"; } else if (fileExt == "bmp") { contentType = "image/bmp"; } else if (fileExt == "webp") { contentType = "image/webp"; } else if (fileExt == "svg") { contentType = "image/svg+xml"; } else if (fileExt == "ico") { contentType = "image/x-icon"; } // Video types else if (fileExt == "mp4") { contentType = "video/mp4"; } else if (fileExt == "avi") { contentType = "video/x-msvideo"; } else if (fileExt == "mov") { contentType = "video/quicktime"; } // Document types else if (fileExt == "pdf") { contentType = "application/pdf"; } else if (fileExt == "json") { contentType = "application/json"; } else if (fileExt == "xml") { contentType = "application/xml"; } else if (fileExt == "txt") { contentType = "text/plain"; } return contentType; } ANSAWSS3::ANSAWSS3() { _unlockCode = "ANSDRC.CB1122026_MEQCIFwO1IFQCG0BhZwsXFO68QUU6mDB5uge4duOsqOJanEyAiAB67ahqnXin4SRy0vIegISgbFlpldmbuS5gbU21GYVqA==";// "ANSDRC.CB1082025_Ax6P3M7F8B3d";// _proxyHost = ""; _proxyPort = 0; _proxyUsername = ""; _proxyPassword = ""; _bProxy = false; _serviceName = "s3"; } ANSAWSS3::~ANSAWSS3() { StopRetry(); std::lock_guard lk(_poolMutex); _pool.clear(); // CkRest destructors handle disconnect } void ANSAWSS3::StopRetry() { _stopRetry = true; if (_retryThread.joinable()) { _retryThread.join(); } _retryInProgress = false; } // TryConnect — actual connection attempt (caller must hold _configMutex) // // First attempts the AWS virtual-host name (s3..); for // S3-compatible servers like MinIO/Ceph that hostname does not exist, so // we fall back to the raw _baseDomain (path-style). The first attempt is // therefore *expected* to fail on MinIO — we suppress its error-level log // (logOnFailure=false) so the operator only sees errors when the fallback // also fails. bool ANSAWSS3::TryConnect(bool& awsPath) { const bool fallbackPossible = (_fullAWSURL != _baseDomain); auto testConn = CreateConnection(/*logOnFailure*/ !fallbackPossible); if (!testConn) { // Fallback: connect directly to baseDomain (MinIO, Ceph, etc.) std::string savedFullURL = _fullAWSURL; _fullAWSURL = _baseDomain; testConn = CreateConnection(/*logOnFailure*/ true); if (!testConn) { _fullAWSURL = savedFullURL; // restore on failure return false; } awsPath = false; _bAwsPath = false; } else { awsPath = true; _bAwsPath = true; } _bConnected = true; // Return test connection to pool for reuse { std::lock_guard lk(_poolMutex); _pool.push_back(std::move(testConn)); } return true; } // Background connect loop, started by Connect() when the endpoint could not // be reached at all (return code 2). // // This loop NEVER gives up on its own. The caller — a LabVIEW VI, an // unattended AIBOX — calls Connect() exactly once at startup, so if this // thread returned while still unconnected the handle would stay dead for // the life of the process and every upload would fail instantly. Backoff // grows 3 s -> 30 s so a long outage costs little, and the loop only exits // once connected or when StopRetry() is called. void ANSAWSS3::RetryLoop() { int attempt = 0; int backoffSec = 3; constexpr int kMaxBackoffSec = 30; while (!_stopRetry) { // Sleep in 1-second increments so we can respond to stop quickly. for (int i = 0; i < backoffSec && !_stopRetry; ++i) { std::this_thread::sleep_for(std::chrono::seconds(1)); } if (_stopRetry) break; attempt++; // Quick DNS check — is internet available? Try AWS virtual-host // form first, then fall back to raw _baseDomain for MinIO/Ceph. { CkSocket dnsCheck; CkString dnsResult; bool resolved = dnsCheck.DnsLookup(_fullAWSURL.c_str(), 3000, dnsResult); if (!resolved && _fullAWSURL != _baseDomain) { CkSocket dnsCheck2; CkString dnsResult2; resolved = dnsCheck2.DnsLookup(_baseDomain.c_str(), 3000, dnsResult2); } if (!resolved) { _logger.LogDebug("ANSAWSS3::RetryLoop", "Retry #" + std::to_string(attempt) + " - no internet, retrying in " + std::to_string(backoffSec) + "s", __FILE__, __LINE__); ANS_DBG("AWSS3", "RetryLoop #%d: DNS still failing, next try in %ds", attempt, backoffSec); backoffSec = (std::min)(kMaxBackoffSec, backoffSec * 2); continue; } } // Internet is available — attempt real connection to validate auth/URL { std::lock_guard lk(_configMutex); bool awsPath = true; if (TryConnect(awsPath)) { _logger.LogDebug("ANSAWSS3::RetryLoop", "Connected successfully after " + std::to_string(attempt) + " retries", __FILE__, __LINE__); ANS_DBG("AWSS3", "RetryLoop: connected after %d attempt(s)", attempt); _retryInProgress = false; return; } } // DNS resolves but the connection failed — bad credentials, wrong // URL, or the endpoint is refusing us. Keep trying anyway: the // caller has no other way back, and this state is often temporary // (endpoint rebooting, TLS interception clearing, clock skew being // corrected). Backoff keeps the cost negligible. _logger.LogError("ANSAWSS3::RetryLoop", "Retry #" + std::to_string(attempt) + " - endpoint resolves but connection failed (check authentication or URL). " "Retrying in " + std::to_string(backoffSec) + "s", __FILE__, __LINE__); ANS_DBG("AWSS3", "RetryLoop #%d: connect failed, next try in %ds", attempt, backoffSec); backoffSec = (std::min)(kMaxBackoffSec, backoffSec * 2); } _retryInProgress = false; } // ── Connection pool helpers ── // logOnFailure=false: caller knows this attempt may legitimately fail // (e.g. TryConnect's first probe of the AWS virtual-host name on a MinIO // server). Failures are still emitted to ANS_DBG; just not as error-level // log entries that would scare ops dashboards. std::unique_ptr ANSAWSS3::CreateConnection(bool logOnFailure) { auto conn = std::make_unique(); // Stamp the config generation this connection is built against so a // later Connect / SetAuthentication / SetServerProxy can invalidate it // even if it lands in the pool after that call cleared it. conn->configGen = _configGen.load(); // Apply request timeouts BEFORE Connect() so ConnectTimeoutMs takes // effect on this very handshake. IdleTimeoutMs is sticky and applies // to every subsequent request on this CkRest. A brand-new connection // gets the FULL idle timeout — it is known-good, so it should be given // every chance to complete; only reuse gets the short leash. conn->rest.put_ConnectTimeoutMs(_connectTimeoutMs.load()); conn->rest.put_IdleTimeoutMs(_idleTimeoutMs.load()); // Connect if (!conn->rest.Connect(_fullAWSURL.c_str(), _port, _bTls, _bAutoReconnect)) { const std::string summary = SummarizeChilkatError(conn->rest.lastErrorText()); if (logOnFailure) { _logger.LogError("ANSAWSS3::CreateConnection", summary, __FILE__, __LINE__); } ANS_DBG("AWSS3", "Connect failed%s: %s", logOnFailure ? "" : " (probe, retry expected)", summary.c_str()); return nullptr; } // Apply proxy if configured if (_bProxy) { conn->socket.put_HttpProxyHostname(_proxyHost.c_str()); conn->socket.put_HttpProxyPort(_proxyPort); conn->socket.put_HttpProxyUsername(_proxyUsername.c_str()); conn->socket.put_HttpProxyPassword(_proxyPassword.c_str()); conn->socket.put_HttpProxyForHttp(_bProxy); if (!conn->rest.UseConnection(conn->socket, true)) { const std::string summary = SummarizeChilkatError(conn->rest.lastErrorText()); if (logOnFailure) { _logger.LogError("ANSAWSS3::CreateConnection - Proxy error", summary, __FILE__, __LINE__); } ANS_DBG("AWSS3", "Proxy error%s: %s", logOnFailure ? "" : " (probe, retry expected)", summary.c_str()); return nullptr; } } // Apply auth if (_authReady) { conn->authAws.put_AccessKey(_accessKey.c_str()); conn->authAws.put_SecretKey(_secretKey.c_str()); conn->authAws.put_ServiceName(_serviceName.c_str()); if (_bucketRegion != "us-east-1") { if (!_bucketRegion.empty()) conn->authAws.put_Region(_bucketRegion.c_str()); } if (!conn->rest.SetAuthAws(conn->authAws)) { const std::string summary = SummarizeChilkatError(conn->rest.lastErrorText()); if (logOnFailure) { _logger.LogError("ANSAWSS3::CreateConnection - Auth error", summary, __FILE__, __LINE__); } ANS_DBG("AWSS3", "Auth error%s: %s", logOnFailure ? "" : " (probe, retry expected)", summary.c_str()); return nullptr; } } return conn; } // Take a connection from the pool, skipping any that are stale (idle too // long) or built against superseded config. Falls through to a fresh // connection when nothing reusable is left. std::unique_ptr ANSAWSS3::AcquireConnection() { const uint64_t gen = _configGen.load(); const auto now = std::chrono::steady_clock::now(); for (;;) { std::unique_ptr conn; { std::lock_guard lk(_poolMutex); if (_pool.empty()) break; conn = std::move(_pool.back()); _pool.pop_back(); } if (conn->configGen != gen) { ANS_DBG("AWSS3", "Acquire: dropping pooled conn from older config gen %llu (now %llu)", (unsigned long long)conn->configGen, (unsigned long long)gen); DiscardConnection(std::move(conn)); // pool lock released first continue; } const long long idleMs = std::chrono::duration_cast( now - conn->lastUsed).count(); if (idleMs > kMaxPooledIdleMs) { ANS_DBG("AWSS3", "Acquire: dropping pooled conn idle for %lldms (> %dms)", idleMs, kMaxPooledIdleMs); DiscardConnection(std::move(conn)); continue; } // Clear per-request state left behind by whoever used this // connection last. Multipart uploads set uploadId / partNumber // query params; a plain PUT that inherited them would target the // wrong resource. Content-Encoding is set by UploadTextData only, // and a JPEG that inherited "gzip" would be stored with metadata // saying it is compressed — browsers would then fail to display it. // NOTE: do NOT call ClearAllHeaders here — it wipes Chilkat's SigV4 // state (see commit fcc985e). Headers that every upload sets // explicitly (Content-Type, Expect) are overwritten by AddHeader // and need no clearing. conn->rest.ClearAllQueryParams(); conn->rest.RemoveHeader("Content-Encoding"); // Short leash on reuse: if this socket died while idle, fail fast // and let the retry run on a fresh connection. conn->rest.put_IdleTimeoutMs(_reusedIdleTimeoutMs.load()); return conn; } // Pool empty (or everything in it was stale) — create a new connection. std::lock_guard cfgLk(_configMutex); return CreateConnection(); } void ANSAWSS3::ReleaseConnection(std::unique_ptr conn) { if (!conn) return; // Mark as fresh and restore the full idle timeout; AcquireConnection // re-applies the short one when it hands the connection out again. conn->lastUsed = std::chrono::steady_clock::now(); conn->rest.put_IdleTimeoutMs(_idleTimeoutMs.load()); std::lock_guard lk(_poolMutex); _pool.push_back(std::move(conn)); } // Close a suspect connection instead of pooling it. Disconnect is given a // small budget only — the point is to release the socket, not to wait on a // peer that has already stopped answering. void ANSAWSS3::DiscardConnection(std::unique_ptr conn) { if (!conn) return; conn->rest.Disconnect(100); ANS_DBG("AWSS3", "Discarded connection (not returned to pool)"); // unique_ptr destructor tears the CkRest down here. } // Pre-populate the pool with `count` ready-to-use connections. Each // connection performs a TLS handshake; doing `count` of them in parallel // (rather than serially as AcquireConnection would on a cold pool) cuts // the first-upload latency for workloads that fan out right after // Connect() / SetAuthentication(). int ANSAWSS3::PrewarmConnectionPool(int count) { if (count <= 0) return 0; if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::PrewarmConnectionPool", "Not ready: call Connect() and SetAuthentication() first", __FILE__, __LINE__); return 0; } // Hold _configMutex once across the whole prewarm. It blocks config // writes for the duration, but does NOT block individual worker // threads since CreateConnection() itself does not re-acquire it. std::lock_guard cfgLk(_configMutex); std::vector> built(count); std::vector workers; workers.reserve(count); for (int i = 0; i < count; ++i) { workers.emplace_back([this, i, &built]() { try { built[i] = CreateConnection(); } catch (...) { built[i] = nullptr; } }); } for (auto& t : workers) t.join(); int added = 0; { std::lock_guard lk(_poolMutex); for (auto& conn : built) { if (conn) { _pool.push_back(std::move(conn)); ++added; } } } _logger.LogDebug("ANSAWSS3::PrewarmConnectionPool", "Added " + std::to_string(added) + "/" + std::to_string(count) + " connections to pool", __FILE__, __LINE__); return added; } // Update the per-connection request timeouts. New values are applied // immediately to all pooled connections (IdleTimeoutMs is picked up on // the next request; ConnectTimeoutMs is harmless on already-connected // CkRest instances but takes effect if they ever reconnect) and baked // into the defaults used by future CreateConnection() calls. bool ANSAWSS3::SetTimeouts(int connectMs, int idleMs) { if (connectMs < 1000 || idleMs < 1000) { _logger.LogError("ANSAWSS3::SetTimeouts", "Timeout values must be >= 1000 ms (got connect=" + std::to_string(connectMs) + ", idle=" + std::to_string(idleMs) + ")", __FILE__, __LINE__); return false; } _connectTimeoutMs.store(connectMs); _idleTimeoutMs.store(idleMs); // Reuse timeout scales with the configured idle timeout (a quarter of // it, clamped to a sane floor) so a deployment on a slow link that // raises idleMs does not end up with an over-aggressive leash on // pooled sockets. // Never longer than the configured idle timeout itself (idleMs can be // as low as 1000), and never so short that a healthy but busy server // gets cut off. const int reusedMs = (std::min)(idleMs, (std::max)(2000, idleMs / 4)); _reusedIdleTimeoutMs.store(reusedMs); // Propagate to existing pooled connections so already-warm workers // start using the new values on their next request. { std::lock_guard lk(_poolMutex); for (auto& conn : _pool) { if (conn) { conn->rest.put_ConnectTimeoutMs(connectMs); conn->rest.put_IdleTimeoutMs(idleMs); } } } _logger.LogDebug("ANSAWSS3::SetTimeouts", "connect=" + std::to_string(connectMs) + "ms, idle=" + std::to_string(idleMs) + "ms, idle-on-reuse=" + std::to_string(reusedMs) + "ms", __FILE__, __LINE__); return true; } // Shared precondition check for every upload entry point. The distinction // matters to the caller: a licence problem needs an operator, whereas "not // connected" clears itself once the background RetryLoop gets through — so // the two report different status codes. bool ANSAWSS3::CheckUploadPreconditions(const std::string& opName) { if (!_isLicenseValid || !_isUnlockCodeValid) { _logger.LogError(opName, !_isLicenseValid ? "Invalid license" : "Invalid unlock code", __FILE__, __LINE__); SetCallStatus(kCallPermanentError); return false; } if (!_bConnected) { const bool retrying = _retryInProgress.load(); _logger.LogError(opName, retrying ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); ANS_DBG("AWSS3", "%s: not connected (background retry %s) — caller should retry later", opName.c_str(), retrying ? "active" : "inactive"); SetCallStatus(kCallNotConnected); return false; } return true; } // See the declaration comment in ANSUtilities.h for why a failed Chilkat // request is not automatically a network problem. ANSAWSS3::AttemptResult ANSAWSS3::ClassifyFailedRequest( std::unique_ptr conn, std::string& lastError) { const int statusCode = conn ? conn->rest.get_ResponseStatusCode() : 0; lastError = SummarizeChilkatRequestError( conn ? conn->rest.lastErrorText() : nullptr, statusCode); // The server answered, and its answer will not change on a retry: // wrong bucket, no permission, bad key. 408 and 429 are the exceptions // — those explicitly ask the client to try again. if (statusCode >= 400 && statusCode <= 499 && statusCode != 408 && statusCode != 429) { ANS_DBG("AWSS3", "Request failed permanently: %s", lastError.c_str()); ReleaseConnection(std::move(conn)); // the socket itself is fine return AttemptResult::Permanent; } ANS_DBG("AWSS3", "Request failed transiently: %s", lastError.c_str()); DiscardConnection(std::move(conn)); // suspect socket — do not pool return AttemptResult::Transient; } // Drives a retry loop for a single upload operation. See the declaration // comment in ANSUtilities.h for the AttemptResult contract. // // Retry policy (shared across all upload entry points): // - kUploadMaxAttempts attempts total, AND kUploadDeadlineMs wall-clock // (whichever comes first — the attempt count alone does not bound how // long the caller blocks, since one attempt can sit on a timeout) // - kUploadRetryDelayMs between attempts (no delay after the last one) // - std::exception thrown inside attemptFn is treated as Transient // - Any other exception is treated as Permanent (bail immediately) bool ANSAWSS3::UploadWithRetry( const std::string& opName, const std::function& attemptFn) { std::string lastError; const auto started = std::chrono::steady_clock::now(); auto elapsedMs = [&started]() { return std::chrono::duration_cast( std::chrono::steady_clock::now() - started).count(); }; for (int attempt = 1; attempt <= kUploadMaxAttempts; ++attempt) { AttemptResult result = AttemptResult::Transient; try { result = attemptFn(lastError); } catch (const std::exception& e) { lastError = std::string("Exception: ") + e.what(); result = AttemptResult::Transient; } catch (...) { lastError = "Unknown exception"; result = AttemptResult::Permanent; } if (result == AttemptResult::Success) { SetCallStatus(kCallSuccess); return true; } if (result == AttemptResult::Permanent) { _logger.LogError(opName, "Permanent failure (no retry): " + lastError, __FILE__, __LINE__); SetCallStatus(kCallPermanentError); return false; } // Transient — retry if both budgets allow it. if (elapsedMs() >= kUploadDeadlineMs) { _logger.LogError(opName, "Upload gave up after " + std::to_string(attempt) + " attempt(s) / " + std::to_string(elapsedMs()) + "ms (deadline " + std::to_string(kUploadDeadlineMs) + "ms). Last error: " + lastError, __FILE__, __LINE__); ANS_DBG("AWSS3", "%s: deadline exceeded after %d attempt(s), %lldms", opName.c_str(), attempt, elapsedMs()); // Ran out of time on transient errors — the caller should retry. SetCallStatus(kCallTransientError); return false; } if (attempt < kUploadMaxAttempts) { _logger.LogDebug(opName, "Attempt " + std::to_string(attempt) + "/" + std::to_string(kUploadMaxAttempts) + " failed after " + std::to_string(elapsedMs()) + "ms: " + lastError + " — retrying in " + std::to_string(kUploadRetryDelayMs) + "ms", __FILE__, __LINE__); std::this_thread::sleep_for( std::chrono::milliseconds(kUploadRetryDelayMs)); } } _logger.LogError(opName, "Upload failed after " + std::to_string(kUploadMaxAttempts) + " attempts / " + std::to_string(elapsedMs()) + "ms. Last error: " + lastError, __FILE__, __LINE__); // Every attempt hit a transient error — worth retrying from the caller. SetCallStatus(kCallTransientError); return false; } void ANSAWSS3::CheckLicense() { // Note: caller (Initialize) already holds _configMutex try { // Check once globally std::call_once(ansawss3LicenseOnceFlag, [this]() { VerifyGlobalANSAWSS3License(_licenseKey); }); // Update this instance's local license flag _isLicenseValid = ansawss3LicenceValid; } catch (const std::exception& e) { this->_logger.LogFatal("ANSAWSS3::CheckLicense. Error:", e.what(), __FILE__, __LINE__); } } void ANSAWSS3::CheckUnlockCode() { // Note: caller (Initialize) already holds _configMutex try { CkGlobal glob; _unlockCode = "ANSDRC.CB1122026_MEQCIFwO1IFQCG0BhZwsXFO68QUU6mDB5uge4duOsqOJanEyAiAB67ahqnXin4SRy0vIegISgbFlpldmbuS5gbU21GYVqA==";// "ANSDRC.CB1082025_Ax6P3M7F8B3d"; _isUnlockCodeValid = glob.UnlockBundle(_unlockCode.c_str()); if (!_isUnlockCodeValid) { _logger.LogFatal("ANSAWSS3::CheckUnlockCode", glob.lastErrorText(), __FILE__, __LINE__); return; } int status = glob.get_UnlockStatus(); if (status != 2) { _logger.LogDebug("ANSAWSS3::CheckUnlockCode", "Unlocked in trial mode.", __FILE__, __LINE__); } } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::CheckUnlockCode", e.what(), __FILE__, __LINE__); } } bool ANSAWSS3::Initialize(const std::string& licenseKey) { std::lock_guard lock(_configMutex); try { _licenseKey = licenseKey; CheckLicense(); CheckUnlockCode(); return _isLicenseValid && _isUnlockCodeValid; } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::Initialize", e.what(), __FILE__, __LINE__); return false; } } bool ANSAWSS3::SetServerProxy(const std::string& proxyHost, int proxyPort,const std::string& proxyUsername,const std::string& proxyPassword) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid) { _logger.LogError("ANSAWSS3::SetServerProxy", !_isLicenseValid ? "Invalid license" : "Invalid unlock code", __FILE__, __LINE__); return false; } std::lock_guard lock(_configMutex); try { _proxyHost = proxyHost; _proxyPort = proxyPort; _proxyUsername = proxyUsername; _proxyPassword = proxyPassword; // Simplified proxy validation logic _bProxy = !proxyHost.empty() && proxyPort > 0; // Clear pool so new connections pick up proxy config. Bumping the // generation also invalidates any connection still in flight // elsewhere, which would otherwise be pooled after this clear. _configGen.fetch_add(1); { std::lock_guard lk(_poolMutex); _pool.clear(); } return true; } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::SetServerProxy", e.what(), __FILE__, __LINE__); return false; } } // Returns: 1 = connected, 0 = failed (bad auth/URL), 2 = no internet (background retry started) int ANSAWSS3::Connect(const std::string& baseDomain, const std::string& bucketRegion, const std::string& serviceName, int port, bool bTls, bool autoReconnect, bool &awsPath) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid) { _logger.LogError("ANSAWSS3::Connect", !_isLicenseValid ? "Invalid license" : "Invalid unlock code", __FILE__, __LINE__); return 0; } // Stop any existing background retry StopRetry(); std::lock_guard lock(_configMutex); try { // Store parameters — strip http:// or https:// from baseDomain { std::string domain = baseDomain.empty() ? "amazonaws.com" : baseDomain; if (domain.rfind("https://", 0) == 0) { domain = domain.substr(8); bTls = true; // caller said https, honour it } else if (domain.rfind("http://", 0) == 0) { domain = domain.substr(7); } // Remove trailing slash if any if (!domain.empty() && domain.back() == '/') domain.pop_back(); _baseDomain = domain; } _bucketRegion = bucketRegion.empty() ? "us-east-1" : bucketRegion; _serviceName = serviceName.empty() ? "s3" : serviceName; _fullAWSURL = _serviceName + "." + _bucketRegion + "." + _baseDomain; _port = port; _bTls = bTls; _bAutoReconnect = autoReconnect; // Clear pool so new connections use updated config (see the // generation note in SetServerProxy). _configGen.fetch_add(1); { std::lock_guard lk(_poolMutex); _pool.clear(); } // Check if internet is available (fast DNS check). Try the AWS // virtual-host form first; if that synthetic name doesn't exist // (MinIO, Ceph, custom S3-compatible servers), fall back to the // raw _baseDomain — that is the actual host the user gave us. { CkSocket dnsCheck; CkString dnsResult; bool resolved = dnsCheck.DnsLookup(_fullAWSURL.c_str(), 3000, dnsResult); if (!resolved && _fullAWSURL != _baseDomain) { CkSocket dnsCheck2; CkString dnsResult2; resolved = dnsCheck2.DnsLookup(_baseDomain.c_str(), 3000, dnsResult2); if (resolved) { ANS_DBG("AWSS3", "DNS for '%s' failed but '%s' resolved — proceeding with path-style/MinIO fallback", _fullAWSURL.c_str(), _baseDomain.c_str()); } } if (!resolved) { // No internet — start background retry, return immediately _logger.LogDebug("ANSAWSS3::Connect", "No internet available, starting background retry for '" + _fullAWSURL + "' / '" + _baseDomain + "'...", __FILE__, __LINE__); _stopRetry = false; _retryInProgress = true; _retryThread = std::thread(&ANSAWSS3::RetryLoop, this); return 2; // no internet, retrying in background } } // Internet is available — try real connection if (TryConnect(awsPath)) { return 1; // connected successfully } // Internet available but connection failed (bad auth, wrong URL, etc.) return 0; } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::Connect", e.what(), __FILE__, __LINE__); return 0; } } bool ANSAWSS3::SetAuthentication(const std::string& accessKey,const std::string& secretKey) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid) { _logger.LogError("ANSAWSS3::SetAuthentication", !_isLicenseValid ? "Invalid license" : "Invalid unlock code", __FILE__, __LINE__); return false; } // Write credentials + clear the pool under _configMutex. The lock is // released before the auto-prewarm below so PrewarmConnectionPool // (which also takes _configMutex) doesn't deadlock. { std::lock_guard lock(_configMutex); try { _accessKey = accessKey; _secretKey = secretKey; _authReady = true; // Clear pool so new connections pick up new credentials. The // generation bump matters most here: the background RetryLoop // can complete a connect at any moment, and a connection it // built before _authReady was set carries no SigV4 state — if // it landed in the pool after this clear, every upload using it // would come back 403 (a permanent failure, so not retried). _configGen.fetch_add(1); std::lock_guard lk(_poolMutex); _pool.clear(); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::SetAuthentication", e.what(), __FILE__, __LINE__); return false; } } // Auto-prewarm the connection pool so the first N concurrent uploads // don't serialize on TLS handshake. No-op if not connected yet. static constexpr int kDefaultPrewarmCount = 8; PrewarmConnectionPool(kDefaultPrewarmCount); return true; } std::vector ANSAWSS3::ListBuckets() { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::ListBuckets", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return {}; } try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("ListBuckets", "Failed to acquire S3 connection", __FILE__, __LINE__); return {}; } // Make request and capture response conn->rest.put_Host(_fullAWSURL.c_str()); CkStringBuilder sbResponse; if (!conn->rest.FullRequestNoBodySb("GET", "/", sbResponse)) { _logger.LogError("ANSAWSS3::ListBuckets - Request failed", conn->rest.lastErrorText(), __FILE__, __LINE__); DiscardConnection(std::move(conn)); // transport failed — do not pool a suspect socket return {}; } // Check HTTP status code int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode != 200) { _logger.LogError("ANSAWSS3::ListBuckets - HTTP " + std::to_string(statusCode), sbResponse.getAsString(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return {}; } // Parse XML response CkXml xml; if (!xml.LoadSb(sbResponse, true)) { _logger.LogError("ANSAWSS3::ListBuckets - XML parse error", xml.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return {}; } // Extract bucket names std::vector bucketList; int bucketCount = xml.NumChildrenHavingTag("Buckets|Bucket"); if (bucketCount > 0) { bucketList.reserve(bucketCount); for (int i = 0; i < bucketCount; ++i) { xml.put_I(i); const char* name = xml.getChildContent("Buckets|Bucket[i]|Name"); // Validate name before adding if (name != nullptr && name[0] != '\0') { bucketList.emplace_back(name); } } } ReleaseConnection(std::move(conn)); return bucketList; } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::ListBuckets", e.what(), __FILE__, __LINE__); return {}; } } std::vector ANSAWSS3::ListBucketObjects(const std::string& bucketName) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::ListBucketObjects", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return {}; } if (bucketName.empty()) { _logger.LogError("ANSAWSS3::ListBucketObjects", "Bucket name is empty", __FILE__, __LINE__); return {}; } std::vector objectList; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("ListBucketObjects", "Failed to acquire S3 connection", __FILE__, __LINE__); return {}; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } std::string marker = ""; bool isTruncated = true; int pageCount = 0; // Loop to handle pagination while (isTruncated) { pageCount++; // Build the request path with marker if available std::string basePath = _bAwsPath ? "/" : ("/" + bucketName + "/"); std::string requestPath = basePath; if (!marker.empty()) { requestPath = basePath + "?marker=" + marker; } CkStringBuilder sbResponse; bool success = conn->rest.FullRequestNoBodySb("GET", requestPath.c_str(), sbResponse); if (!success) { _logger.LogError("ANSAWSS3::ListBucketObjects - Request failed on page " + std::to_string(pageCount), conn->rest.lastErrorText(), __FILE__, __LINE__); DiscardConnection(std::move(conn)); // transport failed — do not pool a suspect socket return objectList; // Return what we have so far } int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode != 200) { std::string errorMsg = "HTTP " + std::to_string(statusCode); std::string response = sbResponse.getAsString(); if (!response.empty()) { errorMsg += " - " + response; } _logger.LogError("ANSAWSS3::ListBucketObjects - Page " + std::to_string(pageCount), errorMsg, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return objectList; // Return what we have so far } // Parse XML response CkXml xml; bool loadSuccess = xml.LoadSb(sbResponse, true); if (!loadSuccess) { _logger.LogError("ANSAWSS3::ListBucketObjects", "Failed to parse XML response on page " + std::to_string(pageCount) + ": " + std::string(xml.lastErrorText()), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return objectList; // Return what we have so far } // Check if results are truncated (more pages to fetch) const char* truncatedStr = xml.getChildContent("IsTruncated"); isTruncated = (truncatedStr != nullptr && std::string(truncatedStr) == "true"); // Get the marker for next page if (isTruncated) { // First try to get NextMarker const char* nextMarker = xml.getChildContent("NextMarker"); if (nextMarker != nullptr && nextMarker[0] != '\0') { marker = std::string(nextMarker); } else { // If NextMarker is not present, use the last Key as marker int count = xml.NumChildrenHavingTag("Contents"); if (count > 0) { xml.put_I(count - 1); // Last element const char* lastKey = xml.getChildContent("Contents[i]|Key"); if (lastKey != nullptr && lastKey[0] != '\0') { marker = std::string(lastKey); } else { isTruncated = false; } } else { isTruncated = false; } } } // Iterate through all Contents elements in this page int count = xml.NumChildrenHavingTag("Contents"); for (int i = 0; i < count; i++) { xml.put_I(i); const char* key = xml.getChildContent("Contents[i]|Key"); if (key != nullptr && key[0] != '\0') { objectList.emplace_back(key); } } _logger.LogDebug("ANSAWSS3::ListBucketObjects", "Page " + std::to_string(pageCount) + ": Retrieved " + std::to_string(count) + " objects. Total so far: " + std::to_string(objectList.size()), __FILE__, __LINE__); } _logger.LogDebug("ANSAWSS3::ListBucketObjects", "Successfully listed " + std::to_string(objectList.size()) + " objects in bucket: " + bucketName + " (" + std::to_string(pageCount) + " pages)", __FILE__, __LINE__); ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::ListBucketObjects", std::string("Exception: ") + e.what(), __FILE__, __LINE__); objectList.clear(); // Ensure we return empty list on exception } return objectList; } std::vector ANSAWSS3::ListBucketObjectsWithPrefix(const std::string& bucketName, const std::string& prefix) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::ListBucketObjectsWithPrefix", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return {}; } if (bucketName.empty()) { _logger.LogError("ANSAWSS3::ListBucketObjectsWithPrefix", "Bucket name is empty", __FILE__, __LINE__); return {}; } // prefix can be empty to list all objects std::vector objectList; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("ListBucketObjectsWithPrefix", "Failed to acquire S3 connection", __FILE__, __LINE__); return {}; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } std::string marker = ""; bool isTruncated = true; int pageCount = 0; // Loop to handle pagination while (isTruncated) { pageCount++; // Build the request path with prefix and marker std::string basePath = _bAwsPath ? "/" : ("/" + bucketName + "/"); std::string requestPath = basePath; bool hasParams = false; // Add prefix parameter if provided if (!prefix.empty()) { requestPath += "?prefix=" + prefix; hasParams = true; } // Add marker parameter if available if (!marker.empty()) { if (hasParams) { requestPath += "&marker=" + marker; } else { requestPath += "?marker=" + marker; hasParams = true; } } CkStringBuilder sbResponse; bool success = conn->rest.FullRequestNoBodySb("GET", requestPath.c_str(), sbResponse); if (!success) { _logger.LogError("ANSAWSS3::ListBucketObjectsWithPrefix - Request failed on page " + std::to_string(pageCount), conn->rest.lastErrorText(), __FILE__, __LINE__); DiscardConnection(std::move(conn)); // transport failed — do not pool a suspect socket return objectList; // Return what we have so far } int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode != 200) { std::string errorMsg = "HTTP " + std::to_string(statusCode); std::string response = sbResponse.getAsString(); if (!response.empty()) { errorMsg += " - " + response; } _logger.LogError("ANSAWSS3::ListBucketObjectsWithPrefix - Page " + std::to_string(pageCount), errorMsg, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return objectList; // Return what we have so far } // Parse XML response CkXml xml; bool loadSuccess = xml.LoadSb(sbResponse, true); if (!loadSuccess) { _logger.LogError("ANSAWSS3::ListBucketObjectsWithPrefix", "Failed to parse XML response on page " + std::to_string(pageCount) + ": " + std::string(xml.lastErrorText()), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return objectList; // Return what we have so far } // Check if results are truncated (more pages to fetch) const char* truncatedStr = xml.getChildContent("IsTruncated"); isTruncated = (truncatedStr != nullptr && std::string(truncatedStr) == "true"); // Get the marker for next page if (isTruncated) { // First try to get NextMarker const char* nextMarker = xml.getChildContent("NextMarker"); if (nextMarker != nullptr && nextMarker[0] != '\0') { marker = std::string(nextMarker); } else { // If NextMarker is not present, use the last Key as marker int count = xml.NumChildrenHavingTag("Contents"); if (count > 0) { xml.put_I(count - 1); // Last element const char* lastKey = xml.getChildContent("Contents[i]|Key"); if (lastKey != nullptr && lastKey[0] != '\0') { marker = std::string(lastKey); } else { isTruncated = false; } } else { isTruncated = false; } } } // Iterate through all Contents elements in this page int count = xml.NumChildrenHavingTag("Contents"); for (int i = 0; i < count; i++) { xml.put_I(i); const char* key = xml.getChildContent("Contents[i]|Key"); if (key != nullptr && key[0] != '\0') { objectList.emplace_back(key); } } std::string prefixInfo = prefix.empty() ? "all objects" : "prefix '" + prefix + "'"; _logger.LogDebug("ANSAWSS3::ListBucketObjectsWithPrefix", "Page " + std::to_string(pageCount) + ": Retrieved " + std::to_string(count) + " objects for " + prefixInfo + ". Total so far: " + std::to_string(objectList.size()), __FILE__, __LINE__); } std::string prefixInfo = prefix.empty() ? "" : " with prefix '" + prefix + "'"; _logger.LogDebug("ANSAWSS3::ListBucketObjectsWithPrefix", "Successfully listed " + std::to_string(objectList.size()) + " objects in bucket: " + bucketName + prefixInfo + " (" + std::to_string(pageCount) + " pages)", __FILE__, __LINE__); ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::ListBucketObjectsWithPrefix", std::string("Exception: ") + e.what(), __FILE__, __LINE__); objectList.clear(); // Ensure we return empty list on exception } return objectList; } bool ANSAWSS3::CreateBucket(const std::string& bucketName) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::CreateBucket", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } // Validate input parameters if (bucketName.empty()) { _logger.LogError("ANSAWSS3::CreateBucket", "Bucket name cannot be empty", __FILE__, __LINE__); return false; } bool createSuccess = false; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("CreateBucket", "Failed to acquire S3 connection", __FILE__, __LINE__); return false; } // Build global endpoint std::string globalEndpoint = _serviceName + "." + _baseDomain; CkStringBuilder sbBucketRegion; sbBucketRegion.Append(_bucketRegion.c_str()); // We only need to specify the LocationConstraint if the bucket's region is NOT us-east-1 CkXml xml; if (!sbBucketRegion.ContentsEqual("us-east-1", true)) { xml.put_Tag("CreateBucketConfiguration"); xml.AddAttribute("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/"); xml.UpdateChildContent("LocationConstraint", _bucketRegion.c_str()); } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + globalEndpoint).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // Make the call to create the bucket std::string createPath = _bAwsPath ? "/" : ("/" + bucketName); const char* responseStr = nullptr; if (!sbBucketRegion.ContentsEqual("us-east-1", true)) { responseStr = conn->rest.fullRequestString("PUT", createPath.c_str(), xml.getXml()); } else { // If the bucket is to be created in the us-east-1 region (the default region) // just send a PUT with no body responseStr = conn->rest.fullRequestNoBody("PUT", createPath.c_str()); } if (conn->rest.get_LastMethodSuccess() != true) { _logger.LogError("ANSAWSS3::CreateBucket - Request failed", conn->rest.lastErrorText(), __FILE__, __LINE__); DiscardConnection(std::move(conn)); // transport failed — do not pool a suspect socket return false; } int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode == 200) { _logger.LogDebug("ANSAWSS3::CreateBucket", "Successfully created bucket: " + bucketName + " in region: " + _bucketRegion, __FILE__, __LINE__); createSuccess = true; } else { std::string errorMsg = "HTTP " + std::to_string(statusCode); if (responseStr != nullptr && responseStr[0] != '\0') { CkXml responseXml; if (responseXml.LoadXml(responseStr)) { const char* errorCode = responseXml.getChildContent("Code"); const char* errorMessage = responseXml.getChildContent("Message"); if (errorCode != nullptr) { errorMsg += " - " + std::string(errorCode); } if (errorMessage != nullptr) { errorMsg += ": " + std::string(errorMessage); } } else { errorMsg += "\n" + std::string(responseStr); } } if (statusCode == 409) { _logger.LogError("ANSAWSS3::CreateBucket", errorMsg, __FILE__, __LINE__); } else { _logger.LogError("ANSAWSS3::CreateBucket", errorMsg, __FILE__, __LINE__); } } ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::CreateBucket", std::string("Exception: ") + e.what(), __FILE__, __LINE__); } return createSuccess; } bool ANSAWSS3::DeleteBucket(const std::string& bucketName) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::DeleteBucket", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty()) { _logger.LogError("ANSAWSS3::DeleteBucket", "Bucket name cannot be empty", __FILE__, __LINE__); return false; } bool deleteSuccess = false; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("DeleteBucket", "Failed to acquire S3 connection", __FILE__, __LINE__); return false; } // Configure AWS authentication conn->rest.ClearAuth(); conn->authAws.put_AccessKey(_accessKey.c_str()); conn->authAws.put_SecretKey(_secretKey.c_str()); conn->authAws.put_ServiceName(_serviceName.c_str()); bool authSuccess = conn->rest.SetAuthAws(conn->authAws); if (!authSuccess) { _logger.LogError("ANSAWSS3::DeleteBucket", "Failed to set AWS authentication: " + std::string(conn->rest.lastErrorText()), __FILE__, __LINE__); DiscardConnection(std::move(conn)); // transport failed — do not pool a suspect socket return false; } // Set bucket-specific endpoint std::string globalEndpoint = _serviceName + "." + _baseDomain; if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + globalEndpoint).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } std::string deletePath = _bAwsPath ? "/" : ("/" + bucketName); CkStringBuilder sbResponse; bool success = conn->rest.FullRequestNoBodySb("DELETE", deletePath.c_str(), sbResponse); if (!success) { _logger.LogError("ANSAWSS3::DeleteBucket", "Failed to send DELETE request: " + std::string(conn->rest.lastErrorText()), __FILE__, __LINE__); } else { int statusCode = conn->rest.get_ResponseStatusCode(); // S3 returns 204 (No Content) for successful bucket deletion if (statusCode == 204) { _logger.LogDebug("ANSAWSS3::DeleteBucket", "Successfully deleted bucket: " + bucketName, __FILE__, __LINE__); deleteSuccess = true; } else if (statusCode == 404) { _logger.LogWarn("ANSAWSS3::DeleteBucket", "Bucket not found (already deleted?): " + bucketName, __FILE__, __LINE__); } else if (statusCode == 409) { // Bucket not empty std::string errorMsg = "HTTP 409 - Bucket is not empty and cannot be deleted"; std::string response = sbResponse.getAsString(); if (!response.empty()) { CkXml responseXml; if (responseXml.LoadXml(response.c_str())) { const char* errorCode = responseXml.getChildContent("Code"); const char* errorMessage = responseXml.getChildContent("Message"); if (errorCode != nullptr) { errorMsg += " - " + std::string(errorCode); } if (errorMessage != nullptr) { errorMsg += ": " + std::string(errorMessage); } } } _logger.LogError("ANSAWSS3::DeleteBucket", errorMsg, __FILE__, __LINE__); } else { std::string errorMsg = "HTTP " + std::to_string(statusCode); std::string response = sbResponse.getAsString(); if (!response.empty()) { errorMsg += " - " + response; } _logger.LogError("ANSAWSS3::DeleteBucket", errorMsg, __FILE__, __LINE__); } } ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::DeleteBucket", std::string("Exception: ") + e.what(), __FILE__, __LINE__); } return deleteSuccess; } bool ANSAWSS3::CreateFolder(const std::string& bucketName, const std::string& prefix) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::CreateFolder", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || prefix.empty()) { _logger.LogError("ANSAWSS3::CreateFolder", "Bucket name or folder path is empty", __FILE__, __LINE__); return false; } bool createSuccess = false; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("CreateFolder", "Failed to acquire S3 connection", __FILE__, __LINE__); return false; } // Ensure folder path ends with "/" std::string normalizedPath = prefix; if (normalizedPath.back() != '/') { normalizedPath += '/'; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // Object path is the folder name with trailing slash std::string objectPath = _bAwsPath ? ("/" + normalizedPath) : ("/" + bucketName + "/" + normalizedPath); // Create a zero-byte object to represent the folder // Use PUT with empty body const char* responseStr = conn->rest.fullRequestNoBody("PUT", objectPath.c_str()); if (conn->rest.get_LastMethodSuccess() != true) { _logger.LogError("ANSAWSS3::CreateFolder - Request failed", conn->rest.lastErrorText(), __FILE__, __LINE__); DiscardConnection(std::move(conn)); // transport failed — do not pool a suspect socket return false; } int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode == 200) { _logger.LogDebug("ANSAWSS3::CreateFolder", "Successfully created folder: " + normalizedPath + " in bucket: " + bucketName, __FILE__, __LINE__); createSuccess = true; } else { std::string errorMsg = "HTTP " + std::to_string(statusCode); if (responseStr != nullptr && responseStr[0] != '\0') { CkXml responseXml; if (responseXml.LoadXml(responseStr)) { const char* errorCode = responseXml.getChildContent("Code"); const char* errorMessage = responseXml.getChildContent("Message"); if (errorCode != nullptr) { errorMsg += " - " + std::string(errorCode); } if (errorMessage != nullptr) { errorMsg += ": " + std::string(errorMessage); } } else { errorMsg += " - " + std::string(responseStr); } } _logger.LogError("ANSAWSS3::CreateFolder", errorMsg, __FILE__, __LINE__); } ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::CreateFolder", std::string("Exception: ") + e.what(), __FILE__, __LINE__); } return createSuccess; } bool ANSAWSS3::DeleteFolder(const std::string& bucketName, const std::string& prefix) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::DeleteFolder", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || prefix.empty()) { _logger.LogError("ANSAWSS3::DeleteFolder", "Bucket name or folder path is empty", __FILE__, __LINE__); return false; } bool deleteSuccess = false; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("DeleteFolder", "Failed to acquire S3 connection", __FILE__, __LINE__); return false; } // Ensure folder path ends with "/" std::string normalizedPath = prefix; if (normalizedPath.back() != '/') { normalizedPath += '/'; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } std::string objectPath = _bAwsPath ? ("/" + normalizedPath) : ("/" + bucketName + "/" + normalizedPath); CkStringBuilder sbResponse; bool success = conn->rest.FullRequestNoBodySb("DELETE", objectPath.c_str(), sbResponse); if (!success) { _logger.LogError("ANSAWSS3::DeleteFolder", "Failed to send DELETE request: " + std::string(conn->rest.lastErrorText()), __FILE__, __LINE__); } else { int statusCode = conn->rest.get_ResponseStatusCode(); // S3 returns 204 (No Content) for successful deletion if (statusCode == 204 || statusCode == 200) { _logger.LogDebug("ANSAWSS3::DeleteFolder", "Successfully deleted folder marker: " + normalizedPath + " from bucket: " + bucketName, __FILE__, __LINE__); deleteSuccess = true; } else if (statusCode == 404) { _logger.LogWarn("ANSAWSS3::DeleteFolder", "Folder marker not found (already deleted?): " + normalizedPath, __FILE__, __LINE__); } else { std::string errorMsg = "HTTP " + std::to_string(statusCode); std::string response = sbResponse.getAsString(); if (!response.empty()) { errorMsg += " - " + response; } _logger.LogError("ANSAWSS3::DeleteFolder", errorMsg, __FILE__, __LINE__); } } ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::DeleteFolder", std::string("Exception: ") + e.what(), __FILE__, __LINE__); } return deleteSuccess; } std::string ANSAWSS3::GetBucketRegion(const std::string& bucketName) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::GetBucketRegion", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return ""; } if (bucketName.empty()) { _logger.LogError("ANSAWSS3::GetBucketRegion", "Bucket name is empty", __FILE__, __LINE__); return ""; } std::string region = ""; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("GetBucketRegion", "Failed to acquire S3 connection", __FILE__, __LINE__); return ""; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // Send the GET request to query the bucket location std::string locationPath = _bAwsPath ? "/?location" : ("/" + bucketName + "?location"); const char* strResult = conn->rest.fullRequestNoBody("GET", locationPath.c_str()); if (conn->rest.get_LastMethodSuccess() != true) { _logger.LogError("ANSAWSS3::GetBucketRegion - Request failed", conn->rest.lastErrorText(), __FILE__, __LINE__); DiscardConnection(std::move(conn)); // transport failed — do not pool a suspect socket return ""; } int responseStatusCode = conn->rest.get_ResponseStatusCode(); if (responseStatusCode != 200) { std::string errorMsg = "HTTP " + std::to_string(responseStatusCode); if (strResult != nullptr && strResult[0] != '\0') { errorMsg += " - " + std::string(strResult); } _logger.LogError("ANSAWSS3::GetBucketRegion", errorMsg, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return ""; } // Parse XML response CkXml xml; bool success = xml.LoadXml(strResult); if (!success) { _logger.LogError("ANSAWSS3::GetBucketRegion - Failed to parse XML", xml.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return ""; } // Get the region from XML content const char* regionContent = xml.content(); if (regionContent != nullptr && regionContent[0] != '\0') { region = std::string(regionContent); // AWS returns empty string for us-east-1 (classic region) // Some implementations may return "null" or empty if (region.empty() || region == "null") { region = "us-east-1"; _logger.LogDebug("ANSAWSS3::GetBucketRegion", "Bucket '" + bucketName + "' is in default region: us-east-1", __FILE__, __LINE__); } else { _logger.LogDebug("ANSAWSS3::GetBucketRegion", "Bucket '" + bucketName + "' is in region: " + region, __FILE__, __LINE__); } } else { // Empty response typically means us-east-1 region = "us-east-1"; _logger.LogDebug("ANSAWSS3::GetBucketRegion", "Empty response - assuming default region: us-east-1", __FILE__, __LINE__); } ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::GetBucketRegion", std::string("Exception: ") + e.what(), __FILE__, __LINE__); } return region; } // Uploads text data from a file to the specified S3 bucket bool ANSAWSS3::UploadTextData(const std::string& bucketName, const std::string& textFilePath, std::string& uploadedFilePath) { const std::string kOp = "ANSAWSS3::UploadTextData"; // _logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__); // Early validation (permanent — do NOT retry) if (!CheckUploadPreconditions(kOp)) return false; if (bucketName.empty() || textFilePath.empty()) { _logger.LogError(kOp, "Bucket name or file path is empty", __FILE__, __LINE__); SetCallStatus(kCallPermanentError); // caller bug — retrying cannot help return false; } // Filename and content type are attempt-independent. const std::string fileName = ExtractFileName(textFilePath); if (fileName.empty()) { _logger.LogError(kOp, "Failed to extract filename from path: " + textFilePath, __FILE__, __LINE__); SetCallStatus(kCallPermanentError); // bad input path — retrying cannot help return false; } const std::string contentType = GetContentType(textFilePath); const std::string objectPath = _bAwsPath ? ("/" + fileName) : ("/" + bucketName + "/" + fileName); return UploadWithRetry(kOp, [&](std::string& lastError) -> AttemptResult { auto conn = AcquireConnection(); if (!conn) { lastError = "Failed to acquire S3 connection"; return AttemptResult::Transient; } if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // Read the file on each attempt (cheap; keeps the body fresh). CkFileAccess fac; const char* fileContents = fac.readEntireTextFile(textFilePath.c_str(), "utf-8"); if (!fac.get_LastMethodSuccess()) { // File read failure is permanent (disk/IO, not network). lastError = std::string("Failed to read file: ") + fac.lastErrorText(); ReleaseConnection(std::move(conn)); return AttemptResult::Permanent; } if (fileContents == nullptr || fileContents[0] == '\0') { lastError = "File is empty: " + textFilePath; ReleaseConnection(std::move(conn)); return AttemptResult::Permanent; } conn->rest.AddHeader("Content-Type", contentType.c_str()); conn->rest.AddHeader("Content-Encoding", "gzip"); conn->rest.AddHeader("Expect", "100-continue"); const char* responseBodyStr = conn->rest.fullRequestString("PUT", objectPath.c_str(), fileContents); if (!conn->rest.get_LastMethodSuccess()) { return ClassifyFailedRequest(std::move(conn), lastError); } int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode == 200) { std::string scheme = _bTls ? "https://" : "http://"; uploadedFilePath = scheme + (_bAwsPath ? _fullAWSURL : _baseDomain) + "/" + bucketName + "/" + fileName; _logger.LogDebug(kOp, "Successfully uploaded: " + fileName + " | URL: " + uploadedFilePath, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return AttemptResult::Success; } lastError = "HTTP " + std::to_string(statusCode); if (responseBodyStr != nullptr && responseBodyStr[0] != '\0') { lastError += " - " + std::string(responseBodyStr); } ReleaseConnection(std::move(conn)); bool retryable = (statusCode >= 500 && statusCode <= 599) || statusCode == 408 || statusCode == 429; return retryable ? AttemptResult::Transient : AttemptResult::Permanent; }); } bool ANSAWSS3::UploadBinaryData(const std::string& bucketName, const std::string& dataFilePath, std::string& uploadedFilePath) { const std::string kOp = "ANSAWSS3::UploadBinaryData"; // _logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__); // Early validation (permanent — do NOT retry) if (!CheckUploadPreconditions(kOp)) return false; if (bucketName.empty() || dataFilePath.empty()) { _logger.LogError(kOp, "Bucket name or file path is empty", __FILE__, __LINE__); SetCallStatus(kCallPermanentError); // caller bug — retrying cannot help return false; } const std::string fileName = ExtractFileName(dataFilePath); if (fileName.empty()) { _logger.LogError(kOp, "Failed to extract filename from path: " + dataFilePath, __FILE__, __LINE__); SetCallStatus(kCallPermanentError); // bad input path — retrying cannot help return false; } const std::string contentType = GetContentType(dataFilePath); const std::string objectPath = _bAwsPath ? ("/" + fileName) : ("/" + bucketName + "/" + fileName); return UploadWithRetry(kOp, [&](std::string& lastError) -> AttemptResult { auto conn = AcquireConnection(); if (!conn) { lastError = "Failed to acquire S3 connection"; return AttemptResult::Transient; } if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } CkBinData binData; if (!binData.LoadFile(dataFilePath.c_str())) { lastError = std::string("Failed to load file: ") + binData.lastErrorText(); ReleaseConnection(std::move(conn)); return AttemptResult::Permanent; // disk/IO, not network } conn->rest.AddHeader("Content-Type", contentType.c_str()); conn->rest.AddHeader("Expect", "100-continue"); CkStringBuilder sbResponse; if (!conn->rest.FullRequestBd("PUT", objectPath.c_str(), binData, sbResponse)) { return ClassifyFailedRequest(std::move(conn), lastError); } int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode == 200) { std::string scheme = _bTls ? "https://" : "http://"; uploadedFilePath = scheme + (_bAwsPath ? _fullAWSURL : _baseDomain) + "/" + bucketName + "/" + fileName; _logger.LogDebug(kOp, "Successfully uploaded: " + fileName + " | URL: " + uploadedFilePath, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return AttemptResult::Success; } lastError = "HTTP " + std::to_string(statusCode); std::string body = sbResponse.getAsString(); if (!body.empty()) lastError += " - " + body; ReleaseConnection(std::move(conn)); bool retryable = (statusCode >= 500 && statusCode <= 599) || statusCode == 408 || statusCode == 429; return retryable ? AttemptResult::Transient : AttemptResult::Permanent; }); } bool ANSAWSS3::UploadPrefixBinaryData(const std::string& bucketName, const std::string& prefix, const std::string& dataFilePath, const std::string& objectName, std::string& uploadedFilePath) { const std::string kOp = "ANSAWSS3::UploadPrefixBinaryData"; //_logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__); // Early validation (permanent — do NOT retry) if (!CheckUploadPreconditions(kOp)) return false; if (bucketName.empty() || dataFilePath.empty()) { _logger.LogError(kOp, "Bucket name or file path is empty", __FILE__, __LINE__); SetCallStatus(kCallPermanentError); // caller bug — retrying cannot help return false; } // Resolve filename (from objectName, fall back to path). std::string fileName = objectName; if (fileName.empty()) { fileName = ExtractFileName(dataFilePath); if (fileName.empty()) { _logger.LogError(kOp, "Failed to extract filename from path: " + dataFilePath, __FILE__, __LINE__); SetCallStatus(kCallPermanentError); // bad input path — retrying cannot help return false; } } // Build once: the object key and the HTTP request path. Keeping // them separate avoids the path-style bucket-duplication bug in // the returned URL. std::string objectKey; // "/" — no leading '/' if (!prefix.empty()) { std::string normalizedPrefix = prefix; if (!normalizedPrefix.empty() && normalizedPrefix.front() == '/') { normalizedPrefix = normalizedPrefix.substr(1); } if (!normalizedPrefix.empty() && normalizedPrefix.back() != '/') { normalizedPrefix += '/'; } objectKey = normalizedPrefix; } objectKey += fileName; const std::string objectPath = _bAwsPath ? ("/" + objectKey) : ("/" + bucketName + "/" + objectKey); const std::string contentType = GetContentType(dataFilePath); return UploadWithRetry(kOp, [&](std::string& lastError) -> AttemptResult { auto conn = AcquireConnection(); if (!conn) { lastError = "Failed to acquire S3 connection"; return AttemptResult::Transient; } if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } CkBinData binData; if (!binData.LoadFile(dataFilePath.c_str())) { lastError = std::string("Failed to load file: ") + binData.lastErrorText(); ReleaseConnection(std::move(conn)); return AttemptResult::Permanent; } conn->rest.AddHeader("Content-Type", contentType.c_str()); conn->rest.AddHeader("Expect", "100-continue"); CkStringBuilder sbResponse; if (!conn->rest.FullRequestBd("PUT", objectPath.c_str(), binData, sbResponse)) { return ClassifyFailedRequest(std::move(conn), lastError); } int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode == 200) { // Build canonical path-style URL from parts — independent // of the addressing style we actually used. std::string scheme = _bTls ? "https://" : "http://"; uploadedFilePath = scheme + (_bAwsPath ? _fullAWSURL : _baseDomain) + "/" + bucketName + "/" + objectKey; _logger.LogDebug(kOp, "Successfully uploaded to: " + objectKey + " | URL: " + uploadedFilePath, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return AttemptResult::Success; } lastError = "HTTP " + std::to_string(statusCode); std::string body = sbResponse.getAsString(); if (!body.empty()) lastError += " - " + body; ReleaseConnection(std::move(conn)); bool retryable = (statusCode >= 500 && statusCode <= 599) || statusCode == 408 || statusCode == 429; return retryable ? AttemptResult::Transient : AttemptResult::Permanent; }); } bool ANSAWSS3::UploadFileStream(const std::string& bucketName, const std::string& dataFilePath, std::string& uploadedFilePath) { const std::string kOp = "ANSAWSS3::UploadFileStream"; //_logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__); // Early validation (permanent — do NOT retry) if (!CheckUploadPreconditions(kOp)) return false; if (bucketName.empty() || dataFilePath.empty()) { _logger.LogError(kOp, "Bucket name or file path is empty", __FILE__, __LINE__); SetCallStatus(kCallPermanentError); // caller bug — retrying cannot help return false; } const std::string fileName = ExtractFileName(dataFilePath); const std::string objectPath = _bAwsPath ? ("/" + fileName) : ("/" + bucketName + "/" + fileName); return UploadWithRetry(kOp, [&](std::string& lastError) -> AttemptResult { auto conn = AcquireConnection(); if (!conn) { lastError = "Failed to acquire S3 connection"; return AttemptResult::Transient; } if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // Fresh stream per attempt (it holds OS file handle state). CkStream fileStream; fileStream.put_SourceFile(dataFilePath.c_str()); const char* responseStr = conn->rest.fullRequestStream("PUT", objectPath.c_str(), fileStream); if (conn->rest.get_LastMethodSuccess() != true) { return ClassifyFailedRequest(std::move(conn), lastError); } int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode == 200) { std::string scheme = _bTls ? "https://" : "http://"; uploadedFilePath = scheme + (_bAwsPath ? _fullAWSURL : _baseDomain) + "/" + bucketName + "/" + fileName; _logger.LogDebug(kOp, "Successfully uploaded file: " + fileName + " | URL: " + uploadedFilePath, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return AttemptResult::Success; } lastError = "HTTP " + std::to_string(statusCode); if (responseStr != nullptr && responseStr[0] != '\0') { lastError += " - " + std::string(responseStr); } ReleaseConnection(std::move(conn)); bool retryable = (statusCode >= 500 && statusCode <= 599) || statusCode == 408 || statusCode == 429; return retryable ? AttemptResult::Transient : AttemptResult::Permanent; }); } bool ANSAWSS3::UploadMultipartData(const std::string& bucketName,const std::string& dataFilePath, std::string& uploadedFilePath, int partSize) { const std::string kOp = "ANSAWSS3::UploadMultipartData"; //_logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__); // Early validation (permanent — do NOT retry) if (!CheckUploadPreconditions(kOp)) return false; if (bucketName.empty() || dataFilePath.empty()) { _logger.LogError(kOp, "Bucket name or file path is empty", __FILE__, __LINE__); SetCallStatus(kCallPermanentError); // caller bug — retrying cannot help return false; } if (partSize < 5242880) { _logger.LogError(kOp, "Part size must be at least 5MB (5242880 bytes)", __FILE__, __LINE__); SetCallStatus(kCallPermanentError); // caller bug — retrying cannot help return false; } { CkFileAccess fac; if (!fac.FileExists(dataFilePath.c_str())) { _logger.LogError(kOp, "File not found: " + dataFilePath, __FILE__, __LINE__); SetCallStatus(kCallPermanentError); // bad input path — retrying cannot help return false; } } const std::string fileName = ExtractFileName(dataFilePath); if (fileName.empty()) { _logger.LogError(kOp, "Failed to extract filename from path: " + dataFilePath, __FILE__, __LINE__); SetCallStatus(kCallPermanentError); // bad input path — retrying cannot help return false; } const std::string objectPath = _bAwsPath ? ("/" + fileName) : ("/" + bucketName + "/" + fileName); const std::string fileFolderPath = dataFilePath.substr(0, dataFilePath.find_last_of("/\\")); // Fix 2: include a per-call unique token in the scratch-file name so // two parallel multipart uploads of the same local file don't share // the same partsList XML (which would race on create/delete). const std::string partsListFilePath = fileFolderPath + "/partsList_" + fileName + "_" + MakeUniqueMultipartToken(this) + ".xml"; // Note on retry semantics for multipart: // Each attempt performs a fresh Initiate → Parts → Complete cycle. We // reset the on-disk partsList XML at the start of each attempt because // ETags returned by a prior attempt are bound to that attempt's // UploadId and are not valid for a new Initiate. Parts uploaded during // a failed attempt therefore become orphans on S3 until the service's // multipart-lifecycle rule cleans them up (default 7 days, configurable). return UploadWithRetry(kOp, [&](std::string& lastError) -> AttemptResult { CkFileAccess fac; // Ensure no stale parts list from a prior attempt. if (fac.FileExists(partsListFilePath.c_str())) { fac.FileDelete(partsListFilePath.c_str()); } auto conn = AcquireConnection(); if (!conn) { lastError = "Failed to acquire S3 connection"; return AttemptResult::Transient; } if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // ---- STEP 1: INITIATE ---- conn->rest.ClearAllQueryParams(); conn->rest.AddQueryParam("uploads", ""); const char* responseXml = conn->rest.fullRequestNoBody("POST", objectPath.c_str()); if (!conn->rest.get_LastMethodSuccess()) { { AttemptResult r = ClassifyFailedRequest(std::move(conn), lastError); lastError = "Initiate: " + lastError; return r; } } int initStatus = conn->rest.get_ResponseStatusCode(); if (initStatus != 200) { lastError = "Initiate HTTP " + std::to_string(initStatus); if (responseXml != nullptr && responseXml[0] != '\0') lastError += " - " + std::string(responseXml); ReleaseConnection(std::move(conn)); bool retryable = (initStatus >= 500 && initStatus <= 599) || initStatus == 408 || initStatus == 429; return retryable ? AttemptResult::Transient : AttemptResult::Permanent; } CkXml xmlInit; if (!xmlInit.LoadXml(responseXml)) { lastError = std::string("Initiate XML parse error: ") + xmlInit.lastErrorText(); ReleaseConnection(std::move(conn)); return AttemptResult::Transient; } const char* uploadId = xmlInit.getChildContent("UploadId"); if (uploadId == nullptr || uploadId[0] == '\0') { lastError = "UploadId not found in Initiate response"; ReleaseConnection(std::move(conn)); return AttemptResult::Transient; } _logger.LogDebug(kOp, "Multipart initiated. UploadId: " + std::string(uploadId), __FILE__, __LINE__); // ---- STEP 2: UPLOAD PARTS ---- fac.OpenForRead(dataFilePath.c_str()); int numParts = fac.GetNumBlocks(partSize); fac.FileClose(); CkXml partsListXml; partsListXml.put_Tag("CompleteMultipartUpload"); CkStringBuilder sbPartNumber; for (int partNumber = 1; partNumber <= numParts; ++partNumber) { sbPartNumber.Clear(); sbPartNumber.AppendInt(partNumber); CkStream fileStream; fileStream.put_SourceFile(dataFilePath.c_str()); fileStream.put_SourceFilePartSize(partSize); fileStream.put_SourceFilePart(partNumber - 1); conn->rest.ClearAllQueryParams(); conn->rest.AddQueryParam("partNumber", sbPartNumber.getAsString()); conn->rest.AddQueryParam("uploadId", uploadId); const char* responseStr = conn->rest.fullRequestStream("PUT", objectPath.c_str(), fileStream); if (!conn->rest.get_LastMethodSuccess()) { { AttemptResult r = ClassifyFailedRequest(std::move(conn), lastError); lastError = "Part " + std::to_string(partNumber) + ": " + lastError; return r; } } int partStatus = conn->rest.get_ResponseStatusCode(); if (partStatus != 200) { lastError = "Part " + std::to_string(partNumber) + " HTTP " + std::to_string(partStatus); if (responseStr != nullptr && responseStr[0] != '\0') lastError += " - " + std::string(responseStr); ReleaseConnection(std::move(conn)); bool retryable = (partStatus >= 500 && partStatus <= 599) || partStatus == 408 || partStatus == 429; return retryable ? AttemptResult::Transient : AttemptResult::Permanent; } const char* etag = conn->rest.responseHdrByName("ETag"); if (!conn->rest.get_LastMethodSuccess() || etag == nullptr || etag[0] == '\0') { lastError = "Part " + std::to_string(partNumber) + ": ETag missing from response"; DiscardConnection(std::move(conn)); // transport failed — do not pool a suspect socket return AttemptResult::Transient; } CkXml* xPart = partsListXml.NewChild("Part", ""); xPart->NewChildInt2("PartNumber", partNumber); xPart->NewChild2("ETag", etag); delete xPart; // Persist partsList for diagnostics; not used for resume in this attempt. partsListXml.SaveXml(partsListFilePath.c_str()); } // ---- STEP 3: COMPLETE ---- conn->rest.ClearAllQueryParams(); conn->rest.AddQueryParam("uploadId", uploadId); responseXml = conn->rest.fullRequestString("POST", objectPath.c_str(), partsListXml.getXml()); if (!conn->rest.get_LastMethodSuccess()) { { AttemptResult r = ClassifyFailedRequest(std::move(conn), lastError); lastError = "Complete: " + lastError; return r; } } int completeStatus = conn->rest.get_ResponseStatusCode(); if (completeStatus != 200) { lastError = "Complete HTTP " + std::to_string(completeStatus); if (responseXml != nullptr && responseXml[0] != '\0') lastError += " - " + std::string(responseXml); ReleaseConnection(std::move(conn)); bool retryable = (completeStatus >= 500 && completeStatus <= 599) || completeStatus == 408 || completeStatus == 429; return retryable ? AttemptResult::Transient : AttemptResult::Permanent; } // Cleanup parts list on success. if (fac.FileExists(partsListFilePath.c_str())) { fac.FileDelete(partsListFilePath.c_str()); } std::string scheme = _bTls ? "https://" : "http://"; uploadedFilePath = scheme + (_bAwsPath ? _fullAWSURL : _baseDomain) + "/" + bucketName + "/" + fileName; _logger.LogDebug(kOp, "Multipart upload completed for: " + fileName + " | URL: " + uploadedFilePath, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return AttemptResult::Success; }); } bool ANSAWSS3::UploadPrefixMultipartData(const std::string& bucketName, const std::string& prefix,const std::string& dataFilePath, const std::string& objectName, std::string& uploadedFilePath, int partSize) { const std::string kOp = "ANSAWSS3::UploadPrefixMultipartData"; //_logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__); // Early validation (permanent — do NOT retry) if (!CheckUploadPreconditions(kOp)) return false; if (bucketName.empty() || dataFilePath.empty()) { _logger.LogError(kOp, "Bucket name or file path is empty", __FILE__, __LINE__); SetCallStatus(kCallPermanentError); // caller bug — retrying cannot help return false; } if (partSize < 5242880) { _logger.LogError(kOp, "Part size must be at least 5MB (5242880 bytes)", __FILE__, __LINE__); SetCallStatus(kCallPermanentError); // caller bug — retrying cannot help return false; } { CkFileAccess fac; if (!fac.FileExists(dataFilePath.c_str())) { _logger.LogError(kOp, "File not found: " + dataFilePath, __FILE__, __LINE__); SetCallStatus(kCallPermanentError); // bad input path — retrying cannot help return false; } } std::string fileName = objectName; if (fileName.empty()) { fileName = ExtractFileName(dataFilePath); if (fileName.empty()) { _logger.LogError(kOp, "Failed to extract filename from path: " + dataFilePath, __FILE__, __LINE__); SetCallStatus(kCallPermanentError); // bad input path — retrying cannot help return false; } } // Build once: the object key and the HTTP request path. Keeping // them separate avoids the path-style bucket-duplication bug in // the returned URL. std::string objectKey; // "/" — no leading '/' if (!prefix.empty()) { std::string normalizedPrefix = prefix; if (!normalizedPrefix.empty() && normalizedPrefix.front() == '/') { normalizedPrefix = normalizedPrefix.substr(1); } if (!normalizedPrefix.empty() && normalizedPrefix.back() != '/') { normalizedPrefix += '/'; } objectKey = normalizedPrefix; } objectKey += fileName; const std::string objectPath = _bAwsPath ? ("/" + objectKey) : ("/" + bucketName + "/" + objectKey); // Parts-list file uses a sanitized filename for safe local storage. // Fix 2: also append a per-call unique token so parallel multipart // uploads of the same local file don't share the same scratch file. const std::string fileFolderPath = dataFilePath.substr(0, dataFilePath.find_last_of("/\\")); std::string safeFileName = fileName; std::replace(safeFileName.begin(), safeFileName.end(), '/', '_'); std::replace(safeFileName.begin(), safeFileName.end(), '\\', '_'); const std::string partsListFilePath = fileFolderPath + "/partsList_" + safeFileName + "_" + MakeUniqueMultipartToken(this) + ".xml"; // See note in UploadMultipartData regarding multipart retry semantics. return UploadWithRetry(kOp, [&](std::string& lastError) -> AttemptResult { CkFileAccess fac; if (fac.FileExists(partsListFilePath.c_str())) { fac.FileDelete(partsListFilePath.c_str()); } auto conn = AcquireConnection(); if (!conn) { lastError = "Failed to acquire S3 connection"; return AttemptResult::Transient; } if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // ---- STEP 1: INITIATE ---- conn->rest.ClearAllQueryParams(); conn->rest.AddQueryParam("uploads", ""); const char* responseXml = conn->rest.fullRequestNoBody("POST", objectPath.c_str()); if (!conn->rest.get_LastMethodSuccess()) { { AttemptResult r = ClassifyFailedRequest(std::move(conn), lastError); lastError = "Initiate: " + lastError; return r; } } int initStatus = conn->rest.get_ResponseStatusCode(); if (initStatus != 200) { lastError = "Initiate HTTP " + std::to_string(initStatus); if (responseXml != nullptr && responseXml[0] != '\0') lastError += " - " + std::string(responseXml); ReleaseConnection(std::move(conn)); bool retryable = (initStatus >= 500 && initStatus <= 599) || initStatus == 408 || initStatus == 429; return retryable ? AttemptResult::Transient : AttemptResult::Permanent; } CkXml xmlInit; if (!xmlInit.LoadXml(responseXml)) { lastError = std::string("Initiate XML parse error: ") + xmlInit.lastErrorText(); ReleaseConnection(std::move(conn)); return AttemptResult::Transient; } const char* uploadId = xmlInit.getChildContent("UploadId"); if (uploadId == nullptr || uploadId[0] == '\0') { lastError = "UploadId not found in Initiate response"; ReleaseConnection(std::move(conn)); return AttemptResult::Transient; } _logger.LogDebug(kOp, "Multipart initiated. UploadId: " + std::string(uploadId), __FILE__, __LINE__); // ---- STEP 2: UPLOAD PARTS ---- fac.OpenForRead(dataFilePath.c_str()); int numParts = fac.GetNumBlocks(partSize); fac.FileClose(); CkXml partsListXml; partsListXml.put_Tag("CompleteMultipartUpload"); CkStringBuilder sbPartNumber; for (int partNumber = 1; partNumber <= numParts; ++partNumber) { sbPartNumber.Clear(); sbPartNumber.AppendInt(partNumber); CkStream fileStream; fileStream.put_SourceFile(dataFilePath.c_str()); fileStream.put_SourceFilePartSize(partSize); fileStream.put_SourceFilePart(partNumber - 1); conn->rest.ClearAllQueryParams(); conn->rest.AddQueryParam("partNumber", sbPartNumber.getAsString()); conn->rest.AddQueryParam("uploadId", uploadId); const char* responseStr = conn->rest.fullRequestStream("PUT", objectPath.c_str(), fileStream); if (!conn->rest.get_LastMethodSuccess()) { { AttemptResult r = ClassifyFailedRequest(std::move(conn), lastError); lastError = "Part " + std::to_string(partNumber) + ": " + lastError; return r; } } int partStatus = conn->rest.get_ResponseStatusCode(); if (partStatus != 200) { lastError = "Part " + std::to_string(partNumber) + " HTTP " + std::to_string(partStatus); if (responseStr != nullptr && responseStr[0] != '\0') lastError += " - " + std::string(responseStr); ReleaseConnection(std::move(conn)); bool retryable = (partStatus >= 500 && partStatus <= 599) || partStatus == 408 || partStatus == 429; return retryable ? AttemptResult::Transient : AttemptResult::Permanent; } const char* etag = conn->rest.responseHdrByName("ETag"); if (!conn->rest.get_LastMethodSuccess() || etag == nullptr || etag[0] == '\0') { lastError = "Part " + std::to_string(partNumber) + ": ETag missing from response"; DiscardConnection(std::move(conn)); // transport failed — do not pool a suspect socket return AttemptResult::Transient; } CkXml* xPart = partsListXml.NewChild("Part", ""); xPart->NewChildInt2("PartNumber", partNumber); xPart->NewChild2("ETag", etag); delete xPart; partsListXml.SaveXml(partsListFilePath.c_str()); } // ---- STEP 3: COMPLETE ---- conn->rest.ClearAllQueryParams(); conn->rest.AddQueryParam("uploadId", uploadId); responseXml = conn->rest.fullRequestString("POST", objectPath.c_str(), partsListXml.getXml()); if (!conn->rest.get_LastMethodSuccess()) { { AttemptResult r = ClassifyFailedRequest(std::move(conn), lastError); lastError = "Complete: " + lastError; return r; } } int completeStatus = conn->rest.get_ResponseStatusCode(); if (completeStatus != 200) { lastError = "Complete HTTP " + std::to_string(completeStatus); if (responseXml != nullptr && responseXml[0] != '\0') lastError += " - " + std::string(responseXml); ReleaseConnection(std::move(conn)); bool retryable = (completeStatus >= 500 && completeStatus <= 599) || completeStatus == 408 || completeStatus == 429; return retryable ? AttemptResult::Transient : AttemptResult::Permanent; } if (fac.FileExists(partsListFilePath.c_str())) { fac.FileDelete(partsListFilePath.c_str()); } // Build canonical path-style URL from parts — independent // of the addressing style we actually used. std::string scheme = _bTls ? "https://" : "http://"; uploadedFilePath = scheme + (_bAwsPath ? _fullAWSURL : _baseDomain) + "/" + bucketName + "/" + objectKey; _logger.LogDebug(kOp, "Multipart upload completed. S3 key: " + objectKey + " | URL: " + uploadedFilePath, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return AttemptResult::Success; }); } // Upload jpeg data bool ANSAWSS3::UploadJpegImage(const std::string& bucketName, unsigned char* jpeg_string, int32 bufferLength, const std::string& fileName, std::string& uploadedFilePath) { const std::string kOp = "ANSAWSS3::UploadJpegImage"; // _logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__); // Early validation checks (permanent — do NOT retry) if (!CheckUploadPreconditions(kOp)) return false; if (bucketName.empty() || bufferLength <= 0) { _logger.LogError(kOp, "Bucket name or buffer length is invalid", __FILE__, __LINE__); SetCallStatus(kCallPermanentError); // caller bug — retrying cannot help return false; } // Object path is attempt-independent. const std::string objectPath = _bAwsPath ? ("/" + fileName) : ("/" + bucketName + "/" + fileName); return UploadWithRetry(kOp, [&](std::string& lastError) -> AttemptResult { auto conn = AcquireConnection(); if (!conn) { lastError = "Failed to acquire S3 connection"; return AttemptResult::Transient; } if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } CkBinData binData; CkByteData jpegBytes; jpegBytes.append2(jpeg_string, static_cast(bufferLength)); binData.AppendBinary(jpegBytes); conn->rest.AddHeader("Content-Type", "image/jpeg"); conn->rest.AddHeader("Expect", "100-continue"); CkStringBuilder sbResponse; if (!conn->rest.FullRequestBd("PUT", objectPath.c_str(), binData, sbResponse)) { return ClassifyFailedRequest(std::move(conn), lastError); } int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode == 200) { std::string scheme = _bTls ? "https://" : "http://"; uploadedFilePath = scheme + (_bAwsPath ? _fullAWSURL : _baseDomain) + "/" + bucketName + "/" + fileName; _logger.LogDebug(kOp, "Successfully uploaded: " + fileName + " | URL: " + uploadedFilePath, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return AttemptResult::Success; } lastError = "HTTP " + std::to_string(statusCode); std::string body = sbResponse.getAsString(); if (!body.empty()) lastError += " - " + body; ReleaseConnection(std::move(conn)); bool retryable = (statusCode >= 500 && statusCode <= 599) || statusCode == 408 || statusCode == 429; return retryable ? AttemptResult::Transient : AttemptResult::Permanent; }); } bool ANSAWSS3::UploadPrefixJpegImage(const std::string& bucketName, const std::string& prefix,unsigned char* jpeg_string,int32 bufferLength,const std::string& fileName, std::string& uploadedFilePath) { const std::string kOp = "ANSAWSS3::UploadPrefixJpegImage"; //_logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__); // Early validation checks (permanent failures — do NOT retry) if (!CheckUploadPreconditions(kOp)) return false; if (bucketName.empty() || bufferLength <= 0 || fileName.empty()) { _logger.LogError(kOp, "Bucket name, buffer length, or filename is invalid", __FILE__, __LINE__); SetCallStatus(kCallPermanentError); // caller bug — retrying cannot help return false; } // Build once: the object key (path within the bucket) and the HTTP // request path (which depends on addressing style). Keeping these // separate avoids the path-style bucket-duplication bug in the // returned URL. std::string objectKey; // "/" — no leading '/' if (!prefix.empty()) { std::string normalizedPrefix = prefix; if (!normalizedPrefix.empty() && normalizedPrefix.front() == '/') { normalizedPrefix = normalizedPrefix.substr(1); } if (!normalizedPrefix.empty() && normalizedPrefix.back() != '/') { normalizedPrefix += '/'; } objectKey = normalizedPrefix; } objectKey += fileName; const std::string objectPath = _bAwsPath ? ("/" + objectKey) : ("/" + bucketName + "/" + objectKey); return UploadWithRetry(kOp, [&](std::string& lastError) -> AttemptResult { auto conn = AcquireConnection(); if (!conn) { lastError = "Failed to acquire S3 connection"; return AttemptResult::Transient; } if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } CkBinData binData; CkByteData jpegBytes; jpegBytes.append2(jpeg_string, static_cast(bufferLength)); binData.AppendBinary(jpegBytes); conn->rest.AddHeader("Content-Type", "image/jpeg"); conn->rest.AddHeader("Expect", "100-continue"); CkStringBuilder sbResponse; if (!conn->rest.FullRequestBd("PUT", objectPath.c_str(), binData, sbResponse)) { return ClassifyFailedRequest(std::move(conn), lastError); } int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode == 200) { // Build the returned URL from parts (independent of // addressing style) — always the canonical path-style URL. std::string scheme = _bTls ? "https://" : "http://"; uploadedFilePath = scheme + (_bAwsPath ? _fullAWSURL : _baseDomain) + "/" + bucketName + "/" + objectKey; _logger.LogDebug(kOp, "Successfully uploaded: " + objectKey + " (" + std::to_string(bufferLength) + " bytes) | URL: " + uploadedFilePath, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return AttemptResult::Success; } lastError = "HTTP " + std::to_string(statusCode); std::string body = sbResponse.getAsString(); if (!body.empty()) lastError += " - " + body; ReleaseConnection(std::move(conn)); bool retryable = (statusCode >= 500 && statusCode <= 599) || statusCode == 408 || statusCode == 429; return retryable ? AttemptResult::Transient : AttemptResult::Permanent; }); } // Generates a temporary pre-signed URL (AWS Signature V4) for an object // already stored in S3. Public users can open the URL — without any AWS // credentials — until it expires. SigV4 pre-signing is a purely local // operation: it needs valid credentials and a known endpoint, but performs // no network I/O, so it succeeds even when not connected. bool ANSAWSS3::GeneratePresignedURL(const std::string& bucketName, const std::string& prefix, const std::string& objectName, int validDays, std::string& presignedURL) { const std::string kOp = "ANSAWSS3::GeneratePresignedURL"; // Permanent failures — do NOT proceed. if (!_isLicenseValid || !_isUnlockCodeValid) { _logger.LogError(kOp, !_isLicenseValid ? "Invalid license" : "Invalid unlock code", __FILE__, __LINE__); ANS_DBG("AWSS3", "GeneratePresignedURL: invalid license/unlock code"); return false; } if (bucketName.empty() || objectName.empty()) { _logger.LogError(kOp, "Bucket name or object name is empty", __FILE__, __LINE__); ANS_DBG("AWSS3", "GeneratePresignedURL: empty bucket/object name"); return false; } // AWS SigV4 caps pre-signed URL validity at 7 days. if (validDays < 1 || validDays > 7) { _logger.LogError(kOp, "validDays must be between 1 and 7 (got " + std::to_string(validDays) + ")", __FILE__, __LINE__); ANS_DBG("AWSS3", "GeneratePresignedURL: validDays out of range (%d)", validDays); return false; } // Snapshot the config/auth attributes under the config mutex. std::string accessKey, secretKey, region, endpoint, service; bool useHttps; { std::lock_guard lock(_configMutex); accessKey = _accessKey; secretKey = _secretKey; region = _bucketRegion; endpoint = _fullAWSURL; service = _serviceName; useHttps = _bTls; } if (accessKey.empty() || secretKey.empty()) { _logger.LogError(kOp, "AWS credentials not set (call SetAuthentication first)", __FILE__, __LINE__); ANS_DBG("AWSS3", "GeneratePresignedURL: missing credentials"); return false; } if (endpoint.empty() || region.empty() || service.empty()) { _logger.LogError(kOp, "S3 endpoint/region/service not configured (call Connect first)", __FILE__, __LINE__); ANS_DBG("AWSS3", "GeneratePresignedURL: endpoint/region/service not configured"); return false; } // Build the object key the same way UploadPrefixJpegImage does, so the // URL is generated for exactly the key that was uploaded. std::string objectKey; if (!prefix.empty()) { std::string normalizedPrefix = prefix; if (normalizedPrefix.front() == '/') { normalizedPrefix = normalizedPrefix.substr(1); } if (!normalizedPrefix.empty() && normalizedPrefix.back() != '/') { normalizedPrefix += '/'; } objectKey = normalizedPrefix; } objectKey += objectName; // Configure a CkHttp with the same credentials/endpoint the upload // path uses so the signed host matches the stored object's host. CkHttp http; http.put_AwsAccessKey(accessKey.c_str()); http.put_AwsSecretKey(secretKey.c_str()); http.put_AwsRegion(region.c_str()); http.put_AwsEndpoint(endpoint.c_str()); const int numSecondsValid = validDays * 86400; const char* url = http.s3_GenPresignedUrl("GET", useHttps, bucketName.c_str(), objectKey.c_str(), numSecondsValid, service.c_str()); if (!http.get_LastMethodSuccess()) { _logger.LogError(kOp, std::string("Pre-signed URL generation failed: ") + http.lastErrorText(), __FILE__, __LINE__); ANS_DBG("AWSS3", "GeneratePresignedURL FAILED: key=%s err=%s", objectKey.c_str(), http.lastErrorText()); return false; } presignedURL = (url != nullptr) ? url : ""; _logger.LogDebug(kOp, "Generated pre-signed URL for '" + objectKey + "' valid " + std::to_string(validDays) + " day(s): " + presignedURL, __FILE__, __LINE__); ANS_DBG("AWSS3", "GeneratePresignedURL OK: key=%s validDays=%d url=%s", objectKey.c_str(), validDays, presignedURL.c_str()); return true; } // Downloads bool ANSAWSS3::DownloadFile(const std::string& bucketName, const std::string& objectName, const std::string& saveFilePath) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::DownloadFile", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || objectName.empty() || saveFilePath.empty()) { _logger.LogError("ANSAWSS3::DownloadFile", "Bucket name, object name, or save path is empty", __FILE__, __LINE__); return false; } bool downloadSuccess = false; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("DownloadFile", "Failed to acquire S3 connection", __FILE__, __LINE__); return false; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // Normalize object name (remove leading slash if present) std::string normalizedObjectName = objectName; if (!normalizedObjectName.empty() && normalizedObjectName[0] == '/') { normalizedObjectName = normalizedObjectName.substr(1); } std::string objectPath = _bAwsPath ? ("/" + normalizedObjectName) : ("/" + bucketName + "/" + normalizedObjectName); _logger.LogDebug("ANSAWSS3::DownloadFile", "Downloading: " + normalizedObjectName + " from bucket: " + bucketName, __FILE__, __LINE__); // Send GET request if (!conn->rest.SendReqNoBody("GET", objectPath.c_str())) { _logger.LogError("ANSAWSS3::DownloadFile - Request failed", conn->rest.lastErrorText(), __FILE__, __LINE__); DiscardConnection(std::move(conn)); // transport failed — do not pool a suspect socket return false; } // Read the response header int responseStatusCode = conn->rest.ReadResponseHeader(); if (responseStatusCode < 0) { _logger.LogError("ANSAWSS3::DownloadFile - Failed to read response header", conn->rest.lastErrorText(), __FILE__, __LINE__); DiscardConnection(std::move(conn)); // transport failed — do not pool a suspect socket return false; } // We expect a 200 response status if the data is coming if (responseStatusCode == 200) { // Determine the final file path std::string finalFilePath; // Check if saveFilePath is a directory or a file CkFileAccess fac; bool isDirectory = false; // If path exists and is a directory, or ends with path separator if (fac.DirExists(saveFilePath.c_str())) { isDirectory = true; } else if (!saveFilePath.empty() && (saveFilePath.back() == '\\' || saveFilePath.back() == '/')) { isDirectory = true; } if (isDirectory) { // Save path is a directory - append object name finalFilePath = saveFilePath; // Ensure path ends with separator if (!finalFilePath.empty() && finalFilePath.back() != '\\' && finalFilePath.back() != '/') { finalFilePath += "\\"; } // Extract just the filename from object name (in case it has path separators) std::string filename = normalizedObjectName; size_t lastSlash = filename.find_last_of("/\\"); if (lastSlash != std::string::npos) { filename = filename.substr(lastSlash + 1); } finalFilePath += filename; } else { // Save path is the complete file path finalFilePath = saveFilePath; } // Ensure the directory exists size_t lastSeparator = finalFilePath.find_last_of("/\\"); if (lastSeparator != std::string::npos) { std::string directory = finalFilePath.substr(0, lastSeparator); // Create directory if it doesn't exist if (!fac.DirExists(directory.c_str())) { if (!fac.DirCreate(directory.c_str())) { _logger.LogError("ANSAWSS3::DownloadFile - Failed to create directory", "Directory: " + directory + " - " + fac.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } _logger.LogDebug("ANSAWSS3::DownloadFile", "Created directory: " + directory, __FILE__, __LINE__); } } // Setup stream to write to file CkStream bodyStream; bodyStream.put_SinkFile(finalFilePath.c_str()); // Read the response body to the stream if (!conn->rest.ReadRespBodyStream(bodyStream, true)) { _logger.LogError("ANSAWSS3::DownloadFile - Failed to read response body", conn->rest.lastErrorText(), __FILE__, __LINE__); } else { _logger.LogDebug("ANSAWSS3::DownloadFile", "Successfully downloaded: " + normalizedObjectName + " to: " + finalFilePath, __FILE__, __LINE__); downloadSuccess = true; } } else { // Handle non-200 response const char* errResponse = conn->rest.readRespBodyString(); std::string errorMsg = "HTTP " + std::to_string(responseStatusCode); if (!conn->rest.get_LastMethodSuccess()) { errorMsg += " - Failed to read error response: " + std::string(conn->rest.lastErrorText()); } else { if (errResponse != nullptr && errResponse[0] != '\0') { errorMsg += " - " + std::string(errResponse); } } _logger.LogError("ANSAWSS3::DownloadFile", errorMsg, __FILE__, __LINE__); } ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::DownloadFile", std::string("Exception: ") + e.what(), __FILE__, __LINE__); } return downloadSuccess; } // Delete bool ANSAWSS3::DeleteBucketObject(const std::string& bucketName, const std::string& objectName) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::DeleteBucketObject", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || objectName.empty()) { _logger.LogError("ANSAWSS3::DeleteBucketObject", "Bucket name or object name is empty", __FILE__, __LINE__); return false; } bool deleteSuccess = false; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("DeleteBucketObject", "Failed to acquire S3 connection", __FILE__, __LINE__); return false; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } std::string objectPath = _bAwsPath ? ("/" + objectName) : ("/" + bucketName + "/" + objectName); CkStringBuilder sbResponse; bool success = conn->rest.FullRequestNoBodySb("DELETE", objectPath.c_str(), sbResponse); if (!success) { _logger.LogError("ANSAWSS3::DeleteBucketObject", "Failed to send DELETE request: " + std::string(conn->rest.lastErrorText()), __FILE__, __LINE__); } else { int statusCode = conn->rest.get_ResponseStatusCode(); // S3 returns 204 (No Content) for successful deletion // Also accept 200 as success if (statusCode == 204 || statusCode == 200) { _logger.LogDebug("ANSAWSS3::DeleteBucketObject", "Successfully deleted object: " + objectName + " from bucket: " + bucketName, __FILE__, __LINE__); deleteSuccess = true; } else if (statusCode == 404) { // Object doesn't exist - you may want to treat this as success or failure // depending on your use case // Uncomment the next line if you want to treat "not found" as success: deleteSuccess = true; } else { std::string errorMsg = "HTTP " + std::to_string(statusCode); std::string response = sbResponse.getAsString(); if (!response.empty()) { errorMsg += " - " + response; } _logger.LogError("ANSAWSS3::DeleteBucketObject", errorMsg, __FILE__, __LINE__); } } ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::DeleteBucketObject", std::string("Exception: ") + e.what(), __FILE__, __LINE__); } return deleteSuccess; } }