#include "ANSUtilities.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static bool ansawss3LicenceValid = false; static std::once_flag ansawss3LicenseOnceFlag; namespace ANSCENTER { static void VerifyGlobalANSAWSS3License(const std::string& licenseKey) { try { static const std::vector> licenseChecks = { {1000, "ANNHUB-LV"}, {1001, "DLHUB-LV"}, {1002, "ODHUB-LV"}, {1003, "ANSVIS"}, {1004, "ANSFR"}, {1005, "ANSOCR"}, {1006, "ANSALPR"}, {1007, "ANSCV"}, {1008, "ANSSRT"} }; ansawss3LicenceValid = false; for (const auto& [productId, productName] : licenseChecks) { if (ANSCENTER::ANSLicenseHelper::LicenseVerification(licenseKey, productId, productName)) { ansawss3LicenceValid = true; break; // Stop at the first valid license } } } catch (const std::exception& e) { ansawss3LicenceValid = false; } } // Builds a per-call unique suffix for scratch file names used by // multipart uploads. Combining thread id, instance address, and a // monotonically-increasing counter guarantees no collision between // parallel uploads — including two threads uploading the same source // file at the same time (Fix 2). static std::string MakeUniqueMultipartToken(const void* instancePtr) { static std::atomic counter{0}; std::ostringstream oss; oss << std::hex << std::hash{}(std::this_thread::get_id()) << "_" << reinterpret_cast(instancePtr) << "_" << counter.fetch_add(1, std::memory_order_relaxed); return oss.str(); } // Private helper function to extract file name from a path // Helper function to extract filename from path std::string ANSAWSS3::ExtractFileName(const std::string& filePath) { size_t pos = filePath.find_last_of("/\\"); if (pos != std::string::npos) { return filePath.substr(pos + 1); } return filePath; } // Helper function to determine content type from file extension std::string ANSAWSS3::GetContentType(const std::string& filePath) { // Default to application/octet-stream for unknown types std::string contentType = "application/octet-stream"; size_t extPos = filePath.find_last_of('.'); if (extPos == std::string::npos) { return contentType; } // Extract and convert extension to lowercase std::string fileExt = filePath.substr(extPos + 1); std::transform(fileExt.begin(), fileExt.end(), fileExt.begin(), ::tolower); // Image types if (fileExt == "jpg" || fileExt == "jpeg") { contentType = "image/jpeg"; } else if (fileExt == "png") { contentType = "image/png"; } else if (fileExt == "gif") { contentType = "image/gif"; } else if (fileExt == "bmp") { contentType = "image/bmp"; } else if (fileExt == "webp") { contentType = "image/webp"; } else if (fileExt == "svg") { contentType = "image/svg+xml"; } else if (fileExt == "ico") { contentType = "image/x-icon"; } // Video types else if (fileExt == "mp4") { contentType = "video/mp4"; } else if (fileExt == "avi") { contentType = "video/x-msvideo"; } else if (fileExt == "mov") { contentType = "video/quicktime"; } // Document types else if (fileExt == "pdf") { contentType = "application/pdf"; } else if (fileExt == "json") { contentType = "application/json"; } else if (fileExt == "xml") { contentType = "application/xml"; } else if (fileExt == "txt") { contentType = "text/plain"; } return contentType; } ANSAWSS3::ANSAWSS3() { _unlockCode = "ANSDRC.CB1122026_MEQCIFwO1IFQCG0BhZwsXFO68QUU6mDB5uge4duOsqOJanEyAiAB67ahqnXin4SRy0vIegISgbFlpldmbuS5gbU21GYVqA==";// "ANSDRC.CB1082025_Ax6P3M7F8B3d";// _proxyHost = ""; _proxyPort = 0; _proxyUsername = ""; _proxyPassword = ""; _bProxy = false; _serviceName = "s3"; } ANSAWSS3::~ANSAWSS3() { StopRetry(); std::lock_guard lk(_poolMutex); _pool.clear(); // CkRest destructors handle disconnect } void ANSAWSS3::StopRetry() { _stopRetry = true; if (_retryThread.joinable()) { _retryThread.join(); } _retryInProgress = false; } // TryConnect — actual connection attempt (caller must hold _configMutex) bool ANSAWSS3::TryConnect(bool& awsPath) { auto testConn = CreateConnection(); if (!testConn) { // Fallback: connect directly to baseDomain (MinIO, Ceph, etc.) std::string savedFullURL = _fullAWSURL; _fullAWSURL = _baseDomain; testConn = CreateConnection(); if (!testConn) { _fullAWSURL = savedFullURL; // restore on failure return false; } awsPath = false; _bAwsPath = false; } else { awsPath = true; _bAwsPath = true; } _bConnected = true; // Return test connection to pool for reuse { std::lock_guard lk(_poolMutex); _pool.push_back(std::move(testConn)); } return true; } void ANSAWSS3::RetryLoop() { int attempt = 0; while (!_stopRetry) { // Wait 3 seconds (in 1-second increments so we can respond to stop quickly) for (int i = 0; i < 3 && !_stopRetry; ++i) { std::this_thread::sleep_for(std::chrono::seconds(1)); } if (_stopRetry) break; attempt++; // Quick DNS check — is internet available? { CkSocket dnsCheck; CkString dnsResult; if (!dnsCheck.DnsLookup(_fullAWSURL.c_str(), 3000, dnsResult)) { _logger.LogDebug("ANSAWSS3::RetryLoop", "Retry #" + std::to_string(attempt) + " - no internet, retrying in 3s", __FILE__, __LINE__); continue; } } // Internet is available — attempt real connection to validate auth/URL { std::lock_guard lk(_configMutex); bool awsPath = true; if (TryConnect(awsPath)) { _logger.LogDebug("ANSAWSS3::RetryLoop", "Connected successfully after " + std::to_string(attempt) + " retries", __FILE__, __LINE__); _retryInProgress = false; return; } else { // Internet is available but connection failed (bad auth/URL) // Stop retrying — no point, the parameters are wrong _logger.LogError("ANSAWSS3::RetryLoop", "Internet available but connection failed (check authentication or URL). Stopping retry.", __FILE__, __LINE__); _retryInProgress = false; return; } } } _retryInProgress = false; } // ── Connection pool helpers ── std::unique_ptr ANSAWSS3::CreateConnection() { auto conn = std::make_unique(); // Apply request timeouts BEFORE Connect() so ConnectTimeoutMs takes // effect on this very handshake. IdleTimeoutMs is sticky and applies // to every subsequent request on this CkRest. conn->rest.put_ConnectTimeoutMs(_connectTimeoutMs.load()); conn->rest.put_IdleTimeoutMs(_idleTimeoutMs.load()); // Connect if (!conn->rest.Connect(_fullAWSURL.c_str(), _port, _bTls, _bAutoReconnect)) { _logger.LogError("ANSAWSS3::CreateConnection", conn->rest.lastErrorText(), __FILE__, __LINE__); return nullptr; } // Apply proxy if configured if (_bProxy) { conn->socket.put_HttpProxyHostname(_proxyHost.c_str()); conn->socket.put_HttpProxyPort(_proxyPort); conn->socket.put_HttpProxyUsername(_proxyUsername.c_str()); conn->socket.put_HttpProxyPassword(_proxyPassword.c_str()); conn->socket.put_HttpProxyForHttp(_bProxy); if (!conn->rest.UseConnection(conn->socket, true)) { _logger.LogError("ANSAWSS3::CreateConnection - Proxy error", conn->rest.lastErrorText(), __FILE__, __LINE__); return nullptr; } } // Apply auth if (_authReady) { conn->authAws.put_AccessKey(_accessKey.c_str()); conn->authAws.put_SecretKey(_secretKey.c_str()); conn->authAws.put_ServiceName(_serviceName.c_str()); if (_bucketRegion != "us-east-1") { if (!_bucketRegion.empty()) conn->authAws.put_Region(_bucketRegion.c_str()); } if (!conn->rest.SetAuthAws(conn->authAws)) { _logger.LogError("ANSAWSS3::CreateConnection - Auth error", conn->rest.lastErrorText(), __FILE__, __LINE__); return nullptr; } } return conn; } std::unique_ptr ANSAWSS3::AcquireConnection() { { std::lock_guard lk(_poolMutex); if (!_pool.empty()) { auto conn = std::move(_pool.back()); _pool.pop_back(); return conn; } } // Pool empty — create a new connection (no lock held during network I/O) std::lock_guard cfgLk(_configMutex); return CreateConnection(); } void ANSAWSS3::ReleaseConnection(std::unique_ptr conn) { if (!conn) return; std::lock_guard lk(_poolMutex); _pool.push_back(std::move(conn)); } // Pre-populate the pool with `count` ready-to-use connections. Each // connection performs a TLS handshake; doing `count` of them in parallel // (rather than serially as AcquireConnection would on a cold pool) cuts // the first-upload latency for workloads that fan out right after // Connect() / SetAuthentication(). int ANSAWSS3::PrewarmConnectionPool(int count) { if (count <= 0) return 0; if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::PrewarmConnectionPool", "Not ready: call Connect() and SetAuthentication() first", __FILE__, __LINE__); return 0; } // Hold _configMutex once across the whole prewarm. It blocks config // writes for the duration, but does NOT block individual worker // threads since CreateConnection() itself does not re-acquire it. std::lock_guard cfgLk(_configMutex); std::vector> built(count); std::vector workers; workers.reserve(count); for (int i = 0; i < count; ++i) { workers.emplace_back([this, i, &built]() { try { built[i] = CreateConnection(); } catch (...) { built[i] = nullptr; } }); } for (auto& t : workers) t.join(); int added = 0; { std::lock_guard lk(_poolMutex); for (auto& conn : built) { if (conn) { _pool.push_back(std::move(conn)); ++added; } } } _logger.LogDebug("ANSAWSS3::PrewarmConnectionPool", "Added " + std::to_string(added) + "/" + std::to_string(count) + " connections to pool", __FILE__, __LINE__); return added; } // Update the per-connection request timeouts. New values are applied // immediately to all pooled connections (IdleTimeoutMs is picked up on // the next request; ConnectTimeoutMs is harmless on already-connected // CkRest instances but takes effect if they ever reconnect) and baked // into the defaults used by future CreateConnection() calls. bool ANSAWSS3::SetTimeouts(int connectMs, int idleMs) { if (connectMs < 1000 || idleMs < 1000) { _logger.LogError("ANSAWSS3::SetTimeouts", "Timeout values must be >= 1000 ms (got connect=" + std::to_string(connectMs) + ", idle=" + std::to_string(idleMs) + ")", __FILE__, __LINE__); return false; } _connectTimeoutMs.store(connectMs); _idleTimeoutMs.store(idleMs); // Propagate to existing pooled connections so already-warm workers // start using the new values on their next request. { std::lock_guard lk(_poolMutex); for (auto& conn : _pool) { if (conn) { conn->rest.put_ConnectTimeoutMs(connectMs); conn->rest.put_IdleTimeoutMs(idleMs); } } } _logger.LogDebug("ANSAWSS3::SetTimeouts", "connect=" + std::to_string(connectMs) + "ms, idle=" + std::to_string(idleMs) + "ms", __FILE__, __LINE__); return true; } // Drives a retry loop for a single upload operation. See the declaration // comment in ANSUtilities.h for the AttemptResult contract. // // Retry policy (shared across all upload entry points): // - kUploadMaxAttempts attempts total // - kUploadRetryDelayMs between attempts (no delay after the last one) // - std::exception thrown inside attemptFn is treated as Transient // - Any other exception is treated as Permanent (bail immediately) bool ANSAWSS3::UploadWithRetry( const std::string& opName, const std::function& attemptFn) { std::string lastError; for (int attempt = 1; attempt <= kUploadMaxAttempts; ++attempt) { AttemptResult result = AttemptResult::Transient; try { result = attemptFn(lastError); } catch (const std::exception& e) { lastError = std::string("Exception: ") + e.what(); result = AttemptResult::Transient; } catch (...) { lastError = "Unknown exception"; result = AttemptResult::Permanent; } if (result == AttemptResult::Success) { return true; } if (result == AttemptResult::Permanent) { _logger.LogError(opName, "Permanent failure (no retry): " + lastError, __FILE__, __LINE__); return false; } // Transient — retry if attempts remain. if (attempt < kUploadMaxAttempts) { _logger.LogDebug(opName, "Attempt " + std::to_string(attempt) + "/" + std::to_string(kUploadMaxAttempts) + " failed: " + lastError + " — retrying in " + std::to_string(kUploadRetryDelayMs) + "ms", __FILE__, __LINE__); std::this_thread::sleep_for( std::chrono::milliseconds(kUploadRetryDelayMs)); } } _logger.LogError(opName, "Upload failed after " + std::to_string(kUploadMaxAttempts) + " attempts. Last error: " + lastError, __FILE__, __LINE__); return false; } void ANSAWSS3::CheckLicense() { // Note: caller (Initialize) already holds _configMutex try { // Check once globally std::call_once(ansawss3LicenseOnceFlag, [this]() { VerifyGlobalANSAWSS3License(_licenseKey); }); // Update this instance's local license flag _isLicenseValid = ansawss3LicenceValid; } catch (const std::exception& e) { this->_logger.LogFatal("ANSAWSS3::CheckLicense. Error:", e.what(), __FILE__, __LINE__); } } void ANSAWSS3::CheckUnlockCode() { // Note: caller (Initialize) already holds _configMutex try { CkGlobal glob; _unlockCode = "ANSDRC.CB1122026_MEQCIFwO1IFQCG0BhZwsXFO68QUU6mDB5uge4duOsqOJanEyAiAB67ahqnXin4SRy0vIegISgbFlpldmbuS5gbU21GYVqA==";// "ANSDRC.CB1082025_Ax6P3M7F8B3d"; _isUnlockCodeValid = glob.UnlockBundle(_unlockCode.c_str()); if (!_isUnlockCodeValid) { _logger.LogFatal("ANSAWSS3::CheckUnlockCode", glob.lastErrorText(), __FILE__, __LINE__); return; } int status = glob.get_UnlockStatus(); if (status != 2) { _logger.LogDebug("ANSAWSS3::CheckUnlockCode", "Unlocked in trial mode.", __FILE__, __LINE__); } } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::CheckUnlockCode", e.what(), __FILE__, __LINE__); } } bool ANSAWSS3::Initialize(const std::string& licenseKey) { std::lock_guard lock(_configMutex); try { _licenseKey = licenseKey; CheckLicense(); CheckUnlockCode(); return _isLicenseValid && _isUnlockCodeValid; } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::Initialize", e.what(), __FILE__, __LINE__); return false; } } bool ANSAWSS3::SetServerProxy(const std::string& proxyHost, int proxyPort,const std::string& proxyUsername,const std::string& proxyPassword) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid) { _logger.LogError("ANSAWSS3::SetServerProxy", !_isLicenseValid ? "Invalid license" : "Invalid unlock code", __FILE__, __LINE__); return false; } std::lock_guard lock(_configMutex); try { _proxyHost = proxyHost; _proxyPort = proxyPort; _proxyUsername = proxyUsername; _proxyPassword = proxyPassword; // Simplified proxy validation logic _bProxy = !proxyHost.empty() && proxyPort > 0; // Clear pool so new connections pick up proxy config { std::lock_guard lk(_poolMutex); _pool.clear(); } return true; } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::SetServerProxy", e.what(), __FILE__, __LINE__); return false; } } // Returns: 1 = connected, 0 = failed (bad auth/URL), 2 = no internet (background retry started) int ANSAWSS3::Connect(const std::string& baseDomain, const std::string& bucketRegion, const std::string& serviceName, int port, bool bTls, bool autoReconnect, bool &awsPath) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid) { _logger.LogError("ANSAWSS3::Connect", !_isLicenseValid ? "Invalid license" : "Invalid unlock code", __FILE__, __LINE__); return 0; } // Stop any existing background retry StopRetry(); std::lock_guard lock(_configMutex); try { // Store parameters — strip http:// or https:// from baseDomain { std::string domain = baseDomain.empty() ? "amazonaws.com" : baseDomain; if (domain.rfind("https://", 0) == 0) { domain = domain.substr(8); bTls = true; // caller said https, honour it } else if (domain.rfind("http://", 0) == 0) { domain = domain.substr(7); } // Remove trailing slash if any if (!domain.empty() && domain.back() == '/') domain.pop_back(); _baseDomain = domain; } _bucketRegion = bucketRegion.empty() ? "us-east-1" : bucketRegion; _serviceName = serviceName.empty() ? "s3" : serviceName; _fullAWSURL = _serviceName + "." + _bucketRegion + "." + _baseDomain; _port = port; _bTls = bTls; _bAutoReconnect = autoReconnect; // Clear pool so new connections use updated config { std::lock_guard lk(_poolMutex); _pool.clear(); } // Check if internet is available (fast DNS check) { CkSocket dnsCheck; CkString dnsResult; if (!dnsCheck.DnsLookup(_fullAWSURL.c_str(), 3000, dnsResult)) { // No internet — start background retry, return immediately _logger.LogDebug("ANSAWSS3::Connect", "No internet available, starting background retry for '" + _fullAWSURL + "'...", __FILE__, __LINE__); _stopRetry = false; _retryInProgress = true; _retryThread = std::thread(&ANSAWSS3::RetryLoop, this); return 2; // no internet, retrying in background } } // Internet is available — try real connection if (TryConnect(awsPath)) { return 1; // connected successfully } // Internet available but connection failed (bad auth, wrong URL, etc.) return 0; } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::Connect", e.what(), __FILE__, __LINE__); return 0; } } bool ANSAWSS3::SetAuthentication(const std::string& accessKey,const std::string& secretKey) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid) { _logger.LogError("ANSAWSS3::SetAuthentication", !_isLicenseValid ? "Invalid license" : "Invalid unlock code", __FILE__, __LINE__); return false; } // Write credentials + clear the pool under _configMutex. The lock is // released before the auto-prewarm below so PrewarmConnectionPool // (which also takes _configMutex) doesn't deadlock. { std::lock_guard lock(_configMutex); try { _accessKey = accessKey; _secretKey = secretKey; _authReady = true; // Clear pool so new connections pick up new credentials. std::lock_guard lk(_poolMutex); _pool.clear(); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::SetAuthentication", e.what(), __FILE__, __LINE__); return false; } } // Auto-prewarm the connection pool so the first N concurrent uploads // don't serialize on TLS handshake. No-op if not connected yet. static constexpr int kDefaultPrewarmCount = 8; PrewarmConnectionPool(kDefaultPrewarmCount); return true; } std::vector ANSAWSS3::ListBuckets() { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::ListBuckets", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return {}; } try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("ListBuckets", "Failed to acquire S3 connection", __FILE__, __LINE__); return {}; } // Make request and capture response conn->rest.put_Host(_fullAWSURL.c_str()); CkStringBuilder sbResponse; if (!conn->rest.FullRequestNoBodySb("GET", "/", sbResponse)) { _logger.LogError("ANSAWSS3::ListBuckets - Request failed", conn->rest.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return {}; } // Check HTTP status code int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode != 200) { _logger.LogError("ANSAWSS3::ListBuckets - HTTP " + std::to_string(statusCode), sbResponse.getAsString(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return {}; } // Parse XML response CkXml xml; if (!xml.LoadSb(sbResponse, true)) { _logger.LogError("ANSAWSS3::ListBuckets - XML parse error", xml.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return {}; } // Extract bucket names std::vector bucketList; int bucketCount = xml.NumChildrenHavingTag("Buckets|Bucket"); if (bucketCount > 0) { bucketList.reserve(bucketCount); for (int i = 0; i < bucketCount; ++i) { xml.put_I(i); const char* name = xml.getChildContent("Buckets|Bucket[i]|Name"); // Validate name before adding if (name != nullptr && name[0] != '\0') { bucketList.emplace_back(name); } } } ReleaseConnection(std::move(conn)); return bucketList; } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::ListBuckets", e.what(), __FILE__, __LINE__); return {}; } } std::vector ANSAWSS3::ListBucketObjects(const std::string& bucketName) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::ListBucketObjects", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return {}; } if (bucketName.empty()) { _logger.LogError("ANSAWSS3::ListBucketObjects", "Bucket name is empty", __FILE__, __LINE__); return {}; } std::vector objectList; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("ListBucketObjects", "Failed to acquire S3 connection", __FILE__, __LINE__); return {}; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } std::string marker = ""; bool isTruncated = true; int pageCount = 0; // Loop to handle pagination while (isTruncated) { pageCount++; // Build the request path with marker if available std::string basePath = _bAwsPath ? "/" : ("/" + bucketName + "/"); std::string requestPath = basePath; if (!marker.empty()) { requestPath = basePath + "?marker=" + marker; } CkStringBuilder sbResponse; bool success = conn->rest.FullRequestNoBodySb("GET", requestPath.c_str(), sbResponse); if (!success) { _logger.LogError("ANSAWSS3::ListBucketObjects - Request failed on page " + std::to_string(pageCount), conn->rest.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return objectList; // Return what we have so far } int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode != 200) { std::string errorMsg = "HTTP " + std::to_string(statusCode); std::string response = sbResponse.getAsString(); if (!response.empty()) { errorMsg += " - " + response; } _logger.LogError("ANSAWSS3::ListBucketObjects - Page " + std::to_string(pageCount), errorMsg, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return objectList; // Return what we have so far } // Parse XML response CkXml xml; bool loadSuccess = xml.LoadSb(sbResponse, true); if (!loadSuccess) { _logger.LogError("ANSAWSS3::ListBucketObjects", "Failed to parse XML response on page " + std::to_string(pageCount) + ": " + std::string(xml.lastErrorText()), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return objectList; // Return what we have so far } // Check if results are truncated (more pages to fetch) const char* truncatedStr = xml.getChildContent("IsTruncated"); isTruncated = (truncatedStr != nullptr && std::string(truncatedStr) == "true"); // Get the marker for next page if (isTruncated) { // First try to get NextMarker const char* nextMarker = xml.getChildContent("NextMarker"); if (nextMarker != nullptr && nextMarker[0] != '\0') { marker = std::string(nextMarker); } else { // If NextMarker is not present, use the last Key as marker int count = xml.NumChildrenHavingTag("Contents"); if (count > 0) { xml.put_I(count - 1); // Last element const char* lastKey = xml.getChildContent("Contents[i]|Key"); if (lastKey != nullptr && lastKey[0] != '\0') { marker = std::string(lastKey); } else { isTruncated = false; } } else { isTruncated = false; } } } // Iterate through all Contents elements in this page int count = xml.NumChildrenHavingTag("Contents"); for (int i = 0; i < count; i++) { xml.put_I(i); const char* key = xml.getChildContent("Contents[i]|Key"); if (key != nullptr && key[0] != '\0') { objectList.emplace_back(key); } } _logger.LogDebug("ANSAWSS3::ListBucketObjects", "Page " + std::to_string(pageCount) + ": Retrieved " + std::to_string(count) + " objects. Total so far: " + std::to_string(objectList.size()), __FILE__, __LINE__); } _logger.LogDebug("ANSAWSS3::ListBucketObjects", "Successfully listed " + std::to_string(objectList.size()) + " objects in bucket: " + bucketName + " (" + std::to_string(pageCount) + " pages)", __FILE__, __LINE__); ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::ListBucketObjects", std::string("Exception: ") + e.what(), __FILE__, __LINE__); objectList.clear(); // Ensure we return empty list on exception } return objectList; } std::vector ANSAWSS3::ListBucketObjectsWithPrefix(const std::string& bucketName, const std::string& prefix) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::ListBucketObjectsWithPrefix", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return {}; } if (bucketName.empty()) { _logger.LogError("ANSAWSS3::ListBucketObjectsWithPrefix", "Bucket name is empty", __FILE__, __LINE__); return {}; } // prefix can be empty to list all objects std::vector objectList; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("ListBucketObjectsWithPrefix", "Failed to acquire S3 connection", __FILE__, __LINE__); return {}; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } std::string marker = ""; bool isTruncated = true; int pageCount = 0; // Loop to handle pagination while (isTruncated) { pageCount++; // Build the request path with prefix and marker std::string basePath = _bAwsPath ? "/" : ("/" + bucketName + "/"); std::string requestPath = basePath; bool hasParams = false; // Add prefix parameter if provided if (!prefix.empty()) { requestPath += "?prefix=" + prefix; hasParams = true; } // Add marker parameter if available if (!marker.empty()) { if (hasParams) { requestPath += "&marker=" + marker; } else { requestPath += "?marker=" + marker; hasParams = true; } } CkStringBuilder sbResponse; bool success = conn->rest.FullRequestNoBodySb("GET", requestPath.c_str(), sbResponse); if (!success) { _logger.LogError("ANSAWSS3::ListBucketObjectsWithPrefix - Request failed on page " + std::to_string(pageCount), conn->rest.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return objectList; // Return what we have so far } int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode != 200) { std::string errorMsg = "HTTP " + std::to_string(statusCode); std::string response = sbResponse.getAsString(); if (!response.empty()) { errorMsg += " - " + response; } _logger.LogError("ANSAWSS3::ListBucketObjectsWithPrefix - Page " + std::to_string(pageCount), errorMsg, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return objectList; // Return what we have so far } // Parse XML response CkXml xml; bool loadSuccess = xml.LoadSb(sbResponse, true); if (!loadSuccess) { _logger.LogError("ANSAWSS3::ListBucketObjectsWithPrefix", "Failed to parse XML response on page " + std::to_string(pageCount) + ": " + std::string(xml.lastErrorText()), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return objectList; // Return what we have so far } // Check if results are truncated (more pages to fetch) const char* truncatedStr = xml.getChildContent("IsTruncated"); isTruncated = (truncatedStr != nullptr && std::string(truncatedStr) == "true"); // Get the marker for next page if (isTruncated) { // First try to get NextMarker const char* nextMarker = xml.getChildContent("NextMarker"); if (nextMarker != nullptr && nextMarker[0] != '\0') { marker = std::string(nextMarker); } else { // If NextMarker is not present, use the last Key as marker int count = xml.NumChildrenHavingTag("Contents"); if (count > 0) { xml.put_I(count - 1); // Last element const char* lastKey = xml.getChildContent("Contents[i]|Key"); if (lastKey != nullptr && lastKey[0] != '\0') { marker = std::string(lastKey); } else { isTruncated = false; } } else { isTruncated = false; } } } // Iterate through all Contents elements in this page int count = xml.NumChildrenHavingTag("Contents"); for (int i = 0; i < count; i++) { xml.put_I(i); const char* key = xml.getChildContent("Contents[i]|Key"); if (key != nullptr && key[0] != '\0') { objectList.emplace_back(key); } } std::string prefixInfo = prefix.empty() ? "all objects" : "prefix '" + prefix + "'"; _logger.LogDebug("ANSAWSS3::ListBucketObjectsWithPrefix", "Page " + std::to_string(pageCount) + ": Retrieved " + std::to_string(count) + " objects for " + prefixInfo + ". Total so far: " + std::to_string(objectList.size()), __FILE__, __LINE__); } std::string prefixInfo = prefix.empty() ? "" : " with prefix '" + prefix + "'"; _logger.LogDebug("ANSAWSS3::ListBucketObjectsWithPrefix", "Successfully listed " + std::to_string(objectList.size()) + " objects in bucket: " + bucketName + prefixInfo + " (" + std::to_string(pageCount) + " pages)", __FILE__, __LINE__); ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::ListBucketObjectsWithPrefix", std::string("Exception: ") + e.what(), __FILE__, __LINE__); objectList.clear(); // Ensure we return empty list on exception } return objectList; } bool ANSAWSS3::CreateBucket(const std::string& bucketName) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::CreateBucket", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } // Validate input parameters if (bucketName.empty()) { _logger.LogError("ANSAWSS3::CreateBucket", "Bucket name cannot be empty", __FILE__, __LINE__); return false; } bool createSuccess = false; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("CreateBucket", "Failed to acquire S3 connection", __FILE__, __LINE__); return false; } // Build global endpoint std::string globalEndpoint = _serviceName + "." + _baseDomain; CkStringBuilder sbBucketRegion; sbBucketRegion.Append(_bucketRegion.c_str()); // We only need to specify the LocationConstraint if the bucket's region is NOT us-east-1 CkXml xml; if (!sbBucketRegion.ContentsEqual("us-east-1", true)) { xml.put_Tag("CreateBucketConfiguration"); xml.AddAttribute("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/"); xml.UpdateChildContent("LocationConstraint", _bucketRegion.c_str()); } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + globalEndpoint).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // Make the call to create the bucket std::string createPath = _bAwsPath ? "/" : ("/" + bucketName); const char* responseStr = nullptr; if (!sbBucketRegion.ContentsEqual("us-east-1", true)) { responseStr = conn->rest.fullRequestString("PUT", createPath.c_str(), xml.getXml()); } else { // If the bucket is to be created in the us-east-1 region (the default region) // just send a PUT with no body responseStr = conn->rest.fullRequestNoBody("PUT", createPath.c_str()); } if (conn->rest.get_LastMethodSuccess() != true) { _logger.LogError("ANSAWSS3::CreateBucket - Request failed", conn->rest.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode == 200) { _logger.LogDebug("ANSAWSS3::CreateBucket", "Successfully created bucket: " + bucketName + " in region: " + _bucketRegion, __FILE__, __LINE__); createSuccess = true; } else { std::string errorMsg = "HTTP " + std::to_string(statusCode); if (responseStr != nullptr && responseStr[0] != '\0') { CkXml responseXml; if (responseXml.LoadXml(responseStr)) { const char* errorCode = responseXml.getChildContent("Code"); const char* errorMessage = responseXml.getChildContent("Message"); if (errorCode != nullptr) { errorMsg += " - " + std::string(errorCode); } if (errorMessage != nullptr) { errorMsg += ": " + std::string(errorMessage); } } else { errorMsg += "\n" + std::string(responseStr); } } if (statusCode == 409) { _logger.LogError("ANSAWSS3::CreateBucket", errorMsg, __FILE__, __LINE__); } else { _logger.LogError("ANSAWSS3::CreateBucket", errorMsg, __FILE__, __LINE__); } } ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::CreateBucket", std::string("Exception: ") + e.what(), __FILE__, __LINE__); } return createSuccess; } bool ANSAWSS3::DeleteBucket(const std::string& bucketName) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::DeleteBucket", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty()) { _logger.LogError("ANSAWSS3::DeleteBucket", "Bucket name cannot be empty", __FILE__, __LINE__); return false; } bool deleteSuccess = false; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("DeleteBucket", "Failed to acquire S3 connection", __FILE__, __LINE__); return false; } // Configure AWS authentication conn->rest.ClearAuth(); conn->authAws.put_AccessKey(_accessKey.c_str()); conn->authAws.put_SecretKey(_secretKey.c_str()); conn->authAws.put_ServiceName(_serviceName.c_str()); bool authSuccess = conn->rest.SetAuthAws(conn->authAws); if (!authSuccess) { _logger.LogError("ANSAWSS3::DeleteBucket", "Failed to set AWS authentication: " + std::string(conn->rest.lastErrorText()), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } // Set bucket-specific endpoint std::string globalEndpoint = _serviceName + "." + _baseDomain; if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + globalEndpoint).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } std::string deletePath = _bAwsPath ? "/" : ("/" + bucketName); CkStringBuilder sbResponse; bool success = conn->rest.FullRequestNoBodySb("DELETE", deletePath.c_str(), sbResponse); if (!success) { _logger.LogError("ANSAWSS3::DeleteBucket", "Failed to send DELETE request: " + std::string(conn->rest.lastErrorText()), __FILE__, __LINE__); } else { int statusCode = conn->rest.get_ResponseStatusCode(); // S3 returns 204 (No Content) for successful bucket deletion if (statusCode == 204) { _logger.LogDebug("ANSAWSS3::DeleteBucket", "Successfully deleted bucket: " + bucketName, __FILE__, __LINE__); deleteSuccess = true; } else if (statusCode == 404) { _logger.LogWarn("ANSAWSS3::DeleteBucket", "Bucket not found (already deleted?): " + bucketName, __FILE__, __LINE__); } else if (statusCode == 409) { // Bucket not empty std::string errorMsg = "HTTP 409 - Bucket is not empty and cannot be deleted"; std::string response = sbResponse.getAsString(); if (!response.empty()) { CkXml responseXml; if (responseXml.LoadXml(response.c_str())) { const char* errorCode = responseXml.getChildContent("Code"); const char* errorMessage = responseXml.getChildContent("Message"); if (errorCode != nullptr) { errorMsg += " - " + std::string(errorCode); } if (errorMessage != nullptr) { errorMsg += ": " + std::string(errorMessage); } } } _logger.LogError("ANSAWSS3::DeleteBucket", errorMsg, __FILE__, __LINE__); } else { std::string errorMsg = "HTTP " + std::to_string(statusCode); std::string response = sbResponse.getAsString(); if (!response.empty()) { errorMsg += " - " + response; } _logger.LogError("ANSAWSS3::DeleteBucket", errorMsg, __FILE__, __LINE__); } } ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::DeleteBucket", std::string("Exception: ") + e.what(), __FILE__, __LINE__); } return deleteSuccess; } bool ANSAWSS3::CreateFolder(const std::string& bucketName, const std::string& prefix) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::CreateFolder", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || prefix.empty()) { _logger.LogError("ANSAWSS3::CreateFolder", "Bucket name or folder path is empty", __FILE__, __LINE__); return false; } bool createSuccess = false; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("CreateFolder", "Failed to acquire S3 connection", __FILE__, __LINE__); return false; } // Ensure folder path ends with "/" std::string normalizedPath = prefix; if (normalizedPath.back() != '/') { normalizedPath += '/'; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // Object path is the folder name with trailing slash std::string objectPath = _bAwsPath ? ("/" + normalizedPath) : ("/" + bucketName + "/" + normalizedPath); // Create a zero-byte object to represent the folder // Use PUT with empty body const char* responseStr = conn->rest.fullRequestNoBody("PUT", objectPath.c_str()); if (conn->rest.get_LastMethodSuccess() != true) { _logger.LogError("ANSAWSS3::CreateFolder - Request failed", conn->rest.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode == 200) { _logger.LogDebug("ANSAWSS3::CreateFolder", "Successfully created folder: " + normalizedPath + " in bucket: " + bucketName, __FILE__, __LINE__); createSuccess = true; } else { std::string errorMsg = "HTTP " + std::to_string(statusCode); if (responseStr != nullptr && responseStr[0] != '\0') { CkXml responseXml; if (responseXml.LoadXml(responseStr)) { const char* errorCode = responseXml.getChildContent("Code"); const char* errorMessage = responseXml.getChildContent("Message"); if (errorCode != nullptr) { errorMsg += " - " + std::string(errorCode); } if (errorMessage != nullptr) { errorMsg += ": " + std::string(errorMessage); } } else { errorMsg += " - " + std::string(responseStr); } } _logger.LogError("ANSAWSS3::CreateFolder", errorMsg, __FILE__, __LINE__); } ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::CreateFolder", std::string("Exception: ") + e.what(), __FILE__, __LINE__); } return createSuccess; } bool ANSAWSS3::DeleteFolder(const std::string& bucketName, const std::string& prefix) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::DeleteFolder", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || prefix.empty()) { _logger.LogError("ANSAWSS3::DeleteFolder", "Bucket name or folder path is empty", __FILE__, __LINE__); return false; } bool deleteSuccess = false; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("DeleteFolder", "Failed to acquire S3 connection", __FILE__, __LINE__); return false; } // Ensure folder path ends with "/" std::string normalizedPath = prefix; if (normalizedPath.back() != '/') { normalizedPath += '/'; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } std::string objectPath = _bAwsPath ? ("/" + normalizedPath) : ("/" + bucketName + "/" + normalizedPath); CkStringBuilder sbResponse; bool success = conn->rest.FullRequestNoBodySb("DELETE", objectPath.c_str(), sbResponse); if (!success) { _logger.LogError("ANSAWSS3::DeleteFolder", "Failed to send DELETE request: " + std::string(conn->rest.lastErrorText()), __FILE__, __LINE__); } else { int statusCode = conn->rest.get_ResponseStatusCode(); // S3 returns 204 (No Content) for successful deletion if (statusCode == 204 || statusCode == 200) { _logger.LogDebug("ANSAWSS3::DeleteFolder", "Successfully deleted folder marker: " + normalizedPath + " from bucket: " + bucketName, __FILE__, __LINE__); deleteSuccess = true; } else if (statusCode == 404) { _logger.LogWarn("ANSAWSS3::DeleteFolder", "Folder marker not found (already deleted?): " + normalizedPath, __FILE__, __LINE__); } else { std::string errorMsg = "HTTP " + std::to_string(statusCode); std::string response = sbResponse.getAsString(); if (!response.empty()) { errorMsg += " - " + response; } _logger.LogError("ANSAWSS3::DeleteFolder", errorMsg, __FILE__, __LINE__); } } ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::DeleteFolder", std::string("Exception: ") + e.what(), __FILE__, __LINE__); } return deleteSuccess; } std::string ANSAWSS3::GetBucketRegion(const std::string& bucketName) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::GetBucketRegion", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return ""; } if (bucketName.empty()) { _logger.LogError("ANSAWSS3::GetBucketRegion", "Bucket name is empty", __FILE__, __LINE__); return ""; } std::string region = ""; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("GetBucketRegion", "Failed to acquire S3 connection", __FILE__, __LINE__); return ""; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // Send the GET request to query the bucket location std::string locationPath = _bAwsPath ? "/?location" : ("/" + bucketName + "?location"); const char* strResult = conn->rest.fullRequestNoBody("GET", locationPath.c_str()); if (conn->rest.get_LastMethodSuccess() != true) { _logger.LogError("ANSAWSS3::GetBucketRegion - Request failed", conn->rest.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return ""; } int responseStatusCode = conn->rest.get_ResponseStatusCode(); if (responseStatusCode != 200) { std::string errorMsg = "HTTP " + std::to_string(responseStatusCode); if (strResult != nullptr && strResult[0] != '\0') { errorMsg += " - " + std::string(strResult); } _logger.LogError("ANSAWSS3::GetBucketRegion", errorMsg, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return ""; } // Parse XML response CkXml xml; bool success = xml.LoadXml(strResult); if (!success) { _logger.LogError("ANSAWSS3::GetBucketRegion - Failed to parse XML", xml.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return ""; } // Get the region from XML content const char* regionContent = xml.content(); if (regionContent != nullptr && regionContent[0] != '\0') { region = std::string(regionContent); // AWS returns empty string for us-east-1 (classic region) // Some implementations may return "null" or empty if (region.empty() || region == "null") { region = "us-east-1"; _logger.LogDebug("ANSAWSS3::GetBucketRegion", "Bucket '" + bucketName + "' is in default region: us-east-1", __FILE__, __LINE__); } else { _logger.LogDebug("ANSAWSS3::GetBucketRegion", "Bucket '" + bucketName + "' is in region: " + region, __FILE__, __LINE__); } } else { // Empty response typically means us-east-1 region = "us-east-1"; _logger.LogDebug("ANSAWSS3::GetBucketRegion", "Empty response - assuming default region: us-east-1", __FILE__, __LINE__); } ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::GetBucketRegion", std::string("Exception: ") + e.what(), __FILE__, __LINE__); } return region; } // Uploads text data from a file to the specified S3 bucket bool ANSAWSS3::UploadTextData(const std::string& bucketName, const std::string& textFilePath, std::string& uploadedFilePath) { const std::string kOp = "ANSAWSS3::UploadTextData"; // _logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__); // Early validation (permanent — do NOT retry) if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError(kOp, !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || textFilePath.empty()) { _logger.LogError(kOp, "Bucket name or file path is empty", __FILE__, __LINE__); return false; } // Filename and content type are attempt-independent. const std::string fileName = ExtractFileName(textFilePath); if (fileName.empty()) { _logger.LogError(kOp, "Failed to extract filename from path: " + textFilePath, __FILE__, __LINE__); return false; } const std::string contentType = GetContentType(textFilePath); const std::string objectPath = _bAwsPath ? ("/" + fileName) : ("/" + bucketName + "/" + fileName); return UploadWithRetry(kOp, [&](std::string& lastError) -> AttemptResult { auto conn = AcquireConnection(); if (!conn) { lastError = "Failed to acquire S3 connection"; return AttemptResult::Transient; } if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // Read the file on each attempt (cheap; keeps the body fresh). CkFileAccess fac; const char* fileContents = fac.readEntireTextFile(textFilePath.c_str(), "utf-8"); if (!fac.get_LastMethodSuccess()) { // File read failure is permanent (disk/IO, not network). lastError = std::string("Failed to read file: ") + fac.lastErrorText(); ReleaseConnection(std::move(conn)); return AttemptResult::Permanent; } if (fileContents == nullptr || fileContents[0] == '\0') { lastError = "File is empty: " + textFilePath; ReleaseConnection(std::move(conn)); return AttemptResult::Permanent; } conn->rest.AddHeader("Content-Type", contentType.c_str()); conn->rest.AddHeader("Content-Encoding", "gzip"); conn->rest.AddHeader("Expect", "100-continue"); const char* responseBodyStr = conn->rest.fullRequestString("PUT", objectPath.c_str(), fileContents); if (!conn->rest.get_LastMethodSuccess()) { lastError = conn->rest.lastErrorText(); ReleaseConnection(std::move(conn)); return AttemptResult::Transient; } int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode == 200) { std::string scheme = _bTls ? "https://" : "http://"; uploadedFilePath = scheme + _fullAWSURL + "/" + bucketName + "/" + fileName; _logger.LogDebug(kOp, "Successfully uploaded: " + fileName + " | URL: " + uploadedFilePath, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return AttemptResult::Success; } lastError = "HTTP " + std::to_string(statusCode); if (responseBodyStr != nullptr && responseBodyStr[0] != '\0') { lastError += " - " + std::string(responseBodyStr); } ReleaseConnection(std::move(conn)); bool retryable = (statusCode >= 500 && statusCode <= 599) || statusCode == 408 || statusCode == 429; return retryable ? AttemptResult::Transient : AttemptResult::Permanent; }); } bool ANSAWSS3::UploadBinaryData(const std::string& bucketName, const std::string& dataFilePath, std::string& uploadedFilePath) { const std::string kOp = "ANSAWSS3::UploadBinaryData"; // _logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__); // Early validation (permanent — do NOT retry) if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError(kOp, !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || dataFilePath.empty()) { _logger.LogError(kOp, "Bucket name or file path is empty", __FILE__, __LINE__); return false; } const std::string fileName = ExtractFileName(dataFilePath); if (fileName.empty()) { _logger.LogError(kOp, "Failed to extract filename from path: " + dataFilePath, __FILE__, __LINE__); return false; } const std::string contentType = GetContentType(dataFilePath); const std::string objectPath = _bAwsPath ? ("/" + fileName) : ("/" + bucketName + "/" + fileName); return UploadWithRetry(kOp, [&](std::string& lastError) -> AttemptResult { auto conn = AcquireConnection(); if (!conn) { lastError = "Failed to acquire S3 connection"; return AttemptResult::Transient; } if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } CkBinData binData; if (!binData.LoadFile(dataFilePath.c_str())) { lastError = std::string("Failed to load file: ") + binData.lastErrorText(); ReleaseConnection(std::move(conn)); return AttemptResult::Permanent; // disk/IO, not network } conn->rest.AddHeader("Content-Type", contentType.c_str()); conn->rest.AddHeader("Expect", "100-continue"); CkStringBuilder sbResponse; if (!conn->rest.FullRequestBd("PUT", objectPath.c_str(), binData, sbResponse)) { lastError = conn->rest.lastErrorText(); ReleaseConnection(std::move(conn)); return AttemptResult::Transient; } int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode == 200) { std::string scheme = _bTls ? "https://" : "http://"; uploadedFilePath = scheme + _fullAWSURL + "/" + bucketName + "/" + fileName; _logger.LogDebug(kOp, "Successfully uploaded: " + fileName + " | URL: " + uploadedFilePath, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return AttemptResult::Success; } lastError = "HTTP " + std::to_string(statusCode); std::string body = sbResponse.getAsString(); if (!body.empty()) lastError += " - " + body; ReleaseConnection(std::move(conn)); bool retryable = (statusCode >= 500 && statusCode <= 599) || statusCode == 408 || statusCode == 429; return retryable ? AttemptResult::Transient : AttemptResult::Permanent; }); } bool ANSAWSS3::UploadPrefixBinaryData(const std::string& bucketName, const std::string& prefix, const std::string& dataFilePath, const std::string& objectName, std::string& uploadedFilePath) { const std::string kOp = "ANSAWSS3::UploadPrefixBinaryData"; //_logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__); // Early validation (permanent — do NOT retry) if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError(kOp, !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || dataFilePath.empty()) { _logger.LogError(kOp, "Bucket name or file path is empty", __FILE__, __LINE__); return false; } // Resolve filename (from objectName, fall back to path). std::string fileName = objectName; if (fileName.empty()) { fileName = ExtractFileName(dataFilePath); if (fileName.empty()) { _logger.LogError(kOp, "Failed to extract filename from path: " + dataFilePath, __FILE__, __LINE__); return false; } } // Build once: the object key and the HTTP request path. Keeping // them separate avoids the path-style bucket-duplication bug in // the returned URL. std::string objectKey; // "/" — no leading '/' if (!prefix.empty()) { std::string normalizedPrefix = prefix; if (!normalizedPrefix.empty() && normalizedPrefix.front() == '/') { normalizedPrefix = normalizedPrefix.substr(1); } if (!normalizedPrefix.empty() && normalizedPrefix.back() != '/') { normalizedPrefix += '/'; } objectKey = normalizedPrefix; } objectKey += fileName; const std::string objectPath = _bAwsPath ? ("/" + objectKey) : ("/" + bucketName + "/" + objectKey); const std::string contentType = GetContentType(dataFilePath); return UploadWithRetry(kOp, [&](std::string& lastError) -> AttemptResult { auto conn = AcquireConnection(); if (!conn) { lastError = "Failed to acquire S3 connection"; return AttemptResult::Transient; } if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } CkBinData binData; if (!binData.LoadFile(dataFilePath.c_str())) { lastError = std::string("Failed to load file: ") + binData.lastErrorText(); ReleaseConnection(std::move(conn)); return AttemptResult::Permanent; } conn->rest.AddHeader("Content-Type", contentType.c_str()); conn->rest.AddHeader("Expect", "100-continue"); CkStringBuilder sbResponse; if (!conn->rest.FullRequestBd("PUT", objectPath.c_str(), binData, sbResponse)) { lastError = conn->rest.lastErrorText(); ReleaseConnection(std::move(conn)); return AttemptResult::Transient; } int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode == 200) { // Build canonical path-style URL from parts — independent // of the addressing style we actually used. std::string scheme = _bTls ? "https://" : "http://"; uploadedFilePath = scheme + _fullAWSURL + "/" + bucketName + "/" + objectKey; _logger.LogDebug(kOp, "Successfully uploaded to: " + objectKey + " | URL: " + uploadedFilePath, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return AttemptResult::Success; } lastError = "HTTP " + std::to_string(statusCode); std::string body = sbResponse.getAsString(); if (!body.empty()) lastError += " - " + body; ReleaseConnection(std::move(conn)); bool retryable = (statusCode >= 500 && statusCode <= 599) || statusCode == 408 || statusCode == 429; return retryable ? AttemptResult::Transient : AttemptResult::Permanent; }); } bool ANSAWSS3::UploadFileStream(const std::string& bucketName, const std::string& dataFilePath, std::string& uploadedFilePath) { const std::string kOp = "ANSAWSS3::UploadFileStream"; //_logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__); // Early validation (permanent — do NOT retry) if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError(kOp, !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || dataFilePath.empty()) { _logger.LogError(kOp, "Bucket name or file path is empty", __FILE__, __LINE__); return false; } const std::string fileName = ExtractFileName(dataFilePath); const std::string objectPath = _bAwsPath ? ("/" + fileName) : ("/" + bucketName + "/" + fileName); return UploadWithRetry(kOp, [&](std::string& lastError) -> AttemptResult { auto conn = AcquireConnection(); if (!conn) { lastError = "Failed to acquire S3 connection"; return AttemptResult::Transient; } if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // Fresh stream per attempt (it holds OS file handle state). CkStream fileStream; fileStream.put_SourceFile(dataFilePath.c_str()); const char* responseStr = conn->rest.fullRequestStream("PUT", objectPath.c_str(), fileStream); if (conn->rest.get_LastMethodSuccess() != true) { lastError = conn->rest.lastErrorText(); ReleaseConnection(std::move(conn)); return AttemptResult::Transient; } int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode == 200) { std::string scheme = _bTls ? "https://" : "http://"; uploadedFilePath = scheme + _fullAWSURL + "/" + bucketName + "/" + fileName; _logger.LogDebug(kOp, "Successfully uploaded file: " + fileName + " | URL: " + uploadedFilePath, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return AttemptResult::Success; } lastError = "HTTP " + std::to_string(statusCode); if (responseStr != nullptr && responseStr[0] != '\0') { lastError += " - " + std::string(responseStr); } ReleaseConnection(std::move(conn)); bool retryable = (statusCode >= 500 && statusCode <= 599) || statusCode == 408 || statusCode == 429; return retryable ? AttemptResult::Transient : AttemptResult::Permanent; }); } bool ANSAWSS3::UploadMultipartData(const std::string& bucketName,const std::string& dataFilePath, std::string& uploadedFilePath, int partSize) { const std::string kOp = "ANSAWSS3::UploadMultipartData"; //_logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__); // Early validation (permanent — do NOT retry) if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError(kOp, !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || dataFilePath.empty()) { _logger.LogError(kOp, "Bucket name or file path is empty", __FILE__, __LINE__); return false; } if (partSize < 5242880) { _logger.LogError(kOp, "Part size must be at least 5MB (5242880 bytes)", __FILE__, __LINE__); return false; } { CkFileAccess fac; if (!fac.FileExists(dataFilePath.c_str())) { _logger.LogError(kOp, "File not found: " + dataFilePath, __FILE__, __LINE__); return false; } } const std::string fileName = ExtractFileName(dataFilePath); if (fileName.empty()) { _logger.LogError(kOp, "Failed to extract filename from path: " + dataFilePath, __FILE__, __LINE__); return false; } const std::string objectPath = _bAwsPath ? ("/" + fileName) : ("/" + bucketName + "/" + fileName); const std::string fileFolderPath = dataFilePath.substr(0, dataFilePath.find_last_of("/\\")); // Fix 2: include a per-call unique token in the scratch-file name so // two parallel multipart uploads of the same local file don't share // the same partsList XML (which would race on create/delete). const std::string partsListFilePath = fileFolderPath + "/partsList_" + fileName + "_" + MakeUniqueMultipartToken(this) + ".xml"; // Note on retry semantics for multipart: // Each attempt performs a fresh Initiate → Parts → Complete cycle. We // reset the on-disk partsList XML at the start of each attempt because // ETags returned by a prior attempt are bound to that attempt's // UploadId and are not valid for a new Initiate. Parts uploaded during // a failed attempt therefore become orphans on S3 until the service's // multipart-lifecycle rule cleans them up (default 7 days, configurable). return UploadWithRetry(kOp, [&](std::string& lastError) -> AttemptResult { CkFileAccess fac; // Ensure no stale parts list from a prior attempt. if (fac.FileExists(partsListFilePath.c_str())) { fac.FileDelete(partsListFilePath.c_str()); } auto conn = AcquireConnection(); if (!conn) { lastError = "Failed to acquire S3 connection"; return AttemptResult::Transient; } if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // ---- STEP 1: INITIATE ---- conn->rest.ClearAllQueryParams(); conn->rest.AddQueryParam("uploads", ""); const char* responseXml = conn->rest.fullRequestNoBody("POST", objectPath.c_str()); if (!conn->rest.get_LastMethodSuccess()) { lastError = std::string("Initiate failed: ") + conn->rest.lastErrorText(); ReleaseConnection(std::move(conn)); return AttemptResult::Transient; } int initStatus = conn->rest.get_ResponseStatusCode(); if (initStatus != 200) { lastError = "Initiate HTTP " + std::to_string(initStatus); if (responseXml != nullptr && responseXml[0] != '\0') lastError += " - " + std::string(responseXml); ReleaseConnection(std::move(conn)); bool retryable = (initStatus >= 500 && initStatus <= 599) || initStatus == 408 || initStatus == 429; return retryable ? AttemptResult::Transient : AttemptResult::Permanent; } CkXml xmlInit; if (!xmlInit.LoadXml(responseXml)) { lastError = std::string("Initiate XML parse error: ") + xmlInit.lastErrorText(); ReleaseConnection(std::move(conn)); return AttemptResult::Transient; } const char* uploadId = xmlInit.getChildContent("UploadId"); if (uploadId == nullptr || uploadId[0] == '\0') { lastError = "UploadId not found in Initiate response"; ReleaseConnection(std::move(conn)); return AttemptResult::Transient; } _logger.LogDebug(kOp, "Multipart initiated. UploadId: " + std::string(uploadId), __FILE__, __LINE__); // ---- STEP 2: UPLOAD PARTS ---- fac.OpenForRead(dataFilePath.c_str()); int numParts = fac.GetNumBlocks(partSize); fac.FileClose(); CkXml partsListXml; partsListXml.put_Tag("CompleteMultipartUpload"); CkStringBuilder sbPartNumber; for (int partNumber = 1; partNumber <= numParts; ++partNumber) { sbPartNumber.Clear(); sbPartNumber.AppendInt(partNumber); CkStream fileStream; fileStream.put_SourceFile(dataFilePath.c_str()); fileStream.put_SourceFilePartSize(partSize); fileStream.put_SourceFilePart(partNumber - 1); conn->rest.ClearAllQueryParams(); conn->rest.AddQueryParam("partNumber", sbPartNumber.getAsString()); conn->rest.AddQueryParam("uploadId", uploadId); const char* responseStr = conn->rest.fullRequestStream("PUT", objectPath.c_str(), fileStream); if (!conn->rest.get_LastMethodSuccess()) { lastError = "Part " + std::to_string(partNumber) + " failed: " + conn->rest.lastErrorText(); ReleaseConnection(std::move(conn)); return AttemptResult::Transient; } int partStatus = conn->rest.get_ResponseStatusCode(); if (partStatus != 200) { lastError = "Part " + std::to_string(partNumber) + " HTTP " + std::to_string(partStatus); if (responseStr != nullptr && responseStr[0] != '\0') lastError += " - " + std::string(responseStr); ReleaseConnection(std::move(conn)); bool retryable = (partStatus >= 500 && partStatus <= 599) || partStatus == 408 || partStatus == 429; return retryable ? AttemptResult::Transient : AttemptResult::Permanent; } const char* etag = conn->rest.responseHdrByName("ETag"); if (!conn->rest.get_LastMethodSuccess() || etag == nullptr || etag[0] == '\0') { lastError = "Part " + std::to_string(partNumber) + ": ETag missing from response"; ReleaseConnection(std::move(conn)); return AttemptResult::Transient; } CkXml* xPart = partsListXml.NewChild("Part", ""); xPart->NewChildInt2("PartNumber", partNumber); xPart->NewChild2("ETag", etag); delete xPart; // Persist partsList for diagnostics; not used for resume in this attempt. partsListXml.SaveXml(partsListFilePath.c_str()); } // ---- STEP 3: COMPLETE ---- conn->rest.ClearAllQueryParams(); conn->rest.AddQueryParam("uploadId", uploadId); responseXml = conn->rest.fullRequestString("POST", objectPath.c_str(), partsListXml.getXml()); if (!conn->rest.get_LastMethodSuccess()) { lastError = std::string("Complete failed: ") + conn->rest.lastErrorText(); ReleaseConnection(std::move(conn)); return AttemptResult::Transient; } int completeStatus = conn->rest.get_ResponseStatusCode(); if (completeStatus != 200) { lastError = "Complete HTTP " + std::to_string(completeStatus); if (responseXml != nullptr && responseXml[0] != '\0') lastError += " - " + std::string(responseXml); ReleaseConnection(std::move(conn)); bool retryable = (completeStatus >= 500 && completeStatus <= 599) || completeStatus == 408 || completeStatus == 429; return retryable ? AttemptResult::Transient : AttemptResult::Permanent; } // Cleanup parts list on success. if (fac.FileExists(partsListFilePath.c_str())) { fac.FileDelete(partsListFilePath.c_str()); } std::string scheme = _bTls ? "https://" : "http://"; uploadedFilePath = scheme + _fullAWSURL + "/" + bucketName + "/" + fileName; _logger.LogDebug(kOp, "Multipart upload completed for: " + fileName + " | URL: " + uploadedFilePath, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return AttemptResult::Success; }); } bool ANSAWSS3::UploadPrefixMultipartData(const std::string& bucketName, const std::string& prefix,const std::string& dataFilePath, const std::string& objectName, std::string& uploadedFilePath, int partSize) { const std::string kOp = "ANSAWSS3::UploadPrefixMultipartData"; //_logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__); // Early validation (permanent — do NOT retry) if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError(kOp, !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || dataFilePath.empty()) { _logger.LogError(kOp, "Bucket name or file path is empty", __FILE__, __LINE__); return false; } if (partSize < 5242880) { _logger.LogError(kOp, "Part size must be at least 5MB (5242880 bytes)", __FILE__, __LINE__); return false; } { CkFileAccess fac; if (!fac.FileExists(dataFilePath.c_str())) { _logger.LogError(kOp, "File not found: " + dataFilePath, __FILE__, __LINE__); return false; } } std::string fileName = objectName; if (fileName.empty()) { fileName = ExtractFileName(dataFilePath); if (fileName.empty()) { _logger.LogError(kOp, "Failed to extract filename from path: " + dataFilePath, __FILE__, __LINE__); return false; } } // Build once: the object key and the HTTP request path. Keeping // them separate avoids the path-style bucket-duplication bug in // the returned URL. std::string objectKey; // "/" — no leading '/' if (!prefix.empty()) { std::string normalizedPrefix = prefix; if (!normalizedPrefix.empty() && normalizedPrefix.front() == '/') { normalizedPrefix = normalizedPrefix.substr(1); } if (!normalizedPrefix.empty() && normalizedPrefix.back() != '/') { normalizedPrefix += '/'; } objectKey = normalizedPrefix; } objectKey += fileName; const std::string objectPath = _bAwsPath ? ("/" + objectKey) : ("/" + bucketName + "/" + objectKey); // Parts-list file uses a sanitized filename for safe local storage. // Fix 2: also append a per-call unique token so parallel multipart // uploads of the same local file don't share the same scratch file. const std::string fileFolderPath = dataFilePath.substr(0, dataFilePath.find_last_of("/\\")); std::string safeFileName = fileName; std::replace(safeFileName.begin(), safeFileName.end(), '/', '_'); std::replace(safeFileName.begin(), safeFileName.end(), '\\', '_'); const std::string partsListFilePath = fileFolderPath + "/partsList_" + safeFileName + "_" + MakeUniqueMultipartToken(this) + ".xml"; // See note in UploadMultipartData regarding multipart retry semantics. return UploadWithRetry(kOp, [&](std::string& lastError) -> AttemptResult { CkFileAccess fac; if (fac.FileExists(partsListFilePath.c_str())) { fac.FileDelete(partsListFilePath.c_str()); } auto conn = AcquireConnection(); if (!conn) { lastError = "Failed to acquire S3 connection"; return AttemptResult::Transient; } if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // ---- STEP 1: INITIATE ---- conn->rest.ClearAllQueryParams(); conn->rest.AddQueryParam("uploads", ""); const char* responseXml = conn->rest.fullRequestNoBody("POST", objectPath.c_str()); if (!conn->rest.get_LastMethodSuccess()) { lastError = std::string("Initiate failed: ") + conn->rest.lastErrorText(); ReleaseConnection(std::move(conn)); return AttemptResult::Transient; } int initStatus = conn->rest.get_ResponseStatusCode(); if (initStatus != 200) { lastError = "Initiate HTTP " + std::to_string(initStatus); if (responseXml != nullptr && responseXml[0] != '\0') lastError += " - " + std::string(responseXml); ReleaseConnection(std::move(conn)); bool retryable = (initStatus >= 500 && initStatus <= 599) || initStatus == 408 || initStatus == 429; return retryable ? AttemptResult::Transient : AttemptResult::Permanent; } CkXml xmlInit; if (!xmlInit.LoadXml(responseXml)) { lastError = std::string("Initiate XML parse error: ") + xmlInit.lastErrorText(); ReleaseConnection(std::move(conn)); return AttemptResult::Transient; } const char* uploadId = xmlInit.getChildContent("UploadId"); if (uploadId == nullptr || uploadId[0] == '\0') { lastError = "UploadId not found in Initiate response"; ReleaseConnection(std::move(conn)); return AttemptResult::Transient; } _logger.LogDebug(kOp, "Multipart initiated. UploadId: " + std::string(uploadId), __FILE__, __LINE__); // ---- STEP 2: UPLOAD PARTS ---- fac.OpenForRead(dataFilePath.c_str()); int numParts = fac.GetNumBlocks(partSize); fac.FileClose(); CkXml partsListXml; partsListXml.put_Tag("CompleteMultipartUpload"); CkStringBuilder sbPartNumber; for (int partNumber = 1; partNumber <= numParts; ++partNumber) { sbPartNumber.Clear(); sbPartNumber.AppendInt(partNumber); CkStream fileStream; fileStream.put_SourceFile(dataFilePath.c_str()); fileStream.put_SourceFilePartSize(partSize); fileStream.put_SourceFilePart(partNumber - 1); conn->rest.ClearAllQueryParams(); conn->rest.AddQueryParam("partNumber", sbPartNumber.getAsString()); conn->rest.AddQueryParam("uploadId", uploadId); const char* responseStr = conn->rest.fullRequestStream("PUT", objectPath.c_str(), fileStream); if (!conn->rest.get_LastMethodSuccess()) { lastError = "Part " + std::to_string(partNumber) + " failed: " + conn->rest.lastErrorText(); ReleaseConnection(std::move(conn)); return AttemptResult::Transient; } int partStatus = conn->rest.get_ResponseStatusCode(); if (partStatus != 200) { lastError = "Part " + std::to_string(partNumber) + " HTTP " + std::to_string(partStatus); if (responseStr != nullptr && responseStr[0] != '\0') lastError += " - " + std::string(responseStr); ReleaseConnection(std::move(conn)); bool retryable = (partStatus >= 500 && partStatus <= 599) || partStatus == 408 || partStatus == 429; return retryable ? AttemptResult::Transient : AttemptResult::Permanent; } const char* etag = conn->rest.responseHdrByName("ETag"); if (!conn->rest.get_LastMethodSuccess() || etag == nullptr || etag[0] == '\0') { lastError = "Part " + std::to_string(partNumber) + ": ETag missing from response"; ReleaseConnection(std::move(conn)); return AttemptResult::Transient; } CkXml* xPart = partsListXml.NewChild("Part", ""); xPart->NewChildInt2("PartNumber", partNumber); xPart->NewChild2("ETag", etag); delete xPart; partsListXml.SaveXml(partsListFilePath.c_str()); } // ---- STEP 3: COMPLETE ---- conn->rest.ClearAllQueryParams(); conn->rest.AddQueryParam("uploadId", uploadId); responseXml = conn->rest.fullRequestString("POST", objectPath.c_str(), partsListXml.getXml()); if (!conn->rest.get_LastMethodSuccess()) { lastError = std::string("Complete failed: ") + conn->rest.lastErrorText(); ReleaseConnection(std::move(conn)); return AttemptResult::Transient; } int completeStatus = conn->rest.get_ResponseStatusCode(); if (completeStatus != 200) { lastError = "Complete HTTP " + std::to_string(completeStatus); if (responseXml != nullptr && responseXml[0] != '\0') lastError += " - " + std::string(responseXml); ReleaseConnection(std::move(conn)); bool retryable = (completeStatus >= 500 && completeStatus <= 599) || completeStatus == 408 || completeStatus == 429; return retryable ? AttemptResult::Transient : AttemptResult::Permanent; } if (fac.FileExists(partsListFilePath.c_str())) { fac.FileDelete(partsListFilePath.c_str()); } // Build canonical path-style URL from parts — independent // of the addressing style we actually used. std::string scheme = _bTls ? "https://" : "http://"; uploadedFilePath = scheme + _fullAWSURL + "/" + bucketName + "/" + objectKey; _logger.LogDebug(kOp, "Multipart upload completed. S3 key: " + objectKey + " | URL: " + uploadedFilePath, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return AttemptResult::Success; }); } // Upload jpeg data bool ANSAWSS3::UploadJpegImage(const std::string& bucketName, unsigned char* jpeg_string, int32 bufferLength, const std::string& fileName, std::string& uploadedFilePath) { const std::string kOp = "ANSAWSS3::UploadJpegImage"; // _logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__); // Early validation checks (permanent — do NOT retry) if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError(kOp, !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || bufferLength <= 0) { _logger.LogError(kOp, "Bucket name or buffer length is invalid", __FILE__, __LINE__); return false; } // Object path is attempt-independent. const std::string objectPath = _bAwsPath ? ("/" + fileName) : ("/" + bucketName + "/" + fileName); return UploadWithRetry(kOp, [&](std::string& lastError) -> AttemptResult { auto conn = AcquireConnection(); if (!conn) { lastError = "Failed to acquire S3 connection"; return AttemptResult::Transient; } if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } CkBinData binData; CkByteData jpegBytes; jpegBytes.append2(jpeg_string, static_cast(bufferLength)); binData.AppendBinary(jpegBytes); conn->rest.AddHeader("Content-Type", "image/jpeg"); conn->rest.AddHeader("Expect", "100-continue"); CkStringBuilder sbResponse; if (!conn->rest.FullRequestBd("PUT", objectPath.c_str(), binData, sbResponse)) { lastError = conn->rest.lastErrorText(); ReleaseConnection(std::move(conn)); return AttemptResult::Transient; } int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode == 200) { std::string scheme = _bTls ? "https://" : "http://"; uploadedFilePath = scheme + _fullAWSURL + "/" + bucketName + "/" + fileName; _logger.LogDebug(kOp, "Successfully uploaded: " + fileName + " | URL: " + uploadedFilePath, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return AttemptResult::Success; } lastError = "HTTP " + std::to_string(statusCode); std::string body = sbResponse.getAsString(); if (!body.empty()) lastError += " - " + body; ReleaseConnection(std::move(conn)); bool retryable = (statusCode >= 500 && statusCode <= 599) || statusCode == 408 || statusCode == 429; return retryable ? AttemptResult::Transient : AttemptResult::Permanent; }); } bool ANSAWSS3::UploadPrefixJpegImage(const std::string& bucketName, const std::string& prefix,unsigned char* jpeg_string,int32 bufferLength,const std::string& fileName, std::string& uploadedFilePath) { const std::string kOp = "ANSAWSS3::UploadPrefixJpegImage"; //_logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__); // Early validation checks (permanent failures — do NOT retry) if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError(kOp, !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || bufferLength <= 0 || fileName.empty()) { _logger.LogError(kOp, "Bucket name, buffer length, or filename is invalid", __FILE__, __LINE__); return false; } // Build once: the object key (path within the bucket) and the HTTP // request path (which depends on addressing style). Keeping these // separate avoids the path-style bucket-duplication bug in the // returned URL. std::string objectKey; // "/" — no leading '/' if (!prefix.empty()) { std::string normalizedPrefix = prefix; if (!normalizedPrefix.empty() && normalizedPrefix.front() == '/') { normalizedPrefix = normalizedPrefix.substr(1); } if (!normalizedPrefix.empty() && normalizedPrefix.back() != '/') { normalizedPrefix += '/'; } objectKey = normalizedPrefix; } objectKey += fileName; const std::string objectPath = _bAwsPath ? ("/" + objectKey) : ("/" + bucketName + "/" + objectKey); return UploadWithRetry(kOp, [&](std::string& lastError) -> AttemptResult { auto conn = AcquireConnection(); if (!conn) { lastError = "Failed to acquire S3 connection"; return AttemptResult::Transient; } if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } CkBinData binData; CkByteData jpegBytes; jpegBytes.append2(jpeg_string, static_cast(bufferLength)); binData.AppendBinary(jpegBytes); conn->rest.AddHeader("Content-Type", "image/jpeg"); conn->rest.AddHeader("Expect", "100-continue"); CkStringBuilder sbResponse; if (!conn->rest.FullRequestBd("PUT", objectPath.c_str(), binData, sbResponse)) { lastError = conn->rest.lastErrorText(); ReleaseConnection(std::move(conn)); return AttemptResult::Transient; } int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode == 200) { // Build the returned URL from parts (independent of // addressing style) — always the canonical path-style URL. std::string scheme = _bTls ? "https://" : "http://"; uploadedFilePath = scheme + _fullAWSURL + "/" + bucketName + "/" + objectKey; _logger.LogError(kOp, "Successfully uploaded: " + objectKey + " (" + std::to_string(bufferLength) + " bytes) | URL: " + uploadedFilePath, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return AttemptResult::Success; } lastError = "HTTP " + std::to_string(statusCode); std::string body = sbResponse.getAsString(); if (!body.empty()) lastError += " - " + body; ReleaseConnection(std::move(conn)); bool retryable = (statusCode >= 500 && statusCode <= 599) || statusCode == 408 || statusCode == 429; return retryable ? AttemptResult::Transient : AttemptResult::Permanent; }); } // Downloads bool ANSAWSS3::DownloadFile(const std::string& bucketName, const std::string& objectName, const std::string& saveFilePath) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::DownloadFile", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || objectName.empty() || saveFilePath.empty()) { _logger.LogError("ANSAWSS3::DownloadFile", "Bucket name, object name, or save path is empty", __FILE__, __LINE__); return false; } bool downloadSuccess = false; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("DownloadFile", "Failed to acquire S3 connection", __FILE__, __LINE__); return false; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // Normalize object name (remove leading slash if present) std::string normalizedObjectName = objectName; if (!normalizedObjectName.empty() && normalizedObjectName[0] == '/') { normalizedObjectName = normalizedObjectName.substr(1); } std::string objectPath = _bAwsPath ? ("/" + normalizedObjectName) : ("/" + bucketName + "/" + normalizedObjectName); _logger.LogDebug("ANSAWSS3::DownloadFile", "Downloading: " + normalizedObjectName + " from bucket: " + bucketName, __FILE__, __LINE__); // Send GET request if (!conn->rest.SendReqNoBody("GET", objectPath.c_str())) { _logger.LogError("ANSAWSS3::DownloadFile - Request failed", conn->rest.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } // Read the response header int responseStatusCode = conn->rest.ReadResponseHeader(); if (responseStatusCode < 0) { _logger.LogError("ANSAWSS3::DownloadFile - Failed to read response header", conn->rest.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } // We expect a 200 response status if the data is coming if (responseStatusCode == 200) { // Determine the final file path std::string finalFilePath; // Check if saveFilePath is a directory or a file CkFileAccess fac; bool isDirectory = false; // If path exists and is a directory, or ends with path separator if (fac.DirExists(saveFilePath.c_str())) { isDirectory = true; } else if (!saveFilePath.empty() && (saveFilePath.back() == '\\' || saveFilePath.back() == '/')) { isDirectory = true; } if (isDirectory) { // Save path is a directory - append object name finalFilePath = saveFilePath; // Ensure path ends with separator if (!finalFilePath.empty() && finalFilePath.back() != '\\' && finalFilePath.back() != '/') { finalFilePath += "\\"; } // Extract just the filename from object name (in case it has path separators) std::string filename = normalizedObjectName; size_t lastSlash = filename.find_last_of("/\\"); if (lastSlash != std::string::npos) { filename = filename.substr(lastSlash + 1); } finalFilePath += filename; } else { // Save path is the complete file path finalFilePath = saveFilePath; } // Ensure the directory exists size_t lastSeparator = finalFilePath.find_last_of("/\\"); if (lastSeparator != std::string::npos) { std::string directory = finalFilePath.substr(0, lastSeparator); // Create directory if it doesn't exist if (!fac.DirExists(directory.c_str())) { if (!fac.DirCreate(directory.c_str())) { _logger.LogError("ANSAWSS3::DownloadFile - Failed to create directory", "Directory: " + directory + " - " + fac.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } _logger.LogDebug("ANSAWSS3::DownloadFile", "Created directory: " + directory, __FILE__, __LINE__); } } // Setup stream to write to file CkStream bodyStream; bodyStream.put_SinkFile(finalFilePath.c_str()); // Read the response body to the stream if (!conn->rest.ReadRespBodyStream(bodyStream, true)) { _logger.LogError("ANSAWSS3::DownloadFile - Failed to read response body", conn->rest.lastErrorText(), __FILE__, __LINE__); } else { _logger.LogDebug("ANSAWSS3::DownloadFile", "Successfully downloaded: " + normalizedObjectName + " to: " + finalFilePath, __FILE__, __LINE__); downloadSuccess = true; } } else { // Handle non-200 response const char* errResponse = conn->rest.readRespBodyString(); std::string errorMsg = "HTTP " + std::to_string(responseStatusCode); if (!conn->rest.get_LastMethodSuccess()) { errorMsg += " - Failed to read error response: " + std::string(conn->rest.lastErrorText()); } else { if (errResponse != nullptr && errResponse[0] != '\0') { errorMsg += " - " + std::string(errResponse); } } _logger.LogError("ANSAWSS3::DownloadFile", errorMsg, __FILE__, __LINE__); } ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::DownloadFile", std::string("Exception: ") + e.what(), __FILE__, __LINE__); } return downloadSuccess; } // Delete bool ANSAWSS3::DeleteBucketObject(const std::string& bucketName, const std::string& objectName) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::DeleteBucketObject", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || objectName.empty()) { _logger.LogError("ANSAWSS3::DeleteBucketObject", "Bucket name or object name is empty", __FILE__, __LINE__); return false; } bool deleteSuccess = false; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("DeleteBucketObject", "Failed to acquire S3 connection", __FILE__, __LINE__); return false; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } std::string objectPath = _bAwsPath ? ("/" + objectName) : ("/" + bucketName + "/" + objectName); CkStringBuilder sbResponse; bool success = conn->rest.FullRequestNoBodySb("DELETE", objectPath.c_str(), sbResponse); if (!success) { _logger.LogError("ANSAWSS3::DeleteBucketObject", "Failed to send DELETE request: " + std::string(conn->rest.lastErrorText()), __FILE__, __LINE__); } else { int statusCode = conn->rest.get_ResponseStatusCode(); // S3 returns 204 (No Content) for successful deletion // Also accept 200 as success if (statusCode == 204 || statusCode == 200) { _logger.LogDebug("ANSAWSS3::DeleteBucketObject", "Successfully deleted object: " + objectName + " from bucket: " + bucketName, __FILE__, __LINE__); deleteSuccess = true; } else if (statusCode == 404) { // Object doesn't exist - you may want to treat this as success or failure // depending on your use case // Uncomment the next line if you want to treat "not found" as success: deleteSuccess = true; } else { std::string errorMsg = "HTTP " + std::to_string(statusCode); std::string response = sbResponse.getAsString(); if (!response.empty()) { errorMsg += " - " + response; } _logger.LogError("ANSAWSS3::DeleteBucketObject", errorMsg, __FILE__, __LINE__); } } ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::DeleteBucketObject", std::string("Exception: ") + e.what(), __FILE__, __LINE__); } return deleteSuccess; } }