#include "ANSUtilities.h" #include #include #include #include #include #include #include #include #include #include #include #include #include static bool ansawss3LicenceValid = false; static std::once_flag ansawss3LicenseOnceFlag; namespace ANSCENTER { static void VerifyGlobalANSAWSS3License(const std::string& licenseKey) { try { static const std::vector> licenseChecks = { {1000, "ANNHUB-LV"}, {1001, "DLHUB-LV"}, {1002, "ODHUB-LV"}, {1003, "ANSVIS"}, {1004, "ANSFR"}, {1005, "ANSOCR"}, {1006, "ANSALPR"}, {1007, "ANSCV"}, {1008, "ANSSRT"} }; ansawss3LicenceValid = false; for (const auto& [productId, productName] : licenseChecks) { if (ANSCENTER::ANSLicenseHelper::LicenseVerification(licenseKey, productId, productName)) { ansawss3LicenceValid = true; break; // Stop at the first valid license } } } catch (const std::exception& e) { ansawss3LicenceValid = false; } } // Private helper function to extract file name from a path // Helper function to extract filename from path std::string ANSAWSS3::ExtractFileName(const std::string& filePath) { size_t pos = filePath.find_last_of("/\\"); if (pos != std::string::npos) { return filePath.substr(pos + 1); } return filePath; } // Helper function to determine content type from file extension std::string ANSAWSS3::GetContentType(const std::string& filePath) { // Default to application/octet-stream for unknown types std::string contentType = "application/octet-stream"; size_t extPos = filePath.find_last_of('.'); if (extPos == std::string::npos) { return contentType; } // Extract and convert extension to lowercase std::string fileExt = filePath.substr(extPos + 1); std::transform(fileExt.begin(), fileExt.end(), fileExt.begin(), ::tolower); // Image types if (fileExt == "jpg" || fileExt == "jpeg") { contentType = "image/jpeg"; } else if (fileExt == "png") { contentType = "image/png"; } else if (fileExt == "gif") { contentType = "image/gif"; } else if (fileExt == "bmp") { contentType = "image/bmp"; } else if (fileExt == "webp") { contentType = "image/webp"; } else if (fileExt == "svg") { contentType = "image/svg+xml"; } else if (fileExt == "ico") { contentType = "image/x-icon"; } // Video types else if (fileExt == "mp4") { contentType = "video/mp4"; } else if (fileExt == "avi") { contentType = "video/x-msvideo"; } else if (fileExt == "mov") { contentType = "video/quicktime"; } // Document types else if (fileExt == "pdf") { contentType = "application/pdf"; } else if (fileExt == "json") { contentType = "application/json"; } else if (fileExt == "xml") { contentType = "application/xml"; } else if (fileExt == "txt") { contentType = "text/plain"; } return contentType; } ANSAWSS3::ANSAWSS3() { _unlockCode = "ANSDRC.CB1122026_MEQCIFwO1IFQCG0BhZwsXFO68QUU6mDB5uge4duOsqOJanEyAiAB67ahqnXin4SRy0vIegISgbFlpldmbuS5gbU21GYVqA==";// "ANSDRC.CB1082025_Ax6P3M7F8B3d";// _proxyHost = ""; _proxyPort = 0; _proxyUsername = ""; _proxyPassword = ""; _bProxy = false; _serviceName = "s3"; } ANSAWSS3::~ANSAWSS3() { StopRetry(); std::lock_guard lk(_poolMutex); _pool.clear(); // CkRest destructors handle disconnect } void ANSAWSS3::StopRetry() { _stopRetry = true; if (_retryThread.joinable()) { _retryThread.join(); } _retryInProgress = false; } // TryConnect — actual connection attempt (caller must hold _configMutex) bool ANSAWSS3::TryConnect(bool& awsPath) { auto testConn = CreateConnection(); if (!testConn) { // Fallback: connect directly to baseDomain (MinIO, Ceph, etc.) std::string savedFullURL = _fullAWSURL; _fullAWSURL = _baseDomain; testConn = CreateConnection(); if (!testConn) { _fullAWSURL = savedFullURL; // restore on failure return false; } awsPath = false; _bAwsPath = false; } else { awsPath = true; _bAwsPath = true; } _bConnected = true; // Return test connection to pool for reuse { std::lock_guard lk(_poolMutex); _pool.push_back(std::move(testConn)); } return true; } void ANSAWSS3::RetryLoop() { int attempt = 0; while (!_stopRetry) { // Wait 3 seconds (in 1-second increments so we can respond to stop quickly) for (int i = 0; i < 3 && !_stopRetry; ++i) { std::this_thread::sleep_for(std::chrono::seconds(1)); } if (_stopRetry) break; attempt++; // Quick DNS check — is internet available? { CkSocket dnsCheck; CkString dnsResult; if (!dnsCheck.DnsLookup(_fullAWSURL.c_str(), 3000, dnsResult)) { _logger.LogDebug("ANSAWSS3::RetryLoop", "Retry #" + std::to_string(attempt) + " - no internet, retrying in 3s", __FILE__, __LINE__); continue; } } // Internet is available — attempt real connection to validate auth/URL { std::lock_guard lk(_configMutex); bool awsPath = true; if (TryConnect(awsPath)) { _logger.LogDebug("ANSAWSS3::RetryLoop", "Connected successfully after " + std::to_string(attempt) + " retries", __FILE__, __LINE__); _retryInProgress = false; return; } else { // Internet is available but connection failed (bad auth/URL) // Stop retrying — no point, the parameters are wrong _logger.LogError("ANSAWSS3::RetryLoop", "Internet available but connection failed (check authentication or URL). Stopping retry.", __FILE__, __LINE__); _retryInProgress = false; return; } } } _retryInProgress = false; } // ── Connection pool helpers ── std::unique_ptr ANSAWSS3::CreateConnection() { auto conn = std::make_unique(); // Connect if (!conn->rest.Connect(_fullAWSURL.c_str(), _port, _bTls, _bAutoReconnect)) { _logger.LogError("ANSAWSS3::CreateConnection", conn->rest.lastErrorText(), __FILE__, __LINE__); return nullptr; } // Apply proxy if configured if (_bProxy) { conn->socket.put_HttpProxyHostname(_proxyHost.c_str()); conn->socket.put_HttpProxyPort(_proxyPort); conn->socket.put_HttpProxyUsername(_proxyUsername.c_str()); conn->socket.put_HttpProxyPassword(_proxyPassword.c_str()); conn->socket.put_HttpProxyForHttp(_bProxy); if (!conn->rest.UseConnection(conn->socket, true)) { _logger.LogError("ANSAWSS3::CreateConnection - Proxy error", conn->rest.lastErrorText(), __FILE__, __LINE__); return nullptr; } } // Apply auth if (_authReady) { conn->authAws.put_AccessKey(_accessKey.c_str()); conn->authAws.put_SecretKey(_secretKey.c_str()); conn->authAws.put_ServiceName(_serviceName.c_str()); if (_bucketRegion != "us-east-1") { if (!_bucketRegion.empty()) conn->authAws.put_Region(_bucketRegion.c_str()); } if (!conn->rest.SetAuthAws(conn->authAws)) { _logger.LogError("ANSAWSS3::CreateConnection - Auth error", conn->rest.lastErrorText(), __FILE__, __LINE__); return nullptr; } } return conn; } std::unique_ptr ANSAWSS3::AcquireConnection() { { std::lock_guard lk(_poolMutex); if (!_pool.empty()) { auto conn = std::move(_pool.back()); _pool.pop_back(); return conn; } } // Pool empty — create a new connection (no lock held during network I/O) std::lock_guard cfgLk(_configMutex); return CreateConnection(); } void ANSAWSS3::ReleaseConnection(std::unique_ptr conn) { if (!conn) return; std::lock_guard lk(_poolMutex); _pool.push_back(std::move(conn)); } void ANSAWSS3::CheckLicense() { // Note: caller (Initialize) already holds _configMutex try { // Check once globally std::call_once(ansawss3LicenseOnceFlag, [this]() { VerifyGlobalANSAWSS3License(_licenseKey); }); // Update this instance's local license flag _isLicenseValid = ansawss3LicenceValid; } catch (const std::exception& e) { this->_logger.LogFatal("ANSAWSS3::CheckLicense. Error:", e.what(), __FILE__, __LINE__); } } void ANSAWSS3::CheckUnlockCode() { // Note: caller (Initialize) already holds _configMutex try { CkGlobal glob; _unlockCode = "ANSDRC.CB1122026_MEQCIFwO1IFQCG0BhZwsXFO68QUU6mDB5uge4duOsqOJanEyAiAB67ahqnXin4SRy0vIegISgbFlpldmbuS5gbU21GYVqA==";// "ANSDRC.CB1082025_Ax6P3M7F8B3d"; _isUnlockCodeValid = glob.UnlockBundle(_unlockCode.c_str()); if (!_isUnlockCodeValid) { _logger.LogFatal("ANSAWSS3::CheckUnlockCode", glob.lastErrorText(), __FILE__, __LINE__); return; } int status = glob.get_UnlockStatus(); if (status != 2) { _logger.LogDebug("ANSAWSS3::CheckUnlockCode", "Unlocked in trial mode.", __FILE__, __LINE__); } } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::CheckUnlockCode", e.what(), __FILE__, __LINE__); } } bool ANSAWSS3::Initialize(const std::string& licenseKey) { std::lock_guard lock(_configMutex); try { _licenseKey = licenseKey; CheckLicense(); CheckUnlockCode(); return _isLicenseValid && _isUnlockCodeValid; } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::Initialize", e.what(), __FILE__, __LINE__); return false; } } bool ANSAWSS3::SetServerProxy(const std::string& proxyHost, int proxyPort,const std::string& proxyUsername,const std::string& proxyPassword) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid) { _logger.LogError("ANSAWSS3::SetServerProxy", !_isLicenseValid ? "Invalid license" : "Invalid unlock code", __FILE__, __LINE__); return false; } std::lock_guard lock(_configMutex); try { _proxyHost = proxyHost; _proxyPort = proxyPort; _proxyUsername = proxyUsername; _proxyPassword = proxyPassword; // Simplified proxy validation logic _bProxy = !proxyHost.empty() && proxyPort > 0; // Clear pool so new connections pick up proxy config { std::lock_guard lk(_poolMutex); _pool.clear(); } return true; } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::SetServerProxy", e.what(), __FILE__, __LINE__); return false; } } // Returns: 1 = connected, 0 = failed (bad auth/URL), 2 = no internet (background retry started) int ANSAWSS3::Connect(const std::string& baseDomain, const std::string& bucketRegion, const std::string& serviceName, int port, bool bTls, bool autoReconnect, bool &awsPath) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid) { _logger.LogError("ANSAWSS3::Connect", !_isLicenseValid ? "Invalid license" : "Invalid unlock code", __FILE__, __LINE__); return 0; } // Stop any existing background retry StopRetry(); std::lock_guard lock(_configMutex); try { // Store parameters — strip http:// or https:// from baseDomain { std::string domain = baseDomain.empty() ? "amazonaws.com" : baseDomain; if (domain.rfind("https://", 0) == 0) { domain = domain.substr(8); bTls = true; // caller said https, honour it } else if (domain.rfind("http://", 0) == 0) { domain = domain.substr(7); } // Remove trailing slash if any if (!domain.empty() && domain.back() == '/') domain.pop_back(); _baseDomain = domain; } _bucketRegion = bucketRegion.empty() ? "us-east-1" : bucketRegion; _serviceName = serviceName.empty() ? "s3" : serviceName; _fullAWSURL = _serviceName + "." + _bucketRegion + "." + _baseDomain; _port = port; _bTls = bTls; _bAutoReconnect = autoReconnect; // Clear pool so new connections use updated config { std::lock_guard lk(_poolMutex); _pool.clear(); } // Check if internet is available (fast DNS check) { CkSocket dnsCheck; CkString dnsResult; if (!dnsCheck.DnsLookup(_fullAWSURL.c_str(), 3000, dnsResult)) { // No internet — start background retry, return immediately _logger.LogDebug("ANSAWSS3::Connect", "No internet available, starting background retry for '" + _fullAWSURL + "'...", __FILE__, __LINE__); _stopRetry = false; _retryInProgress = true; _retryThread = std::thread(&ANSAWSS3::RetryLoop, this); return 2; // no internet, retrying in background } } // Internet is available — try real connection if (TryConnect(awsPath)) { return 1; // connected successfully } // Internet available but connection failed (bad auth, wrong URL, etc.) return 0; } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::Connect", e.what(), __FILE__, __LINE__); return 0; } } bool ANSAWSS3::SetAuthentication(const std::string& accessKey,const std::string& secretKey) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid) { _logger.LogError("ANSAWSS3::SetAuthentication", !_isLicenseValid ? "Invalid license" : "Invalid unlock code", __FILE__, __LINE__); return false; } std::lock_guard lock(_configMutex); try { _accessKey = accessKey; _secretKey = secretKey; _authReady = true; // Clear pool so new connections pick up new credentials { std::lock_guard lk(_poolMutex); _pool.clear(); } return true; } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::SetAuthentication", e.what(), __FILE__, __LINE__); return false; } } std::vector ANSAWSS3::ListBuckets() { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::ListBuckets", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return {}; } try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("ListBuckets", "Failed to acquire S3 connection", __FILE__, __LINE__); return {}; } // Make request and capture response conn->rest.put_Host(_fullAWSURL.c_str()); CkStringBuilder sbResponse; if (!conn->rest.FullRequestNoBodySb("GET", "/", sbResponse)) { _logger.LogError("ANSAWSS3::ListBuckets - Request failed", conn->rest.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return {}; } // Check HTTP status code int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode != 200) { _logger.LogError("ANSAWSS3::ListBuckets - HTTP " + std::to_string(statusCode), sbResponse.getAsString(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return {}; } // Parse XML response CkXml xml; if (!xml.LoadSb(sbResponse, true)) { _logger.LogError("ANSAWSS3::ListBuckets - XML parse error", xml.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return {}; } // Extract bucket names std::vector bucketList; int bucketCount = xml.NumChildrenHavingTag("Buckets|Bucket"); if (bucketCount > 0) { bucketList.reserve(bucketCount); for (int i = 0; i < bucketCount; ++i) { xml.put_I(i); const char* name = xml.getChildContent("Buckets|Bucket[i]|Name"); // Validate name before adding if (name != nullptr && name[0] != '\0') { bucketList.emplace_back(name); } } } ReleaseConnection(std::move(conn)); return bucketList; } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::ListBuckets", e.what(), __FILE__, __LINE__); return {}; } } std::vector ANSAWSS3::ListBucketObjects(const std::string& bucketName) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::ListBucketObjects", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return {}; } if (bucketName.empty()) { _logger.LogError("ANSAWSS3::ListBucketObjects", "Bucket name is empty", __FILE__, __LINE__); return {}; } std::vector objectList; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("ListBucketObjects", "Failed to acquire S3 connection", __FILE__, __LINE__); return {}; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } std::string marker = ""; bool isTruncated = true; int pageCount = 0; // Loop to handle pagination while (isTruncated) { pageCount++; // Build the request path with marker if available std::string basePath = _bAwsPath ? "/" : ("/" + bucketName + "/"); std::string requestPath = basePath; if (!marker.empty()) { requestPath = basePath + "?marker=" + marker; } CkStringBuilder sbResponse; bool success = conn->rest.FullRequestNoBodySb("GET", requestPath.c_str(), sbResponse); if (!success) { _logger.LogError("ANSAWSS3::ListBucketObjects - Request failed on page " + std::to_string(pageCount), conn->rest.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return objectList; // Return what we have so far } int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode != 200) { std::string errorMsg = "HTTP " + std::to_string(statusCode); std::string response = sbResponse.getAsString(); if (!response.empty()) { errorMsg += " - " + response; } _logger.LogError("ANSAWSS3::ListBucketObjects - Page " + std::to_string(pageCount), errorMsg, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return objectList; // Return what we have so far } // Parse XML response CkXml xml; bool loadSuccess = xml.LoadSb(sbResponse, true); if (!loadSuccess) { _logger.LogError("ANSAWSS3::ListBucketObjects", "Failed to parse XML response on page " + std::to_string(pageCount) + ": " + std::string(xml.lastErrorText()), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return objectList; // Return what we have so far } // Check if results are truncated (more pages to fetch) const char* truncatedStr = xml.getChildContent("IsTruncated"); isTruncated = (truncatedStr != nullptr && std::string(truncatedStr) == "true"); // Get the marker for next page if (isTruncated) { // First try to get NextMarker const char* nextMarker = xml.getChildContent("NextMarker"); if (nextMarker != nullptr && nextMarker[0] != '\0') { marker = std::string(nextMarker); } else { // If NextMarker is not present, use the last Key as marker int count = xml.NumChildrenHavingTag("Contents"); if (count > 0) { xml.put_I(count - 1); // Last element const char* lastKey = xml.getChildContent("Contents[i]|Key"); if (lastKey != nullptr && lastKey[0] != '\0') { marker = std::string(lastKey); } else { isTruncated = false; } } else { isTruncated = false; } } } // Iterate through all Contents elements in this page int count = xml.NumChildrenHavingTag("Contents"); for (int i = 0; i < count; i++) { xml.put_I(i); const char* key = xml.getChildContent("Contents[i]|Key"); if (key != nullptr && key[0] != '\0') { objectList.emplace_back(key); } } _logger.LogDebug("ANSAWSS3::ListBucketObjects", "Page " + std::to_string(pageCount) + ": Retrieved " + std::to_string(count) + " objects. Total so far: " + std::to_string(objectList.size()), __FILE__, __LINE__); } _logger.LogDebug("ANSAWSS3::ListBucketObjects", "Successfully listed " + std::to_string(objectList.size()) + " objects in bucket: " + bucketName + " (" + std::to_string(pageCount) + " pages)", __FILE__, __LINE__); ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::ListBucketObjects", std::string("Exception: ") + e.what(), __FILE__, __LINE__); objectList.clear(); // Ensure we return empty list on exception } return objectList; } std::vector ANSAWSS3::ListBucketObjectsWithPrefix(const std::string& bucketName, const std::string& prefix) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::ListBucketObjectsWithPrefix", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return {}; } if (bucketName.empty()) { _logger.LogError("ANSAWSS3::ListBucketObjectsWithPrefix", "Bucket name is empty", __FILE__, __LINE__); return {}; } // prefix can be empty to list all objects std::vector objectList; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("ListBucketObjectsWithPrefix", "Failed to acquire S3 connection", __FILE__, __LINE__); return {}; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } std::string marker = ""; bool isTruncated = true; int pageCount = 0; // Loop to handle pagination while (isTruncated) { pageCount++; // Build the request path with prefix and marker std::string basePath = _bAwsPath ? "/" : ("/" + bucketName + "/"); std::string requestPath = basePath; bool hasParams = false; // Add prefix parameter if provided if (!prefix.empty()) { requestPath += "?prefix=" + prefix; hasParams = true; } // Add marker parameter if available if (!marker.empty()) { if (hasParams) { requestPath += "&marker=" + marker; } else { requestPath += "?marker=" + marker; hasParams = true; } } CkStringBuilder sbResponse; bool success = conn->rest.FullRequestNoBodySb("GET", requestPath.c_str(), sbResponse); if (!success) { _logger.LogError("ANSAWSS3::ListBucketObjectsWithPrefix - Request failed on page " + std::to_string(pageCount), conn->rest.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return objectList; // Return what we have so far } int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode != 200) { std::string errorMsg = "HTTP " + std::to_string(statusCode); std::string response = sbResponse.getAsString(); if (!response.empty()) { errorMsg += " - " + response; } _logger.LogError("ANSAWSS3::ListBucketObjectsWithPrefix - Page " + std::to_string(pageCount), errorMsg, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return objectList; // Return what we have so far } // Parse XML response CkXml xml; bool loadSuccess = xml.LoadSb(sbResponse, true); if (!loadSuccess) { _logger.LogError("ANSAWSS3::ListBucketObjectsWithPrefix", "Failed to parse XML response on page " + std::to_string(pageCount) + ": " + std::string(xml.lastErrorText()), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return objectList; // Return what we have so far } // Check if results are truncated (more pages to fetch) const char* truncatedStr = xml.getChildContent("IsTruncated"); isTruncated = (truncatedStr != nullptr && std::string(truncatedStr) == "true"); // Get the marker for next page if (isTruncated) { // First try to get NextMarker const char* nextMarker = xml.getChildContent("NextMarker"); if (nextMarker != nullptr && nextMarker[0] != '\0') { marker = std::string(nextMarker); } else { // If NextMarker is not present, use the last Key as marker int count = xml.NumChildrenHavingTag("Contents"); if (count > 0) { xml.put_I(count - 1); // Last element const char* lastKey = xml.getChildContent("Contents[i]|Key"); if (lastKey != nullptr && lastKey[0] != '\0') { marker = std::string(lastKey); } else { isTruncated = false; } } else { isTruncated = false; } } } // Iterate through all Contents elements in this page int count = xml.NumChildrenHavingTag("Contents"); for (int i = 0; i < count; i++) { xml.put_I(i); const char* key = xml.getChildContent("Contents[i]|Key"); if (key != nullptr && key[0] != '\0') { objectList.emplace_back(key); } } std::string prefixInfo = prefix.empty() ? "all objects" : "prefix '" + prefix + "'"; _logger.LogDebug("ANSAWSS3::ListBucketObjectsWithPrefix", "Page " + std::to_string(pageCount) + ": Retrieved " + std::to_string(count) + " objects for " + prefixInfo + ". Total so far: " + std::to_string(objectList.size()), __FILE__, __LINE__); } std::string prefixInfo = prefix.empty() ? "" : " with prefix '" + prefix + "'"; _logger.LogDebug("ANSAWSS3::ListBucketObjectsWithPrefix", "Successfully listed " + std::to_string(objectList.size()) + " objects in bucket: " + bucketName + prefixInfo + " (" + std::to_string(pageCount) + " pages)", __FILE__, __LINE__); ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::ListBucketObjectsWithPrefix", std::string("Exception: ") + e.what(), __FILE__, __LINE__); objectList.clear(); // Ensure we return empty list on exception } return objectList; } bool ANSAWSS3::CreateBucket(const std::string& bucketName) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::CreateBucket", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } // Validate input parameters if (bucketName.empty()) { _logger.LogError("ANSAWSS3::CreateBucket", "Bucket name cannot be empty", __FILE__, __LINE__); return false; } bool createSuccess = false; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("CreateBucket", "Failed to acquire S3 connection", __FILE__, __LINE__); return false; } // Build global endpoint std::string globalEndpoint = _serviceName + "." + _baseDomain; CkStringBuilder sbBucketRegion; sbBucketRegion.Append(_bucketRegion.c_str()); // We only need to specify the LocationConstraint if the bucket's region is NOT us-east-1 CkXml xml; if (!sbBucketRegion.ContentsEqual("us-east-1", true)) { xml.put_Tag("CreateBucketConfiguration"); xml.AddAttribute("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/"); xml.UpdateChildContent("LocationConstraint", _bucketRegion.c_str()); } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + globalEndpoint).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // Make the call to create the bucket std::string createPath = _bAwsPath ? "/" : ("/" + bucketName); const char* responseStr = nullptr; if (!sbBucketRegion.ContentsEqual("us-east-1", true)) { responseStr = conn->rest.fullRequestString("PUT", createPath.c_str(), xml.getXml()); } else { // If the bucket is to be created in the us-east-1 region (the default region) // just send a PUT with no body responseStr = conn->rest.fullRequestNoBody("PUT", createPath.c_str()); } if (conn->rest.get_LastMethodSuccess() != true) { _logger.LogError("ANSAWSS3::CreateBucket - Request failed", conn->rest.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode == 200) { _logger.LogDebug("ANSAWSS3::CreateBucket", "Successfully created bucket: " + bucketName + " in region: " + _bucketRegion, __FILE__, __LINE__); createSuccess = true; } else { std::string errorMsg = "HTTP " + std::to_string(statusCode); if (responseStr != nullptr && responseStr[0] != '\0') { CkXml responseXml; if (responseXml.LoadXml(responseStr)) { const char* errorCode = responseXml.getChildContent("Code"); const char* errorMessage = responseXml.getChildContent("Message"); if (errorCode != nullptr) { errorMsg += " - " + std::string(errorCode); } if (errorMessage != nullptr) { errorMsg += ": " + std::string(errorMessage); } } else { errorMsg += "\n" + std::string(responseStr); } } if (statusCode == 409) { _logger.LogError("ANSAWSS3::CreateBucket", errorMsg, __FILE__, __LINE__); } else { _logger.LogError("ANSAWSS3::CreateBucket", errorMsg, __FILE__, __LINE__); } } ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::CreateBucket", std::string("Exception: ") + e.what(), __FILE__, __LINE__); } return createSuccess; } bool ANSAWSS3::DeleteBucket(const std::string& bucketName) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::DeleteBucket", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty()) { _logger.LogError("ANSAWSS3::DeleteBucket", "Bucket name cannot be empty", __FILE__, __LINE__); return false; } bool deleteSuccess = false; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("DeleteBucket", "Failed to acquire S3 connection", __FILE__, __LINE__); return false; } // Configure AWS authentication conn->rest.ClearAuth(); conn->authAws.put_AccessKey(_accessKey.c_str()); conn->authAws.put_SecretKey(_secretKey.c_str()); conn->authAws.put_ServiceName(_serviceName.c_str()); bool authSuccess = conn->rest.SetAuthAws(conn->authAws); if (!authSuccess) { _logger.LogError("ANSAWSS3::DeleteBucket", "Failed to set AWS authentication: " + std::string(conn->rest.lastErrorText()), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } // Set bucket-specific endpoint std::string globalEndpoint = _serviceName + "." + _baseDomain; if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + globalEndpoint).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } std::string deletePath = _bAwsPath ? "/" : ("/" + bucketName); CkStringBuilder sbResponse; bool success = conn->rest.FullRequestNoBodySb("DELETE", deletePath.c_str(), sbResponse); if (!success) { _logger.LogError("ANSAWSS3::DeleteBucket", "Failed to send DELETE request: " + std::string(conn->rest.lastErrorText()), __FILE__, __LINE__); } else { int statusCode = conn->rest.get_ResponseStatusCode(); // S3 returns 204 (No Content) for successful bucket deletion if (statusCode == 204) { _logger.LogDebug("ANSAWSS3::DeleteBucket", "Successfully deleted bucket: " + bucketName, __FILE__, __LINE__); deleteSuccess = true; } else if (statusCode == 404) { _logger.LogWarn("ANSAWSS3::DeleteBucket", "Bucket not found (already deleted?): " + bucketName, __FILE__, __LINE__); } else if (statusCode == 409) { // Bucket not empty std::string errorMsg = "HTTP 409 - Bucket is not empty and cannot be deleted"; std::string response = sbResponse.getAsString(); if (!response.empty()) { CkXml responseXml; if (responseXml.LoadXml(response.c_str())) { const char* errorCode = responseXml.getChildContent("Code"); const char* errorMessage = responseXml.getChildContent("Message"); if (errorCode != nullptr) { errorMsg += " - " + std::string(errorCode); } if (errorMessage != nullptr) { errorMsg += ": " + std::string(errorMessage); } } } _logger.LogError("ANSAWSS3::DeleteBucket", errorMsg, __FILE__, __LINE__); } else { std::string errorMsg = "HTTP " + std::to_string(statusCode); std::string response = sbResponse.getAsString(); if (!response.empty()) { errorMsg += " - " + response; } _logger.LogError("ANSAWSS3::DeleteBucket", errorMsg, __FILE__, __LINE__); } } ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::DeleteBucket", std::string("Exception: ") + e.what(), __FILE__, __LINE__); } return deleteSuccess; } bool ANSAWSS3::CreateFolder(const std::string& bucketName, const std::string& prefix) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::CreateFolder", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || prefix.empty()) { _logger.LogError("ANSAWSS3::CreateFolder", "Bucket name or folder path is empty", __FILE__, __LINE__); return false; } bool createSuccess = false; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("CreateFolder", "Failed to acquire S3 connection", __FILE__, __LINE__); return false; } // Ensure folder path ends with "/" std::string normalizedPath = prefix; if (normalizedPath.back() != '/') { normalizedPath += '/'; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // Object path is the folder name with trailing slash std::string objectPath = _bAwsPath ? ("/" + normalizedPath) : ("/" + bucketName + "/" + normalizedPath); // Create a zero-byte object to represent the folder // Use PUT with empty body const char* responseStr = conn->rest.fullRequestNoBody("PUT", objectPath.c_str()); if (conn->rest.get_LastMethodSuccess() != true) { _logger.LogError("ANSAWSS3::CreateFolder - Request failed", conn->rest.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode == 200) { _logger.LogDebug("ANSAWSS3::CreateFolder", "Successfully created folder: " + normalizedPath + " in bucket: " + bucketName, __FILE__, __LINE__); createSuccess = true; } else { std::string errorMsg = "HTTP " + std::to_string(statusCode); if (responseStr != nullptr && responseStr[0] != '\0') { CkXml responseXml; if (responseXml.LoadXml(responseStr)) { const char* errorCode = responseXml.getChildContent("Code"); const char* errorMessage = responseXml.getChildContent("Message"); if (errorCode != nullptr) { errorMsg += " - " + std::string(errorCode); } if (errorMessage != nullptr) { errorMsg += ": " + std::string(errorMessage); } } else { errorMsg += " - " + std::string(responseStr); } } _logger.LogError("ANSAWSS3::CreateFolder", errorMsg, __FILE__, __LINE__); } ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::CreateFolder", std::string("Exception: ") + e.what(), __FILE__, __LINE__); } return createSuccess; } bool ANSAWSS3::DeleteFolder(const std::string& bucketName, const std::string& prefix) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::DeleteFolder", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || prefix.empty()) { _logger.LogError("ANSAWSS3::DeleteFolder", "Bucket name or folder path is empty", __FILE__, __LINE__); return false; } bool deleteSuccess = false; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("DeleteFolder", "Failed to acquire S3 connection", __FILE__, __LINE__); return false; } // Ensure folder path ends with "/" std::string normalizedPath = prefix; if (normalizedPath.back() != '/') { normalizedPath += '/'; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } std::string objectPath = _bAwsPath ? ("/" + normalizedPath) : ("/" + bucketName + "/" + normalizedPath); CkStringBuilder sbResponse; bool success = conn->rest.FullRequestNoBodySb("DELETE", objectPath.c_str(), sbResponse); if (!success) { _logger.LogError("ANSAWSS3::DeleteFolder", "Failed to send DELETE request: " + std::string(conn->rest.lastErrorText()), __FILE__, __LINE__); } else { int statusCode = conn->rest.get_ResponseStatusCode(); // S3 returns 204 (No Content) for successful deletion if (statusCode == 204 || statusCode == 200) { _logger.LogDebug("ANSAWSS3::DeleteFolder", "Successfully deleted folder marker: " + normalizedPath + " from bucket: " + bucketName, __FILE__, __LINE__); deleteSuccess = true; } else if (statusCode == 404) { _logger.LogWarn("ANSAWSS3::DeleteFolder", "Folder marker not found (already deleted?): " + normalizedPath, __FILE__, __LINE__); } else { std::string errorMsg = "HTTP " + std::to_string(statusCode); std::string response = sbResponse.getAsString(); if (!response.empty()) { errorMsg += " - " + response; } _logger.LogError("ANSAWSS3::DeleteFolder", errorMsg, __FILE__, __LINE__); } } ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::DeleteFolder", std::string("Exception: ") + e.what(), __FILE__, __LINE__); } return deleteSuccess; } std::string ANSAWSS3::GetBucketRegion(const std::string& bucketName) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::GetBucketRegion", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return ""; } if (bucketName.empty()) { _logger.LogError("ANSAWSS3::GetBucketRegion", "Bucket name is empty", __FILE__, __LINE__); return ""; } std::string region = ""; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("GetBucketRegion", "Failed to acquire S3 connection", __FILE__, __LINE__); return ""; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // Send the GET request to query the bucket location std::string locationPath = _bAwsPath ? "/?location" : ("/" + bucketName + "?location"); const char* strResult = conn->rest.fullRequestNoBody("GET", locationPath.c_str()); if (conn->rest.get_LastMethodSuccess() != true) { _logger.LogError("ANSAWSS3::GetBucketRegion - Request failed", conn->rest.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return ""; } int responseStatusCode = conn->rest.get_ResponseStatusCode(); if (responseStatusCode != 200) { std::string errorMsg = "HTTP " + std::to_string(responseStatusCode); if (strResult != nullptr && strResult[0] != '\0') { errorMsg += " - " + std::string(strResult); } _logger.LogError("ANSAWSS3::GetBucketRegion", errorMsg, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return ""; } // Parse XML response CkXml xml; bool success = xml.LoadXml(strResult); if (!success) { _logger.LogError("ANSAWSS3::GetBucketRegion - Failed to parse XML", xml.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return ""; } // Get the region from XML content const char* regionContent = xml.content(); if (regionContent != nullptr && regionContent[0] != '\0') { region = std::string(regionContent); // AWS returns empty string for us-east-1 (classic region) // Some implementations may return "null" or empty if (region.empty() || region == "null") { region = "us-east-1"; _logger.LogDebug("ANSAWSS3::GetBucketRegion", "Bucket '" + bucketName + "' is in default region: us-east-1", __FILE__, __LINE__); } else { _logger.LogDebug("ANSAWSS3::GetBucketRegion", "Bucket '" + bucketName + "' is in region: " + region, __FILE__, __LINE__); } } else { // Empty response typically means us-east-1 region = "us-east-1"; _logger.LogDebug("ANSAWSS3::GetBucketRegion", "Empty response - assuming default region: us-east-1", __FILE__, __LINE__); } ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::GetBucketRegion", std::string("Exception: ") + e.what(), __FILE__, __LINE__); } return region; } // Uploads text data from a file to the specified S3 bucket bool ANSAWSS3::UploadTextData(const std::string& bucketName, const std::string& textFilePath, std::string& uploadedFilePath) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::UploadTextData", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || textFilePath.empty()) { _logger.LogError("ANSAWSS3::UploadTextData", "Bucket name or file path is empty", __FILE__, __LINE__); return false; } try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("UploadTextData", "Failed to acquire S3 connection", __FILE__, __LINE__); return false; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // Read text file CkFileAccess fac; const char* fileContents = fac.readEntireTextFile(textFilePath.c_str(), "utf-8"); if (!fac.get_LastMethodSuccess()) { _logger.LogError("ANSAWSS3::UploadTextData - Failed to read file", fac.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } // Validate file contents if (fileContents == nullptr || fileContents[0] == '\0') { _logger.LogError("ANSAWSS3::UploadTextData", "File is empty: " + textFilePath, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } // Extract filename std::string fileName = ExtractFileName(textFilePath); if (fileName.empty()) { _logger.LogError("ANSAWSS3::UploadTextData", "Failed to extract filename from path: " + textFilePath, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } // Determine content type std::string contentType = GetContentType(textFilePath); // Set headers conn->rest.AddHeader("Content-Type", contentType.c_str()); conn->rest.AddHeader("Content-Encoding", "gzip"); conn->rest.AddHeader("Expect", "100-continue"); // Construct S3 object path std::string objectPath = _bAwsPath ? ("/" + fileName) : ("/" + bucketName + "/" + fileName); _logger.LogDebug("ANSAWSS3::UploadTextData", "Uploading file: " + fileName + " (" + contentType + ")", __FILE__, __LINE__); // Upload to S3 const char* responseBodyStr = conn->rest.fullRequestString("PUT", objectPath.c_str(), fileContents); if (!conn->rest.get_LastMethodSuccess()) { _logger.LogError("ANSAWSS3::UploadTextData - Upload failed", conn->rest.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } // Check HTTP status code int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode != 200) { std::string errorMsg = "HTTP " + std::to_string(statusCode); if (responseBodyStr != nullptr && responseBodyStr[0] != '\0') { errorMsg += " - " + std::string(responseBodyStr); } _logger.LogError("ANSAWSS3::UploadTextData", errorMsg, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } { std::string scheme = _bTls ? "https://" : "http://"; std::string uploadedUrl = scheme + _fullAWSURL + "/" + bucketName + "/" + fileName; uploadedFilePath = uploadedUrl; _logger.LogDebug("ANSAWSS3::UploadTextData", "Successfully uploaded: " + fileName + " to bucket: " + bucketName + " | URL: " + uploadedUrl, __FILE__, __LINE__); } ReleaseConnection(std::move(conn)); return true; } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::UploadTextData", e.what(), __FILE__, __LINE__); return false; } } bool ANSAWSS3::UploadBinaryData(const std::string& bucketName, const std::string& dataFilePath, std::string& uploadedFilePath) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::UploadBinaryData", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || dataFilePath.empty()) { _logger.LogError("ANSAWSS3::UploadBinaryData", "Bucket name or file path is empty", __FILE__, __LINE__); return false; } try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("UploadBinaryData", "Failed to acquire S3 connection", __FILE__, __LINE__); return false; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // Load binary file CkBinData binData; if (!binData.LoadFile(dataFilePath.c_str())) { _logger.LogError("ANSAWSS3::UploadBinaryData - Failed to load file", binData.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } // Extract filename from path std::string fileName = ExtractFileName(dataFilePath); if (fileName.empty()) { _logger.LogError("ANSAWSS3::UploadBinaryData", "Failed to extract filename from path: " + dataFilePath, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } // Determine content type from file extension std::string contentType = GetContentType(dataFilePath); // Set headers conn->rest.AddHeader("Content-Type", contentType.c_str()); conn->rest.AddHeader("Expect", "100-continue"); // Construct S3 object path std::string objectPath = _bAwsPath ? ("/" + fileName) : ("/" + bucketName + "/" + fileName); _logger.LogDebug("ANSAWSS3::UploadBinaryData", "Uploading file: " + fileName + " (" + contentType + ")", __FILE__, __LINE__); // Upload to S3 CkStringBuilder sbResponse; if (!conn->rest.FullRequestBd("PUT", objectPath.c_str(), binData, sbResponse)) { _logger.LogError("ANSAWSS3::UploadBinaryData - Upload failed", conn->rest.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } // Check HTTP status code int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode != 200) { std::string errorMsg = "HTTP " + std::to_string(statusCode); std::string response = sbResponse.getAsString(); if (!response.empty()) { errorMsg += " - " + response; } _logger.LogError("ANSAWSS3::UploadBinaryData", errorMsg, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } { std::string scheme = _bTls ? "https://" : "http://"; std::string uploadedUrl = scheme + _fullAWSURL + "/" + bucketName + "/" + fileName; uploadedFilePath = uploadedUrl; _logger.LogDebug("ANSAWSS3::UploadBinaryData", "Successfully uploaded: " + fileName + " to bucket: " + bucketName + " | URL: " + uploadedUrl, __FILE__, __LINE__); } ReleaseConnection(std::move(conn)); return true; } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::UploadBinaryData", e.what(), __FILE__, __LINE__); return false; } } bool ANSAWSS3::UploadPrefixBinaryData(const std::string& bucketName, const std::string& prefix, const std::string& dataFilePath, const std::string& objectName, std::string& uploadedFilePath) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::UploadPrefixBinaryData", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || dataFilePath.empty()) { _logger.LogError("ANSAWSS3::UploadPrefixBinaryData", "Bucket name or file path is empty", __FILE__, __LINE__); return false; } try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("UploadPrefixBinaryData", "Failed to acquire S3 connection", __FILE__, __LINE__); return false; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // Load binary file CkBinData binData; if (!binData.LoadFile(dataFilePath.c_str())) { _logger.LogError("ANSAWSS3::UploadPrefixBinaryData - Failed to load file", binData.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } // Extract filename from path std::string fileName = objectName; if (fileName.empty()) { fileName = ExtractFileName(dataFilePath); if (fileName.empty()) { _logger.LogError("ANSAWSS3::UploadPrefixBinaryData", "Failed to extract filename from path: " + dataFilePath, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } } // Construct object path with optional prefix std::string objectPath; if (!prefix.empty()) { // Normalize prefix: ensure it doesn't start with / but ends with / std::string normalizedPrefix = prefix; // Remove leading slash if present if (normalizedPrefix[0] == '/') { normalizedPrefix = normalizedPrefix.substr(1); } // Add trailing slash if not present if (!normalizedPrefix.empty() && normalizedPrefix.back() != '/') { normalizedPrefix += '/'; } objectPath = _bAwsPath ? ("/" + normalizedPrefix + fileName) : ("/" + bucketName + "/" + normalizedPrefix + fileName); } else { objectPath = _bAwsPath ? ("/" + fileName) : ("/" + bucketName + "/" + fileName); } // Determine content type from file extension std::string contentType = GetContentType(dataFilePath); // Set headers conn->rest.AddHeader("Content-Type", contentType.c_str()); conn->rest.AddHeader("Expect", "100-continue"); _logger.LogDebug("ANSAWSS3::UploadPrefixBinaryData", "Uploading file: " + fileName + " (" + contentType + ") to: " + objectPath, __FILE__, __LINE__); // Upload to S3 CkStringBuilder sbResponse; if (!conn->rest.FullRequestBd("PUT", objectPath.c_str(), binData, sbResponse)) { _logger.LogError("ANSAWSS3::UploadPrefixBinaryData - Upload failed", conn->rest.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } // Check HTTP status code int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode != 200) { std::string errorMsg = "HTTP " + std::to_string(statusCode); std::string response = sbResponse.getAsString(); if (!response.empty()) { errorMsg += " - " + response; } _logger.LogError("ANSAWSS3::UploadPrefixBinaryData", errorMsg, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } { std::string scheme = _bTls ? "https://" : "http://"; std::string uploadedUrl = scheme + _fullAWSURL + "/" + bucketName + objectPath; uploadedFilePath = uploadedUrl; _logger.LogDebug("ANSAWSS3::UploadPrefixBinaryData", "Successfully uploaded to: " + objectPath + " in bucket: " + bucketName + " | URL: " + uploadedUrl, __FILE__, __LINE__); } ReleaseConnection(std::move(conn)); return true; } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::UploadPrefixBinaryData", e.what(), __FILE__, __LINE__); return false; } } bool ANSAWSS3::UploadFileStream(const std::string& bucketName, const std::string& dataFilePath, std::string& uploadedFilePath) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::UploadFileStream", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || dataFilePath.empty()) { _logger.LogError("ANSAWSS3::UploadFileStream", "Bucket name or file path is empty", __FILE__, __LINE__); return false; } bool uploadSuccess = false; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("UploadFileStream", "Failed to acquire S3 connection", __FILE__, __LINE__); return false; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // Set up file stream CkStream fileStream; fileStream.put_SourceFile(dataFilePath.c_str()); // Extract filename for S3 object key std::string fileName = ExtractFileName(dataFilePath); std::string objectPath = _bAwsPath ? ("/" + fileName) : ("/" + bucketName + "/" + fileName); // Upload to S3 // If the application provided the SHA-256 hash of the file contents (as shown above) // then file is streamed and never has to completely reside in memory. const char* responseStr = conn->rest.fullRequestStream("PUT", objectPath.c_str(), fileStream); if (conn->rest.get_LastMethodSuccess() != true) { _logger.LogError("ANSAWSS3::UploadFileStream - Upload failed", conn->rest.lastErrorText(), __FILE__, __LINE__); } else if (conn->rest.get_ResponseStatusCode() != 200) { // Examine the request/response to see what happened. std::string errorMsg = "HTTP " + std::to_string(conn->rest.get_ResponseStatusCode()); if (responseStr != nullptr && responseStr[0] != '\0') { errorMsg += " - " + std::string(responseStr); } _logger.LogError("ANSAWSS3::UploadFileStream", errorMsg, __FILE__, __LINE__); } else { // Success { std::string scheme = _bTls ? "https://" : "http://"; std::string uploadedUrl = scheme + _fullAWSURL + "/" + bucketName + "/" + fileName; uploadedFilePath = uploadedUrl; _logger.LogDebug("ANSAWSS3::UploadFileStream", "Successfully uploaded file: " + fileName + " to bucket: " + bucketName + " | URL: " + uploadedUrl, __FILE__, __LINE__); } uploadSuccess = true; } ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::UploadFileStream", std::string("Exception: ") + e.what(), __FILE__, __LINE__); } return uploadSuccess; } bool ANSAWSS3::UploadMultipartData(const std::string& bucketName,const std::string& dataFilePath, std::string& uploadedFilePath, int partSize) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::UploadMultipartData", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || dataFilePath.empty()) { _logger.LogError("ANSAWSS3::UploadMultipartData", "Bucket name or file path is empty", __FILE__, __LINE__); return false; } // Validate part size (minimum 5MB enforced by AWS) if (partSize < 5242880) { _logger.LogError("ANSAWSS3::UploadMultipartData", "Part size must be at least 5MB (5242880 bytes)", __FILE__, __LINE__); return false; } try { // Check if file exists CkFileAccess fac; if (!fac.FileExists(dataFilePath.c_str())) { _logger.LogError("ANSAWSS3::UploadMultipartData", "File not found: " + dataFilePath, __FILE__, __LINE__); return false; } auto conn = AcquireConnection(); if (!conn) { _logger.LogError("UploadMultipartData", "Failed to acquire S3 connection", __FILE__, __LINE__); return false; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // Extract filename and construct object path std::string fileName = ExtractFileName(dataFilePath); if (fileName.empty()) { _logger.LogError("ANSAWSS3::UploadMultipartData", "Failed to extract filename from path: " + dataFilePath, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } std::string objectPath = _bAwsPath ? ("/" + fileName) : ("/" + bucketName + "/" + fileName); // ==================================================================== // STEP 1: INITIATE MULTIPART UPLOAD // ==================================================================== _logger.LogDebug("ANSAWSS3::UploadMultipartData", "Step 1: Initiating multipart upload for: " + fileName, __FILE__, __LINE__); conn->rest.ClearAllQueryParams(); conn->rest.AddQueryParam("uploads", ""); const char* responseXml = conn->rest.fullRequestNoBody("POST", objectPath.c_str()); if (!conn->rest.get_LastMethodSuccess()) { _logger.LogError("ANSAWSS3::UploadMultipartData - Initiation failed", conn->rest.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } if (conn->rest.get_ResponseStatusCode() != 200) { std::string errorMsg = "HTTP " + std::to_string(conn->rest.get_ResponseStatusCode()); if (responseXml != nullptr && responseXml[0] != '\0') { errorMsg += " - " + std::string(responseXml); } _logger.LogError("ANSAWSS3::UploadMultipartData - Initiation failed", errorMsg, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } // Parse response to get UploadId CkXml xmlInit; if (!xmlInit.LoadXml(responseXml)) { _logger.LogError("ANSAWSS3::UploadMultipartData - XML parse error", xmlInit.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } const char* uploadId = xmlInit.getChildContent("UploadId"); if (uploadId == nullptr || uploadId[0] == '\0') { _logger.LogError("ANSAWSS3::UploadMultipartData", "UploadId not found in response", __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } _logger.LogDebug("ANSAWSS3::UploadMultipartData", "Multipart upload initiated. UploadId: " + std::string(uploadId), __FILE__, __LINE__); // ==================================================================== // STEP 2: UPLOAD PARTS // ==================================================================== _logger.LogDebug("ANSAWSS3::UploadMultipartData", "Step 2: Uploading parts", __FILE__, __LINE__); // Calculate number of parts fac.OpenForRead(dataFilePath.c_str()); int numParts = fac.GetNumBlocks(partSize); fac.FileClose(); _logger.LogDebug("ANSAWSS3::UploadMultipartData", "File will be uploaded in " + std::to_string(numParts) + " parts", __FILE__, __LINE__); // Setup parts list XML file path std::string fileFolderPath = dataFilePath.substr(0, dataFilePath.find_last_of("/\\")); std::string partsListFilePath = fileFolderPath + "/partsList_" + fileName + ".xml"; // Load or create parts list XML CkXml partsListXml; if (fac.FileExists(partsListFilePath.c_str())) { if (!partsListXml.LoadXmlFile(partsListFilePath.c_str())) { _logger.LogDebug("ANSAWSS3::UploadMultipartData", "Failed to load existing parts list, creating new one", __FILE__, __LINE__); } } partsListXml.put_Tag("CompleteMultipartUpload"); // Upload each part CkStringBuilder sbPartNumber; int successfulParts = 0; for (int partNumber = 1; partNumber <= numParts; ++partNumber) { _logger.LogDebug("ANSAWSS3::UploadMultipartData", "Processing part " + std::to_string(partNumber) + " of " + std::to_string(numParts), __FILE__, __LINE__); // Convert part number to string sbPartNumber.Clear(); sbPartNumber.AppendInt(partNumber); // Check if this part was already uploaded bool partAlreadyUploaded = false; int numUploadedParts = partsListXml.get_NumChildren(); if (numUploadedParts > 0) { CkXml* xRec0 = partsListXml.GetChild(0); CkXml* foundRec = xRec0->FindNextRecord("PartNumber", sbPartNumber.getAsString()); if (xRec0->get_LastMethodSuccess()) { partAlreadyUploaded = true; _logger.LogDebug("ANSAWSS3::UploadMultipartData", "Part " + std::to_string(partNumber) + " already uploaded, skipping", __FILE__, __LINE__); successfulParts++; delete foundRec; } delete xRec0; } if (!partAlreadyUploaded) { // Setup stream for this part CkStream fileStream; fileStream.put_SourceFile(dataFilePath.c_str()); fileStream.put_SourceFilePartSize(partSize); fileStream.put_SourceFilePart(partNumber - 1); // 0-based index // Set query parameters for this part conn->rest.ClearAllQueryParams(); conn->rest.AddQueryParam("partNumber", sbPartNumber.getAsString()); conn->rest.AddQueryParam("uploadId", uploadId); // Upload the part - CRITICAL FIX: Use objectPath instead of hardcoded name const char* responseStr = conn->rest.fullRequestStream("PUT", objectPath.c_str(), fileStream); if (!conn->rest.get_LastMethodSuccess()) { _logger.LogError("ANSAWSS3::UploadMultipartData", "Failed to upload part " + std::to_string(partNumber) + ": " + conn->rest.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } if (conn->rest.get_ResponseStatusCode() != 200) { std::string errorMsg = "Part " + std::to_string(partNumber) + " - HTTP " + std::to_string(conn->rest.get_ResponseStatusCode()); if (responseStr != nullptr && responseStr[0] != '\0') { errorMsg += " - " + std::string(responseStr); } _logger.LogError("ANSAWSS3::UploadMultipartData - Upload failed", errorMsg, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } // Get ETag from response header const char* etag = conn->rest.responseHdrByName("ETag"); if (!conn->rest.get_LastMethodSuccess() || etag == nullptr || etag[0] == '\0') { _logger.LogError("ANSAWSS3::UploadMultipartData", "ETag not found in response for part " + std::to_string(partNumber), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } // Add part to XML list CkXml* xPart = partsListXml.NewChild("Part", ""); xPart->NewChildInt2("PartNumber", partNumber); xPart->NewChild2("ETag", etag); delete xPart; // Save parts list after each successful upload if (!partsListXml.SaveXml(partsListFilePath.c_str())) { _logger.LogError("ANSAWSS3::UploadMultipartData", "Failed to save parts list: " + std::string(partsListXml.lastErrorText()), __FILE__, __LINE__); } successfulParts++; _logger.LogDebug("ANSAWSS3::UploadMultipartData", "Part " + std::to_string(partNumber) + " uploaded successfully", __FILE__, __LINE__); } } _logger.LogDebug("ANSAWSS3::UploadMultipartData", "All parts uploaded (" + std::to_string(successfulParts) + "/" + std::to_string(numParts) + ")", __FILE__, __LINE__); // ==================================================================== // STEP 3: COMPLETE MULTIPART UPLOAD // ==================================================================== _logger.LogDebug("ANSAWSS3::UploadMultipartData", "Step 3: Completing multipart upload", __FILE__, __LINE__); conn->rest.ClearAllQueryParams(); conn->rest.AddQueryParam("uploadId", uploadId); responseXml = conn->rest.fullRequestString("POST", objectPath.c_str(), partsListXml.getXml()); if (!conn->rest.get_LastMethodSuccess()) { _logger.LogError("ANSAWSS3::UploadMultipartData - Completion failed", conn->rest.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } if (conn->rest.get_ResponseStatusCode() != 200) { std::string errorMsg = "HTTP " + std::to_string(conn->rest.get_ResponseStatusCode()); if (responseXml != nullptr && responseXml[0] != '\0') { errorMsg += " - " + std::string(responseXml); } _logger.LogError("ANSAWSS3::UploadMultipartData - Completion failed", errorMsg, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } // Parse completion response CkXml xmlComplete; if (xmlComplete.LoadXml(responseXml)) { const char* location = xmlComplete.getChildContent("Location"); const char* eTag = xmlComplete.getChildContent("ETag"); if (location != nullptr) { _logger.LogDebug("ANSAWSS3::UploadMultipartData", "Upload completed. Location: " + std::string(location), __FILE__, __LINE__); } } // Clean up parts list file on success if (fac.FileExists(partsListFilePath.c_str())) { fac.FileDelete(partsListFilePath.c_str()); } std::string scheme = _bTls ? "https://" : "http://"; uploadedFilePath = scheme + _fullAWSURL + "/" + bucketName + "/" + fileName; _logger.LogDebug("ANSAWSS3::UploadMultipartData", "Multipart upload completed successfully for: " + fileName, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return true; } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::UploadMultipartData", e.what(), __FILE__, __LINE__); return false; } } bool ANSAWSS3::UploadPrefixMultipartData(const std::string& bucketName, const std::string& prefix,const std::string& dataFilePath, const std::string& objectName, std::string& uploadedFilePath, int partSize) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::UploadPrefixMultipartData", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || dataFilePath.empty()) { _logger.LogError("ANSAWSS3::UploadPrefixMultipartData", "Bucket name or file path is empty", __FILE__, __LINE__); return false; } // Validate part size (minimum 5MB enforced by AWS) if (partSize < 5242880) { _logger.LogError("ANSAWSS3::UploadPrefixMultipartData", "Part size must be at least 5MB (5242880 bytes)", __FILE__, __LINE__); return false; } try { // Check if file exists CkFileAccess fac; if (!fac.FileExists(dataFilePath.c_str())) { _logger.LogError("ANSAWSS3::UploadPrefixMultipartData", "File not found: " + dataFilePath, __FILE__, __LINE__); return false; } auto conn = AcquireConnection(); if (!conn) { _logger.LogError("UploadPrefixMultipartData", "Failed to acquire S3 connection", __FILE__, __LINE__); return false; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // Extract filename and construct object path with optional prefix std::string fileName = objectName; if (fileName.empty()) { fileName=ExtractFileName(dataFilePath); if (fileName.empty()) { _logger.LogError("ANSAWSS3::UploadPrefixMultipartData", "Failed to extract filename from path: " + dataFilePath, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } } // Construct object path with prefix support std::string objectPath; if (!prefix.empty()) { // Normalize prefix: ensure it doesn't start with / but ends with / std::string normalizedPrefix = prefix; // Remove leading slash if present if (normalizedPrefix[0] == '/') { normalizedPrefix = normalizedPrefix.substr(1); } // Add trailing slash if not present if (!normalizedPrefix.empty() && normalizedPrefix.back() != '/') { normalizedPrefix += '/'; } objectPath = _bAwsPath ? ("/" + normalizedPrefix + fileName) : ("/" + bucketName + "/" + normalizedPrefix + fileName); } else { objectPath = _bAwsPath ? ("/" + fileName) : ("/" + bucketName + "/" + fileName); } _logger.LogDebug("ANSAWSS3::UploadPrefixMultipartData", "Object path: " + objectPath, __FILE__, __LINE__); // ==================================================================== // STEP 1: INITIATE MULTIPART UPLOAD // ==================================================================== _logger.LogDebug("ANSAWSS3::UploadPrefixMultipartData", "Step 1: Initiating multipart upload for: " + fileName, __FILE__, __LINE__); conn->rest.ClearAllQueryParams(); conn->rest.AddQueryParam("uploads", ""); const char* responseXml = conn->rest.fullRequestNoBody("POST", objectPath.c_str()); if (!conn->rest.get_LastMethodSuccess()) { _logger.LogError("ANSAWSS3::UploadPrefixMultipartData - Initiation failed", conn->rest.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } if (conn->rest.get_ResponseStatusCode() != 200) { std::string errorMsg = "HTTP " + std::to_string(conn->rest.get_ResponseStatusCode()); if (responseXml != nullptr && responseXml[0] != '\0') { errorMsg += " - " + std::string(responseXml); } _logger.LogError("ANSAWSS3::UploadPrefixMultipartData - Initiation failed", errorMsg, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } // Parse response to get UploadId CkXml xmlInit; if (!xmlInit.LoadXml(responseXml)) { _logger.LogError("ANSAWSS3::UploadPrefixMultipartData - XML parse error", xmlInit.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } const char* uploadId = xmlInit.getChildContent("UploadId"); if (uploadId == nullptr || uploadId[0] == '\0') { _logger.LogError("ANSAWSS3::UploadPrefixMultipartData", "UploadId not found in response", __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } _logger.LogDebug("ANSAWSS3::UploadPrefixMultipartData", "Multipart upload initiated. UploadId: " + std::string(uploadId), __FILE__, __LINE__); // ==================================================================== // STEP 2: UPLOAD PARTS // ==================================================================== _logger.LogDebug("ANSAWSS3::UploadPrefixMultipartData", "Step 2: Uploading parts", __FILE__, __LINE__); // Calculate number of parts fac.OpenForRead(dataFilePath.c_str()); int numParts = fac.GetNumBlocks(partSize); fac.FileClose(); _logger.LogDebug("ANSAWSS3::UploadPrefixMultipartData", "File will be uploaded in " + std::to_string(numParts) + " parts", __FILE__, __LINE__); // Setup parts list XML file path (use sanitized filename for local storage) std::string fileFolderPath = dataFilePath.substr(0, dataFilePath.find_last_of("/\\")); // Create a safe filename for the parts list (replace path separators) std::string safeFileName = fileName; std::replace(safeFileName.begin(), safeFileName.end(), '/', '_'); std::replace(safeFileName.begin(), safeFileName.end(), '\\', '_'); std::string partsListFilePath = fileFolderPath + "/partsList_" + safeFileName + ".xml"; // Load or create parts list XML CkXml partsListXml; if (fac.FileExists(partsListFilePath.c_str())) { if (!partsListXml.LoadXmlFile(partsListFilePath.c_str())) { _logger.LogDebug("ANSAWSS3::UploadPrefixMultipartData", "Failed to load existing parts list, creating new one", __FILE__, __LINE__); } } partsListXml.put_Tag("CompleteMultipartUpload"); // Upload each part CkStringBuilder sbPartNumber; int successfulParts = 0; for (int partNumber = 1; partNumber <= numParts; ++partNumber) { _logger.LogDebug("ANSAWSS3::UploadPrefixMultipartData", "Processing part " + std::to_string(partNumber) + " of " + std::to_string(numParts), __FILE__, __LINE__); // Convert part number to string sbPartNumber.Clear(); sbPartNumber.AppendInt(partNumber); // Check if this part was already uploaded bool partAlreadyUploaded = false; int numUploadedParts = partsListXml.get_NumChildren(); if (numUploadedParts > 0) { CkXml* xRec0 = partsListXml.GetChild(0); CkXml* foundRec = xRec0->FindNextRecord("PartNumber", sbPartNumber.getAsString()); if (xRec0->get_LastMethodSuccess()) { partAlreadyUploaded = true; _logger.LogDebug("ANSAWSS3::UploadPrefixMultipartData", "Part " + std::to_string(partNumber) + " already uploaded, skipping", __FILE__, __LINE__); successfulParts++; delete foundRec; } delete xRec0; } if (!partAlreadyUploaded) { // Setup stream for this part CkStream fileStream; fileStream.put_SourceFile(dataFilePath.c_str()); fileStream.put_SourceFilePartSize(partSize); fileStream.put_SourceFilePart(partNumber - 1); // 0-based index // Set query parameters for this part conn->rest.ClearAllQueryParams(); conn->rest.AddQueryParam("partNumber", sbPartNumber.getAsString()); conn->rest.AddQueryParam("uploadId", uploadId); // Upload the part using the full object path (with prefix) const char* responseStr = conn->rest.fullRequestStream("PUT", objectPath.c_str(), fileStream); if (!conn->rest.get_LastMethodSuccess()) { _logger.LogError("ANSAWSS3::UploadPrefixMultipartData", "Failed to upload part " + std::to_string(partNumber) + ": " + conn->rest.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } if (conn->rest.get_ResponseStatusCode() != 200) { std::string errorMsg = "Part " + std::to_string(partNumber) + " - HTTP " + std::to_string(conn->rest.get_ResponseStatusCode()); if (responseStr != nullptr && responseStr[0] != '\0') { errorMsg += " - " + std::string(responseStr); } _logger.LogError("ANSAWSS3::UploadPrefixMultipartData - Upload failed", errorMsg, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } // Get ETag from response header const char* etag = conn->rest.responseHdrByName("ETag"); if (!conn->rest.get_LastMethodSuccess() || etag == nullptr || etag[0] == '\0') { _logger.LogError("ANSAWSS3::UploadPrefixMultipartData", "ETag not found in response for part " + std::to_string(partNumber), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } // Add part to XML list CkXml* xPart = partsListXml.NewChild("Part", ""); xPart->NewChildInt2("PartNumber", partNumber); xPart->NewChild2("ETag", etag); delete xPart; // Save parts list after each successful upload if (!partsListXml.SaveXml(partsListFilePath.c_str())) { _logger.LogError("ANSAWSS3::UploadPrefixMultipartData", "Failed to save parts list: " + std::string(partsListXml.lastErrorText()), __FILE__, __LINE__); } successfulParts++; _logger.LogDebug("ANSAWSS3::UploadPrefixMultipartData", "Part " + std::to_string(partNumber) + " uploaded successfully", __FILE__, __LINE__); } } _logger.LogDebug("ANSAWSS3::UploadPrefixMultipartData", "All parts uploaded (" + std::to_string(successfulParts) + "/" + std::to_string(numParts) + ")", __FILE__, __LINE__); // ==================================================================== // STEP 3: COMPLETE MULTIPART UPLOAD // ==================================================================== _logger.LogDebug("ANSAWSS3::UploadPrefixMultipartData", "Step 3: Completing multipart upload", __FILE__, __LINE__); conn->rest.ClearAllQueryParams(); conn->rest.AddQueryParam("uploadId", uploadId); responseXml = conn->rest.fullRequestString("POST", objectPath.c_str(), partsListXml.getXml()); if (!conn->rest.get_LastMethodSuccess()) { _logger.LogError("ANSAWSS3::UploadPrefixMultipartData - Completion failed", conn->rest.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } if (conn->rest.get_ResponseStatusCode() != 200) { std::string errorMsg = "HTTP " + std::to_string(conn->rest.get_ResponseStatusCode()); if (responseXml != nullptr && responseXml[0] != '\0') { errorMsg += " - " + std::string(responseXml); } _logger.LogError("ANSAWSS3::UploadPrefixMultipartData - Completion failed", errorMsg, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } // Parse completion response CkXml xmlComplete; if (xmlComplete.LoadXml(responseXml)) { const char* location = xmlComplete.getChildContent("Location"); const char* eTag = xmlComplete.getChildContent("ETag"); if (location != nullptr) { _logger.LogDebug("ANSAWSS3::UploadPrefixMultipartData", "Upload completed. Location: " + std::string(location), __FILE__, __LINE__); } } // Clean up parts list file on success if (fac.FileExists(partsListFilePath.c_str())) { fac.FileDelete(partsListFilePath.c_str()); } std::string scheme = _bTls ? "https://" : "http://"; uploadedFilePath = scheme + _fullAWSURL + "/" + bucketName + objectPath; _logger.LogDebug("ANSAWSS3::UploadPrefixMultipartData", "Multipart upload completed successfully. S3 path: " + objectPath, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return true; } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::UploadPrefixMultipartData", e.what(), __FILE__, __LINE__); return false; } } // Upload jpeg data bool ANSAWSS3::UploadJpegImage(const std::string& bucketName, unsigned char* jpeg_string, int32 bufferLength, const std::string& fileName, std::string& uploadedFilePath) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::UploadJpegImage", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || bufferLength <= 0) { _logger.LogError("ANSAWSS3::UploadJpegImage", "Bucket name or buffer length is invalid", __FILE__, __LINE__); return false; } try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("UploadJpegImage", "Failed to acquire S3 connection", __FILE__, __LINE__); return false; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // Load binary file CkBinData binData; CkByteData jpegBytes; jpegBytes.append2(jpeg_string, static_cast(bufferLength)); binData.AppendBinary(jpegBytes); // Determine content type from file extension std::string contentType = "image/jpeg"; // Set headers conn->rest.AddHeader("Content-Type", contentType.c_str()); conn->rest.AddHeader("Expect", "100-continue"); // Construct S3 object path std::string objectPath = _bAwsPath ? ("/" + fileName) : ("/" + bucketName + "/" + fileName); // Upload to S3 CkStringBuilder sbResponse; if (!conn->rest.FullRequestBd("PUT", objectPath.c_str(), binData, sbResponse)) { _logger.LogError("ANSAWSS3::UploadJpegImage - Upload failed", conn->rest.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } // Check HTTP status code int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode != 200) { std::string errorMsg = "HTTP " + std::to_string(statusCode); std::string response = sbResponse.getAsString(); if (!response.empty()) { errorMsg += " - " + response; } _logger.LogError("ANSAWSS3::UploadJpegImage", errorMsg, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } { std::string scheme = _bTls ? "https://" : "http://"; std::string uploadedUrl = scheme + _fullAWSURL + "/" + bucketName + "/" + fileName; uploadedFilePath = uploadedUrl; _logger.LogDebug("ANSAWSS3::UploadJpegImage", "Successfully uploaded: " + fileName + " to bucket: " + bucketName + " | URL: " + uploadedUrl, __FILE__, __LINE__); } ReleaseConnection(std::move(conn)); return true; } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::UploadJpegImage", e.what(), __FILE__, __LINE__); return false; } } bool ANSAWSS3::UploadPrefixJpegImage(const std::string& bucketName, const std::string& prefix,unsigned char* jpeg_string,int32 bufferLength,const std::string& fileName, std::string& uploadedFilePath) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::UploadJpegImage", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || bufferLength <= 0 || fileName.empty()) { _logger.LogError("ANSAWSS3::UploadJpegImage", "Bucket name, buffer length, or filename is invalid", __FILE__, __LINE__); return false; } try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("UploadPrefixJpegImage", "Failed to acquire S3 connection", __FILE__, __LINE__); return false; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // Load binary file CkBinData binData; CkByteData jpegBytes; jpegBytes.append2(jpeg_string, static_cast(bufferLength)); binData.AppendBinary(jpegBytes); // Set headers conn->rest.AddHeader("Content-Type", "image/jpeg"); conn->rest.AddHeader("Expect", "100-continue"); // Construct S3 object path with prefix std::string objectPath = _bAwsPath ? "/" : ("/" + bucketName + "/"); if (!prefix.empty()) { // Ensure prefix ends with "/" if it doesn't already std::string normalizedPrefix = prefix; if (normalizedPrefix.back() != '/') { normalizedPrefix += '/'; } objectPath += normalizedPrefix; } objectPath += fileName; _logger.LogDebug("ANSAWSS3::UploadJpegImage", "Uploading to: " + objectPath, __FILE__, __LINE__); // Upload to S3 CkStringBuilder sbResponse; if (!conn->rest.FullRequestBd("PUT", objectPath.c_str(), binData, sbResponse)) { _logger.LogError("ANSAWSS3::UploadJpegImage - Upload failed", conn->rest.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } // Check HTTP status code int statusCode = conn->rest.get_ResponseStatusCode(); if (statusCode != 200) { std::string errorMsg = "HTTP " + std::to_string(statusCode); std::string response = sbResponse.getAsString(); if (!response.empty()) { errorMsg += " - " + response; } _logger.LogError("ANSAWSS3::UploadJpegImage", errorMsg, __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } { std::string scheme = _bTls ? "https://" : "http://"; std::string uploadedUrl = scheme + _fullAWSURL + "/" + bucketName + objectPath; uploadedFilePath = uploadedUrl; _logger.LogDebug("ANSAWSS3::UploadPrefixJpegImage", "Successfully uploaded: " + objectPath + " (" + std::to_string(bufferLength) + " bytes) to bucket: " + bucketName + " | URL: " + uploadedUrl, __FILE__, __LINE__); } ReleaseConnection(std::move(conn)); return true; } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::UploadJpegImage", std::string("Exception: ") + e.what(), __FILE__, __LINE__); return false; } } // Downloads bool ANSAWSS3::DownloadFile(const std::string& bucketName, const std::string& objectName, const std::string& saveFilePath) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::DownloadFile", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || objectName.empty() || saveFilePath.empty()) { _logger.LogError("ANSAWSS3::DownloadFile", "Bucket name, object name, or save path is empty", __FILE__, __LINE__); return false; } bool downloadSuccess = false; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("DownloadFile", "Failed to acquire S3 connection", __FILE__, __LINE__); return false; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } // Normalize object name (remove leading slash if present) std::string normalizedObjectName = objectName; if (!normalizedObjectName.empty() && normalizedObjectName[0] == '/') { normalizedObjectName = normalizedObjectName.substr(1); } std::string objectPath = _bAwsPath ? ("/" + normalizedObjectName) : ("/" + bucketName + "/" + normalizedObjectName); _logger.LogDebug("ANSAWSS3::DownloadFile", "Downloading: " + normalizedObjectName + " from bucket: " + bucketName, __FILE__, __LINE__); // Send GET request if (!conn->rest.SendReqNoBody("GET", objectPath.c_str())) { _logger.LogError("ANSAWSS3::DownloadFile - Request failed", conn->rest.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } // Read the response header int responseStatusCode = conn->rest.ReadResponseHeader(); if (responseStatusCode < 0) { _logger.LogError("ANSAWSS3::DownloadFile - Failed to read response header", conn->rest.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } // We expect a 200 response status if the data is coming if (responseStatusCode == 200) { // Determine the final file path std::string finalFilePath; // Check if saveFilePath is a directory or a file CkFileAccess fac; bool isDirectory = false; // If path exists and is a directory, or ends with path separator if (fac.DirExists(saveFilePath.c_str())) { isDirectory = true; } else if (!saveFilePath.empty() && (saveFilePath.back() == '\\' || saveFilePath.back() == '/')) { isDirectory = true; } if (isDirectory) { // Save path is a directory - append object name finalFilePath = saveFilePath; // Ensure path ends with separator if (!finalFilePath.empty() && finalFilePath.back() != '\\' && finalFilePath.back() != '/') { finalFilePath += "\\"; } // Extract just the filename from object name (in case it has path separators) std::string filename = normalizedObjectName; size_t lastSlash = filename.find_last_of("/\\"); if (lastSlash != std::string::npos) { filename = filename.substr(lastSlash + 1); } finalFilePath += filename; } else { // Save path is the complete file path finalFilePath = saveFilePath; } // Ensure the directory exists size_t lastSeparator = finalFilePath.find_last_of("/\\"); if (lastSeparator != std::string::npos) { std::string directory = finalFilePath.substr(0, lastSeparator); // Create directory if it doesn't exist if (!fac.DirExists(directory.c_str())) { if (!fac.DirCreate(directory.c_str())) { _logger.LogError("ANSAWSS3::DownloadFile - Failed to create directory", "Directory: " + directory + " - " + fac.lastErrorText(), __FILE__, __LINE__); ReleaseConnection(std::move(conn)); return false; } _logger.LogDebug("ANSAWSS3::DownloadFile", "Created directory: " + directory, __FILE__, __LINE__); } } // Setup stream to write to file CkStream bodyStream; bodyStream.put_SinkFile(finalFilePath.c_str()); // Read the response body to the stream if (!conn->rest.ReadRespBodyStream(bodyStream, true)) { _logger.LogError("ANSAWSS3::DownloadFile - Failed to read response body", conn->rest.lastErrorText(), __FILE__, __LINE__); } else { _logger.LogDebug("ANSAWSS3::DownloadFile", "Successfully downloaded: " + normalizedObjectName + " to: " + finalFilePath, __FILE__, __LINE__); downloadSuccess = true; } } else { // Handle non-200 response const char* errResponse = conn->rest.readRespBodyString(); std::string errorMsg = "HTTP " + std::to_string(responseStatusCode); if (!conn->rest.get_LastMethodSuccess()) { errorMsg += " - Failed to read error response: " + std::string(conn->rest.lastErrorText()); } else { if (errResponse != nullptr && errResponse[0] != '\0') { errorMsg += " - " + std::string(errResponse); } } _logger.LogError("ANSAWSS3::DownloadFile", errorMsg, __FILE__, __LINE__); } ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::DownloadFile", std::string("Exception: ") + e.what(), __FILE__, __LINE__); } return downloadSuccess; } // Delete bool ANSAWSS3::DeleteBucketObject(const std::string& bucketName, const std::string& objectName) { // Early validation checks if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) { _logger.LogError("ANSAWSS3::DeleteBucketObject", !_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)", __FILE__, __LINE__); return false; } if (bucketName.empty() || objectName.empty()) { _logger.LogError("ANSAWSS3::DeleteBucketObject", "Bucket name or object name is empty", __FILE__, __LINE__); return false; } bool deleteSuccess = false; try { auto conn = AcquireConnection(); if (!conn) { _logger.LogError("DeleteBucketObject", "Failed to acquire S3 connection", __FILE__, __LINE__); return false; } // Set bucket-specific endpoint if (_bAwsPath) { conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str()); } else { conn->rest.put_Host(_fullAWSURL.c_str()); } std::string objectPath = _bAwsPath ? ("/" + objectName) : ("/" + bucketName + "/" + objectName); CkStringBuilder sbResponse; bool success = conn->rest.FullRequestNoBodySb("DELETE", objectPath.c_str(), sbResponse); if (!success) { _logger.LogError("ANSAWSS3::DeleteBucketObject", "Failed to send DELETE request: " + std::string(conn->rest.lastErrorText()), __FILE__, __LINE__); } else { int statusCode = conn->rest.get_ResponseStatusCode(); // S3 returns 204 (No Content) for successful deletion // Also accept 200 as success if (statusCode == 204 || statusCode == 200) { _logger.LogDebug("ANSAWSS3::DeleteBucketObject", "Successfully deleted object: " + objectName + " from bucket: " + bucketName, __FILE__, __LINE__); deleteSuccess = true; } else if (statusCode == 404) { // Object doesn't exist - you may want to treat this as success or failure // depending on your use case // Uncomment the next line if you want to treat "not found" as success: deleteSuccess = true; } else { std::string errorMsg = "HTTP " + std::to_string(statusCode); std::string response = sbResponse.getAsString(); if (!response.empty()) { errorMsg += " - " + response; } _logger.LogError("ANSAWSS3::DeleteBucketObject", errorMsg, __FILE__, __LINE__); } } ReleaseConnection(std::move(conn)); } catch (const std::exception& e) { _logger.LogFatal("ANSAWSS3::DeleteBucketObject", std::string("Exception: ") + e.what(), __FILE__, __LINE__); } return deleteSuccess; } }