2026-03-28 16:54:11 +11:00
#include "ANSUtilities.h"
#include <iostream>
#include <CkFileAccess.h>
#include <CkAuthGoogle.h>
#include <CkSocket.h>
#include <CkGlobal.h>
2026-05-19 10:37:46 +10:00
#include <CkHttp.h>
2026-03-28 16:54:11 +11:00
#include <CkMailMan.h>
#include <CkEmail.h>
#include <CkMht.h>
#include <vector>
#include <CkBinData.h>
#include <CkStringBuilder.h>
#include <CkStream.h>
#include <fstream>
2026-04-17 07:03:03 +10:00
#include <chrono>
#include <thread>
#include <sstream>
#include <atomic>
2026-03-28 16:54:11 +11:00
static bool ansawss3LicenceValid = false ;
static std :: once_flag ansawss3LicenseOnceFlag ;
namespace ANSCENTER
{
static void VerifyGlobalANSAWSS3License ( const std :: string & licenseKey ) {
try {
static const std :: vector < std :: pair < int , std :: string >> licenseChecks = {
{ 1000 , "ANNHUB-LV" },
{ 1001 , "DLHUB-LV" },
{ 1002 , "ODHUB-LV" },
{ 1003 , "ANSVIS" },
{ 1004 , "ANSFR" },
{ 1005 , "ANSOCR" },
{ 1006 , "ANSALPR" },
{ 1007 , "ANSCV" },
{ 1008 , "ANSSRT" }
};
ansawss3LicenceValid = false ;
for ( const auto & [ productId , productName ] : licenseChecks ) {
if ( ANSCENTER :: ANSLicenseHelper :: LicenseVerification ( licenseKey , productId , productName )) {
ansawss3LicenceValid = true ;
break ; // Stop at the first valid license
}
}
}
catch ( const std :: exception & e ) {
ansawss3LicenceValid = false ;
}
}
2026-04-17 07:03:03 +10:00
// Builds a per-call unique suffix for scratch file names used by
// multipart uploads. Combining thread id, instance address, and a
// monotonically-increasing counter guarantees no collision between
// parallel uploads — including two threads uploading the same source
// file at the same time (Fix 2).
static std :: string MakeUniqueMultipartToken ( const void * instancePtr ) {
static std :: atomic < uint64_t > counter { 0 };
std :: ostringstream oss ;
oss << std :: hex
<< std :: hash < std :: thread :: id > {}( std :: this_thread :: get_id ()) << "_"
<< reinterpret_cast < uintptr_t > ( instancePtr ) << "_"
<< counter . fetch_add ( 1 , std :: memory_order_relaxed );
return oss . str ();
}
2026-04-24 12:54:16 +10:00
// ────────────────────────────────────────────────────────────────────────
// SummarizeChilkatError: turn Chilkat's verbose multi-line lastErrorText()
// into a single diagnostic line suitable for _logger.LogError().
//
// Chilkat's log looks like:
// ChilkatLog: Connect: DllDate: ... ChilkatVersion: ... UnlockStatus: ...
// restConnect: domain_or_ip: s3.us-east-1.amazonaws.com
// socket2Connect: connect2: connectImplicitSsl: connectSocket_v2:
// connect_domain: ckDnsResolveDomainIPv4_n: ...
// clientHandshake2: readHandshakeMessages:
// Failed to read beginning of SSL/TLS record.
// readTlsRecord: Socket operation timeout.
// See https://cknotes.com/...
// ConnectFailReason: 103
// Failed.
//
// Diagnostics we care about:
// • the deepest concrete failure reason (the non-label leaf line)
// • ConnectFailReason: <N> if present
// • the target (domain_or_ip / ip_or_domain + port)
// Rest (DllDate, ChilkatVersion, Architecture, VerboseLogging, ...) is noise.
//
// Returns e.g.:
// "Connect failed [reason=103] s3.us-east-1.amazonaws.com:443 — Socket operation timeout (TLS handshake). See https://cknotes.com/failed-to-read-beginning-of-ssl-tls-record-..."
// ────────────────────────────────────────────────────────────────────────
static std :: string SummarizeChilkatError ( const char * raw ) {
if ( ! raw || !* raw ) return "(empty Chilkat log)" ;
std :: string s ( raw );
// Split into trimmed lines.
std :: vector < std :: string > lines ;
{
std :: istringstream iss ( s );
std :: string ln ;
while ( std :: getline ( iss , ln )) {
// Trim leading whitespace (Chilkat indents by level).
size_t a = ln . find_first_not_of ( " \t\r\n " );
if ( a == std :: string :: npos ) continue ;
size_t b = ln . find_last_not_of ( " \t\r\n " );
lines . push_back ( ln . substr ( a , b - a + 1 ));
}
}
if ( lines . empty ()) return "(empty Chilkat log)" ;
// Lines that are pure labels ("foo:" or "--foo") are scaffolding.
auto isLabelLine = []( const std :: string & ln ) {
if ( ln . empty ()) return true ;
if ( ln . rfind ( "--" , 0 ) == 0 ) return true ; // "--foo" close-marker
// "label: value" — keep only if value is non-empty and line doesn't
// look like pure call-stack breadcrumb (trailing ':').
if ( ! ln . empty () && ln . back () == ':' ) return true ;
return false ;
};
// Fields we want to extract.
std :: string target , failReason , tlsHint , leafError , seeUrl ;
for ( auto & ln : lines ) {
if ( ln . rfind ( "domain_or_ip:" , 0 ) == 0 || ln . rfind ( "ip_or_domain:" , 0 ) == 0 ) {
auto p = ln . find ( ':' );
if ( p != std :: string :: npos ) target = ln . substr ( p + 1 );
} else if ( ln . rfind ( "ConnectFailReason:" , 0 ) == 0 ) {
auto p = ln . find ( ':' );
if ( p != std :: string :: npos ) failReason = ln . substr ( p + 1 );
} else if ( ln . rfind ( "port:" , 0 ) == 0 ) {
auto p = ln . find ( ':' );
if ( p != std :: string :: npos ) {
// Append port to target if we have one.
std :: string port = ln . substr ( p + 1 );
// Trim leading space.
size_t a = port . find_first_not_of ( " \t " );
if ( a != std :: string :: npos ) port = port . substr ( a );
if ( ! target . empty ()) target += ":" + port ;
}
} else if ( ln . rfind ( "See http" , 0 ) == 0 || ln . rfind ( "See https" , 0 ) == 0 ) {
seeUrl = ln . substr ( 4 ); // drop "See "
} else if ( ln . rfind ( "Failed to read beginning of SSL/TLS record" , 0 ) == 0 ) {
tlsHint = "TLS handshake — server sent nothing (firewall/AV TLS inspection, proxy, or packet loss)" ;
} else if ( ! isLabelLine ( ln )) {
// Candidate concrete message. Keep the LATEST one that's not a
// stack label; Chilkat's deepest concrete error is usually the
// most specific explanation.
leafError = ln ;
}
}
// Trim leading space on captured fields.
auto trim = []( std :: string & v ) {
size_t a = v . find_first_not_of ( " \t " );
if ( a == std :: string :: npos ) { v . clear (); return ; }
size_t b = v . find_last_not_of ( " \t " );
v = v . substr ( a , b - a + 1 );
};
trim ( target ); trim ( failReason ); trim ( leafError ); trim ( seeUrl );
// Compose.
std :: ostringstream out ;
out << "Chilkat Connect failed" ;
if ( ! failReason . empty ()) out << " [reason=" << failReason << "]" ;
if ( ! target . empty ()) out << " " << target ;
if ( ! leafError . empty ()) out << " — " << leafError ;
if ( ! tlsHint . empty ()) out << " (" << tlsHint << ")" ;
if ( ! seeUrl . empty ()) out << " (" << seeUrl << ")" ;
return out . str ();
}
2026-08-12 21:24:12 +10:00
// ── Per-thread call status ───────────────────────────────────────────────
// Written by every upload entry point, read by the C exports in dllmain so
// the caller learns WHY a call failed (and therefore whether to retry)
// without any change to the exported signatures. Thread-local because a
// single ANSAWSS3 handle is routinely shared by many worker threads — a
// per-instance field would report another worker's outcome.
static thread_local int t_callStatus = ANSAWSS3 :: kCallPermanentError ;
void ANSAWSS3 :: SetCallStatus ( int status ) { t_callStatus = status ; }
int ANSAWSS3 :: LastCallStatus () { return t_callStatus ; }
// ────────────────────────────────────────────────────────────────────────
// SummarizeChilkatRequestError: one line out of a request-level Chilkat log.
//
// The raw log is ~2 KB of call-stack scaffolding per failure. Written once
// per retry, per worker, it buries the one thing an operator needs: what S3
// actually said. S3 returns its reason as XML in the response body, which
// Chilkat echoes into the log:
// <Error><Code>NoSuchBucket</Code><Message>The specified bucket does
// not exist</Message>...
// Pull that out, prefix the status code, and drop the rest.
// ────────────────────────────────────────────────────────────────────────
static std :: string SummarizeChilkatRequestError ( const char * raw , int statusCode ) {
std :: ostringstream out ;
if ( statusCode > 0 ) out << "HTTP " << statusCode ;
auto tagValue = []( const std :: string & s , const char * tag ) -> std :: string {
const std :: string open = std :: string ( "<" ) + tag + ">" ;
const std :: string close = std :: string ( "</" ) + tag + ">" ;
const size_t a = s . find ( open );
if ( a == std :: string :: npos ) return {};
const size_t b = s . find ( close , a + open . size ());
if ( b == std :: string :: npos ) return {};
return s . substr ( a + open . size (), b - a - open . size ());
};
if ( raw && * raw ) {
const std :: string s ( raw );
const std :: string code = tagValue ( s , "Code" );
const std :: string msg = tagValue ( s , "Message" );
if ( ! code . empty ()) {
if ( statusCode > 0 ) out << " " ;
out << code ;
if ( ! msg . empty ()) out << ": " << msg ;
}
else {
// No S3 error document — fall back to the deepest concrete line
// (same idea as SummarizeChilkatError, without the connect
// specific fields).
std :: istringstream iss ( s );
std :: string ln , leaf ;
while ( std :: getline ( iss , ln )) {
const size_t a = ln . find_first_not_of ( " \t\r\n " );
if ( a == std :: string :: npos ) continue ;
const size_t b = ln . find_last_not_of ( " \t\r\n " );
ln = ln . substr ( a , b - a + 1 );
if ( ln . rfind ( "--" , 0 ) == 0 ) continue ; // close marker
if ( ! ln . empty () && ln . back () == ':' ) continue ; // stack label
if ( ln == "Failed." ) continue ;
leaf = ln ;
}
if ( ! leaf . empty ()) {
if ( statusCode > 0 ) out << " — " ;
out << leaf ;
}
}
}
std :: string result = out . str ();
return result . empty () ? "(no detail in Chilkat log)" : result ;
}
2026-04-17 07:03:03 +10:00
// Private helper function to extract file name from a path
// Helper function to extract filename from path
2026-03-28 16:54:11 +11:00
std :: string ANSAWSS3 :: ExtractFileName ( const std :: string & filePath ) {
size_t pos = filePath . find_last_of ( "/ \\ " );
if ( pos != std :: string :: npos ) {
return filePath . substr ( pos + 1 );
}
return filePath ;
}
// Helper function to determine content type from file extension
std :: string ANSAWSS3 :: GetContentType ( const std :: string & filePath ) {
// Default to application/octet-stream for unknown types
std :: string contentType = "application/octet-stream" ;
size_t extPos = filePath . find_last_of ( '.' );
if ( extPos == std :: string :: npos ) {
return contentType ;
}
// Extract and convert extension to lowercase
std :: string fileExt = filePath . substr ( extPos + 1 );
std :: transform ( fileExt . begin (), fileExt . end (), fileExt . begin (), :: tolower );
// Image types
if ( fileExt == "jpg" || fileExt == "jpeg" ) {
contentType = "image/jpeg" ;
}
else if ( fileExt == "png" ) {
contentType = "image/png" ;
}
else if ( fileExt == "gif" ) {
contentType = "image/gif" ;
}
else if ( fileExt == "bmp" ) {
contentType = "image/bmp" ;
}
else if ( fileExt == "webp" ) {
contentType = "image/webp" ;
}
else if ( fileExt == "svg" ) {
contentType = "image/svg+xml" ;
}
else if ( fileExt == "ico" ) {
contentType = "image/x-icon" ;
}
// Video types
else if ( fileExt == "mp4" ) {
contentType = "video/mp4" ;
}
else if ( fileExt == "avi" ) {
contentType = "video/x-msvideo" ;
}
else if ( fileExt == "mov" ) {
contentType = "video/quicktime" ;
}
// Document types
else if ( fileExt == "pdf" ) {
contentType = "application/pdf" ;
}
else if ( fileExt == "json" ) {
contentType = "application/json" ;
}
else if ( fileExt == "xml" ) {
contentType = "application/xml" ;
}
else if ( fileExt == "txt" ) {
contentType = "text/plain" ;
}
return contentType ;
}
ANSAWSS3 :: ANSAWSS3 () {
_unlockCode = "ANSDRC.CB1122026_MEQCIFwO1IFQCG0BhZwsXFO68QUU6mDB5uge4duOsqOJanEyAiAB67ahqnXin4SRy0vIegISgbFlpldmbuS5gbU21GYVqA==" ; // "ANSDRC.CB1082025_Ax6P3M7F8B3d";//
_proxyHost = "" ;
_proxyPort = 0 ;
_proxyUsername = "" ;
_proxyPassword = "" ;
_bProxy = false ;
_serviceName = "s3" ;
}
ANSAWSS3 ::~ ANSAWSS3 () {
StopRetry ();
std :: lock_guard < std :: mutex > lk ( _poolMutex );
_pool . clear (); // CkRest destructors handle disconnect
}
void ANSAWSS3 :: StopRetry () {
_stopRetry = true ;
if ( _retryThread . joinable ()) {
_retryThread . join ();
}
_retryInProgress = false ;
}
// TryConnect — actual connection attempt (caller must hold _configMutex)
2026-05-01 11:16:47 +10:00
//
// First attempts the AWS virtual-host name (s3.<region>.<base>); for
// S3-compatible servers like MinIO/Ceph that hostname does not exist, so
// we fall back to the raw _baseDomain (path-style). The first attempt is
// therefore *expected* to fail on MinIO — we suppress its error-level log
// (logOnFailure=false) so the operator only sees errors when the fallback
// also fails.
2026-03-28 16:54:11 +11:00
bool ANSAWSS3 :: TryConnect ( bool & awsPath ) {
2026-05-01 11:16:47 +10:00
const bool fallbackPossible = ( _fullAWSURL != _baseDomain );
auto testConn = CreateConnection ( /*logOnFailure*/ ! fallbackPossible );
2026-03-28 16:54:11 +11:00
if ( ! testConn ) {
// Fallback: connect directly to baseDomain (MinIO, Ceph, etc.)
std :: string savedFullURL = _fullAWSURL ;
_fullAWSURL = _baseDomain ;
2026-05-01 11:16:47 +10:00
testConn = CreateConnection ( /*logOnFailure*/ true );
2026-03-28 16:54:11 +11:00
if ( ! testConn ) {
_fullAWSURL = savedFullURL ; // restore on failure
return false ;
}
awsPath = false ;
_bAwsPath = false ;
}
else {
awsPath = true ;
_bAwsPath = true ;
}
_bConnected = true ;
// Return test connection to pool for reuse
{
std :: lock_guard < std :: mutex > lk ( _poolMutex );
_pool . push_back ( std :: move ( testConn ));
}
return true ;
}
2026-08-12 21:24:12 +10:00
// Background connect loop, started by Connect() when the endpoint could not
// be reached at all (return code 2).
//
// This loop NEVER gives up on its own. The caller — a LabVIEW VI, an
// unattended AIBOX — calls Connect() exactly once at startup, so if this
// thread returned while still unconnected the handle would stay dead for
// the life of the process and every upload would fail instantly. Backoff
// grows 3 s -> 30 s so a long outage costs little, and the loop only exits
// once connected or when StopRetry() is called.
2026-03-28 16:54:11 +11:00
void ANSAWSS3 :: RetryLoop () {
int attempt = 0 ;
2026-08-12 21:24:12 +10:00
int backoffSec = 3 ;
constexpr int kMaxBackoffSec = 30 ;
2026-03-28 16:54:11 +11:00
while ( ! _stopRetry ) {
2026-08-12 21:24:12 +10:00
// Sleep in 1-second increments so we can respond to stop quickly.
for ( int i = 0 ; i < backoffSec && ! _stopRetry ; ++ i ) {
2026-03-28 16:54:11 +11:00
std :: this_thread :: sleep_for ( std :: chrono :: seconds ( 1 ));
}
if ( _stopRetry ) break ;
attempt ++ ;
2026-05-01 11:16:47 +10:00
// Quick DNS check — is internet available? Try AWS virtual-host
// form first, then fall back to raw _baseDomain for MinIO/Ceph.
2026-03-28 16:54:11 +11:00
{
CkSocket dnsCheck ;
CkString dnsResult ;
2026-05-01 11:16:47 +10:00
bool resolved = dnsCheck . DnsLookup ( _fullAWSURL . c_str (), 3000 , dnsResult );
if ( ! resolved && _fullAWSURL != _baseDomain ) {
CkSocket dnsCheck2 ;
CkString dnsResult2 ;
resolved = dnsCheck2 . DnsLookup ( _baseDomain . c_str (), 3000 , dnsResult2 );
}
if ( ! resolved ) {
2026-03-28 16:54:11 +11:00
_logger . LogDebug ( "ANSAWSS3::RetryLoop" ,
2026-08-12 21:24:12 +10:00
"Retry #" + std :: to_string ( attempt ) + " - no internet, retrying in " +
std :: to_string ( backoffSec ) + "s" ,
2026-03-28 16:54:11 +11:00
__FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
ANS_DBG ( "AWSS3" , "RetryLoop #%d: DNS still failing, next try in %ds" ,
attempt , backoffSec );
backoffSec = ( std :: min )( kMaxBackoffSec , backoffSec * 2 );
2026-03-28 16:54:11 +11:00
continue ;
}
}
// Internet is available — attempt real connection to validate auth/URL
{
std :: lock_guard < std :: mutex > lk ( _configMutex );
bool awsPath = true ;
if ( TryConnect ( awsPath )) {
_logger . LogDebug ( "ANSAWSS3::RetryLoop" ,
"Connected successfully after " + std :: to_string ( attempt ) + " retries" ,
__FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
ANS_DBG ( "AWSS3" , "RetryLoop: connected after %d attempt(s)" , attempt );
2026-03-28 16:54:11 +11:00
_retryInProgress = false ;
return ;
}
}
2026-08-12 21:24:12 +10:00
// DNS resolves but the connection failed — bad credentials, wrong
// URL, or the endpoint is refusing us. Keep trying anyway: the
// caller has no other way back, and this state is often temporary
// (endpoint rebooting, TLS interception clearing, clock skew being
// corrected). Backoff keeps the cost negligible.
_logger . LogError ( "ANSAWSS3::RetryLoop" ,
"Retry #" + std :: to_string ( attempt ) +
" - endpoint resolves but connection failed (check authentication or URL). "
"Retrying in " + std :: to_string ( backoffSec ) + "s" ,
__FILE__ , __LINE__ );
ANS_DBG ( "AWSS3" , "RetryLoop #%d: connect failed, next try in %ds" , attempt , backoffSec );
backoffSec = ( std :: min )( kMaxBackoffSec , backoffSec * 2 );
2026-03-28 16:54:11 +11:00
}
_retryInProgress = false ;
}
// ── Connection pool helpers ──
2026-05-01 11:16:47 +10:00
// logOnFailure=false: caller knows this attempt may legitimately fail
// (e.g. TryConnect's first probe of the AWS virtual-host name on a MinIO
// server). Failures are still emitted to ANS_DBG; just not as error-level
// log entries that would scare ops dashboards.
std :: unique_ptr < S3Connection > ANSAWSS3 :: CreateConnection ( bool logOnFailure ) {
2026-03-28 16:54:11 +11:00
auto conn = std :: make_unique < S3Connection > ();
2026-08-12 21:24:12 +10:00
// Stamp the config generation this connection is built against so a
// later Connect / SetAuthentication / SetServerProxy can invalidate it
// even if it lands in the pool after that call cleared it.
conn -> configGen = _configGen . load ();
2026-04-17 07:03:03 +10:00
// Apply request timeouts BEFORE Connect() so ConnectTimeoutMs takes
// effect on this very handshake. IdleTimeoutMs is sticky and applies
2026-08-12 21:24:12 +10:00
// to every subsequent request on this CkRest. A brand-new connection
// gets the FULL idle timeout — it is known-good, so it should be given
// every chance to complete; only reuse gets the short leash.
2026-04-17 07:03:03 +10:00
conn -> rest . put_ConnectTimeoutMs ( _connectTimeoutMs . load ());
conn -> rest . put_IdleTimeoutMs ( _idleTimeoutMs . load ());
2026-03-28 16:54:11 +11:00
// Connect
if ( ! conn -> rest . Connect ( _fullAWSURL . c_str (), _port , _bTls , _bAutoReconnect )) {
2026-04-24 12:54:16 +10:00
const std :: string summary = SummarizeChilkatError ( conn -> rest . lastErrorText ());
2026-05-01 11:16:47 +10:00
if ( logOnFailure ) {
_logger . LogError ( "ANSAWSS3::CreateConnection" , summary , __FILE__ , __LINE__ );
}
ANS_DBG ( "AWSS3" , "Connect failed%s: %s" ,
logOnFailure ? "" : " (probe, retry expected)" , summary . c_str ());
2026-03-28 16:54:11 +11:00
return nullptr ;
}
// Apply proxy if configured
if ( _bProxy ) {
conn -> socket . put_HttpProxyHostname ( _proxyHost . c_str ());
conn -> socket . put_HttpProxyPort ( _proxyPort );
conn -> socket . put_HttpProxyUsername ( _proxyUsername . c_str ());
conn -> socket . put_HttpProxyPassword ( _proxyPassword . c_str ());
conn -> socket . put_HttpProxyForHttp ( _bProxy );
if ( ! conn -> rest . UseConnection ( conn -> socket , true )) {
2026-04-24 12:54:16 +10:00
const std :: string summary = SummarizeChilkatError ( conn -> rest . lastErrorText ());
2026-05-01 11:16:47 +10:00
if ( logOnFailure ) {
_logger . LogError ( "ANSAWSS3::CreateConnection - Proxy error" , summary , __FILE__ , __LINE__ );
}
ANS_DBG ( "AWSS3" , "Proxy error%s: %s" ,
logOnFailure ? "" : " (probe, retry expected)" , summary . c_str ());
2026-03-28 16:54:11 +11:00
return nullptr ;
}
}
// Apply auth
if ( _authReady ) {
conn -> authAws . put_AccessKey ( _accessKey . c_str ());
conn -> authAws . put_SecretKey ( _secretKey . c_str ());
conn -> authAws . put_ServiceName ( _serviceName . c_str ());
if ( _bucketRegion != "us-east-1" ) {
if ( ! _bucketRegion . empty ()) conn -> authAws . put_Region ( _bucketRegion . c_str ());
}
if ( ! conn -> rest . SetAuthAws ( conn -> authAws )) {
2026-04-24 12:54:16 +10:00
const std :: string summary = SummarizeChilkatError ( conn -> rest . lastErrorText ());
2026-05-01 11:16:47 +10:00
if ( logOnFailure ) {
_logger . LogError ( "ANSAWSS3::CreateConnection - Auth error" , summary , __FILE__ , __LINE__ );
}
ANS_DBG ( "AWSS3" , "Auth error%s: %s" ,
logOnFailure ? "" : " (probe, retry expected)" , summary . c_str ());
2026-03-28 16:54:11 +11:00
return nullptr ;
}
}
return conn ;
}
2026-08-12 21:24:12 +10:00
// Take a connection from the pool, skipping any that are stale (idle too
// long) or built against superseded config. Falls through to a fresh
// connection when nothing reusable is left.
2026-03-28 16:54:11 +11:00
std :: unique_ptr < S3Connection > ANSAWSS3 :: AcquireConnection () {
2026-08-12 21:24:12 +10:00
const uint64_t gen = _configGen . load ();
const auto now = std :: chrono :: steady_clock :: now ();
for (;;) {
std :: unique_ptr < S3Connection > conn ;
{
std :: lock_guard < std :: mutex > lk ( _poolMutex );
if ( _pool . empty ()) break ;
conn = std :: move ( _pool . back ());
2026-03-28 16:54:11 +11:00
_pool . pop_back ();
}
2026-08-12 21:24:12 +10:00
if ( conn -> configGen != gen ) {
ANS_DBG ( "AWSS3" , "Acquire: dropping pooled conn from older config gen %llu (now %llu)" ,
( unsigned long long ) conn -> configGen , ( unsigned long long ) gen );
DiscardConnection ( std :: move ( conn )); // pool lock released first
continue ;
}
const long long idleMs = std :: chrono :: duration_cast < std :: chrono :: milliseconds > (
now - conn -> lastUsed ). count ();
if ( idleMs > kMaxPooledIdleMs ) {
ANS_DBG ( "AWSS3" , "Acquire: dropping pooled conn idle for %lldms (> %dms)" ,
idleMs , kMaxPooledIdleMs );
DiscardConnection ( std :: move ( conn ));
continue ;
}
// Clear per-request state left behind by whoever used this
// connection last. Multipart uploads set uploadId / partNumber
// query params; a plain PUT that inherited them would target the
// wrong resource. Content-Encoding is set by UploadTextData only,
// and a JPEG that inherited "gzip" would be stored with metadata
// saying it is compressed — browsers would then fail to display it.
// NOTE: do NOT call ClearAllHeaders here — it wipes Chilkat's SigV4
// state (see commit fcc985e). Headers that every upload sets
// explicitly (Content-Type, Expect) are overwritten by AddHeader
// and need no clearing.
conn -> rest . ClearAllQueryParams ();
conn -> rest . RemoveHeader ( "Content-Encoding" );
// Short leash on reuse: if this socket died while idle, fail fast
// and let the retry run on a fresh connection.
conn -> rest . put_IdleTimeoutMs ( _reusedIdleTimeoutMs . load ());
return conn ;
2026-03-28 16:54:11 +11:00
}
2026-08-12 21:24:12 +10:00
// Pool empty (or everything in it was stale) — create a new connection.
2026-03-28 16:54:11 +11:00
std :: lock_guard < std :: mutex > cfgLk ( _configMutex );
return CreateConnection ();
}
void ANSAWSS3 :: ReleaseConnection ( std :: unique_ptr < S3Connection > conn ) {
if ( ! conn ) return ;
2026-08-12 21:24:12 +10:00
// Mark as fresh and restore the full idle timeout; AcquireConnection
// re-applies the short one when it hands the connection out again.
conn -> lastUsed = std :: chrono :: steady_clock :: now ();
conn -> rest . put_IdleTimeoutMs ( _idleTimeoutMs . load ());
2026-03-28 16:54:11 +11:00
std :: lock_guard < std :: mutex > lk ( _poolMutex );
_pool . push_back ( std :: move ( conn ));
}
2026-08-12 21:24:12 +10:00
// Close a suspect connection instead of pooling it. Disconnect is given a
// small budget only — the point is to release the socket, not to wait on a
// peer that has already stopped answering.
void ANSAWSS3 :: DiscardConnection ( std :: unique_ptr < S3Connection > conn ) {
if ( ! conn ) return ;
conn -> rest . Disconnect ( 100 );
ANS_DBG ( "AWSS3" , "Discarded connection (not returned to pool)" );
// unique_ptr destructor tears the CkRest down here.
}
2026-04-17 07:03:03 +10:00
// Pre-populate the pool with `count` ready-to-use connections. Each
// connection performs a TLS handshake; doing `count` of them in parallel
// (rather than serially as AcquireConnection would on a cold pool) cuts
// the first-upload latency for workloads that fan out right after
// Connect() / SetAuthentication().
int ANSAWSS3 :: PrewarmConnectionPool ( int count ) {
if ( count <= 0 ) return 0 ;
if ( ! _isLicenseValid || ! _isUnlockCodeValid || ! _bConnected ) {
_logger . LogError ( "ANSAWSS3::PrewarmConnectionPool" ,
"Not ready: call Connect() and SetAuthentication() first" ,
__FILE__ , __LINE__ );
return 0 ;
}
// Hold _configMutex once across the whole prewarm. It blocks config
// writes for the duration, but does NOT block individual worker
// threads since CreateConnection() itself does not re-acquire it.
std :: lock_guard < std :: mutex > cfgLk ( _configMutex );
std :: vector < std :: unique_ptr < S3Connection >> built ( count );
std :: vector < std :: thread > workers ;
workers . reserve ( count );
for ( int i = 0 ; i < count ; ++ i ) {
workers . emplace_back ([ this , i , & built ]() {
try {
built [ i ] = CreateConnection ();
} catch (...) {
built [ i ] = nullptr ;
}
});
}
for ( auto & t : workers ) t . join ();
int added = 0 ;
{
std :: lock_guard < std :: mutex > lk ( _poolMutex );
for ( auto & conn : built ) {
if ( conn ) {
_pool . push_back ( std :: move ( conn ));
++ added ;
}
}
}
_logger . LogDebug ( "ANSAWSS3::PrewarmConnectionPool" ,
"Added " + std :: to_string ( added ) + "/" + std :: to_string ( count ) + " connections to pool" ,
__FILE__ , __LINE__ );
return added ;
}
// Update the per-connection request timeouts. New values are applied
// immediately to all pooled connections (IdleTimeoutMs is picked up on
// the next request; ConnectTimeoutMs is harmless on already-connected
// CkRest instances but takes effect if they ever reconnect) and baked
// into the defaults used by future CreateConnection() calls.
bool ANSAWSS3 :: SetTimeouts ( int connectMs , int idleMs ) {
if ( connectMs < 1000 || idleMs < 1000 ) {
_logger . LogError ( "ANSAWSS3::SetTimeouts" ,
"Timeout values must be >= 1000 ms (got connect=" +
std :: to_string ( connectMs ) + ", idle=" + std :: to_string ( idleMs ) + ")" ,
__FILE__ , __LINE__ );
return false ;
}
_connectTimeoutMs . store ( connectMs );
_idleTimeoutMs . store ( idleMs );
2026-08-12 21:24:12 +10:00
// Reuse timeout scales with the configured idle timeout (a quarter of
// it, clamped to a sane floor) so a deployment on a slow link that
// raises idleMs does not end up with an over-aggressive leash on
// pooled sockets.
// Never longer than the configured idle timeout itself (idleMs can be
// as low as 1000), and never so short that a healthy but busy server
// gets cut off.
const int reusedMs = ( std :: min )( idleMs , ( std :: max )( 2000 , idleMs / 4 ));
_reusedIdleTimeoutMs . store ( reusedMs );
2026-04-17 07:03:03 +10:00
// Propagate to existing pooled connections so already-warm workers
// start using the new values on their next request.
{
std :: lock_guard < std :: mutex > lk ( _poolMutex );
for ( auto & conn : _pool ) {
if ( conn ) {
conn -> rest . put_ConnectTimeoutMs ( connectMs );
conn -> rest . put_IdleTimeoutMs ( idleMs );
}
}
}
_logger . LogDebug ( "ANSAWSS3::SetTimeouts" ,
2026-08-12 21:24:12 +10:00
"connect=" + std :: to_string ( connectMs ) + "ms, idle=" + std :: to_string ( idleMs ) +
"ms, idle-on-reuse=" + std :: to_string ( reusedMs ) + "ms" ,
2026-04-17 07:03:03 +10:00
__FILE__ , __LINE__ );
return true ;
}
2026-08-12 21:24:12 +10:00
// Shared precondition check for every upload entry point. The distinction
// matters to the caller: a licence problem needs an operator, whereas "not
// connected" clears itself once the background RetryLoop gets through — so
// the two report different status codes.
bool ANSAWSS3 :: CheckUploadPreconditions ( const std :: string & opName ) {
if ( ! _isLicenseValid || ! _isUnlockCodeValid ) {
_logger . LogError ( opName ,
! _isLicenseValid ? "Invalid license" : "Invalid unlock code" ,
__FILE__ , __LINE__ );
SetCallStatus ( kCallPermanentError );
return false ;
}
if ( ! _bConnected ) {
const bool retrying = _retryInProgress . load ();
_logger . LogError ( opName ,
retrying ? "Not connected (waiting for internet, retrying in background)"
: "Not connected (no internet or connection not established)" ,
__FILE__ , __LINE__ );
ANS_DBG ( "AWSS3" , "%s: not connected (background retry %s) — caller should retry later" ,
opName . c_str (), retrying ? "active" : "inactive" );
SetCallStatus ( kCallNotConnected );
return false ;
}
return true ;
}
// See the declaration comment in ANSUtilities.h for why a failed Chilkat
// request is not automatically a network problem.
ANSAWSS3 :: AttemptResult ANSAWSS3 :: ClassifyFailedRequest (
std :: unique_ptr < S3Connection > conn , std :: string & lastError ) {
const int statusCode = conn ? conn -> rest . get_ResponseStatusCode () : 0 ;
lastError = SummarizeChilkatRequestError (
conn ? conn -> rest . lastErrorText () : nullptr , statusCode );
// The server answered, and its answer will not change on a retry:
// wrong bucket, no permission, bad key. 408 and 429 are the exceptions
// — those explicitly ask the client to try again.
if ( statusCode >= 400 && statusCode <= 499 &&
statusCode != 408 && statusCode != 429 ) {
ANS_DBG ( "AWSS3" , "Request failed permanently: %s" , lastError . c_str ());
ReleaseConnection ( std :: move ( conn )); // the socket itself is fine
return AttemptResult :: Permanent ;
}
ANS_DBG ( "AWSS3" , "Request failed transiently: %s" , lastError . c_str ());
DiscardConnection ( std :: move ( conn )); // suspect socket — do not pool
return AttemptResult :: Transient ;
}
2026-04-17 07:03:03 +10:00
// Drives a retry loop for a single upload operation. See the declaration
// comment in ANSUtilities.h for the AttemptResult contract.
//
// Retry policy (shared across all upload entry points):
2026-08-12 21:24:12 +10:00
// - kUploadMaxAttempts attempts total, AND kUploadDeadlineMs wall-clock
// (whichever comes first — the attempt count alone does not bound how
// long the caller blocks, since one attempt can sit on a timeout)
2026-04-17 07:03:03 +10:00
// - kUploadRetryDelayMs between attempts (no delay after the last one)
// - std::exception thrown inside attemptFn is treated as Transient
// - Any other exception is treated as Permanent (bail immediately)
bool ANSAWSS3 :: UploadWithRetry (
const std :: string & opName ,
const std :: function < AttemptResult ( std :: string & lastError ) >& attemptFn ) {
std :: string lastError ;
2026-08-12 21:24:12 +10:00
const auto started = std :: chrono :: steady_clock :: now ();
auto elapsedMs = [ & started ]() {
return std :: chrono :: duration_cast < std :: chrono :: milliseconds > (
std :: chrono :: steady_clock :: now () - started ). count ();
};
2026-04-17 07:03:03 +10:00
for ( int attempt = 1 ; attempt <= kUploadMaxAttempts ; ++ attempt ) {
AttemptResult result = AttemptResult :: Transient ;
try {
result = attemptFn ( lastError );
}
catch ( const std :: exception & e ) {
lastError = std :: string ( "Exception: " ) + e . what ();
result = AttemptResult :: Transient ;
}
catch (...) {
lastError = "Unknown exception" ;
result = AttemptResult :: Permanent ;
}
if ( result == AttemptResult :: Success ) {
2026-08-12 21:24:12 +10:00
SetCallStatus ( kCallSuccess );
2026-04-17 07:03:03 +10:00
return true ;
}
if ( result == AttemptResult :: Permanent ) {
_logger . LogError ( opName ,
"Permanent failure (no retry): " + lastError ,
__FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
SetCallStatus ( kCallPermanentError );
return false ;
}
// Transient — retry if both budgets allow it.
if ( elapsedMs () >= kUploadDeadlineMs ) {
_logger . LogError ( opName ,
"Upload gave up after " + std :: to_string ( attempt ) + " attempt(s) / " +
std :: to_string ( elapsedMs ()) + "ms (deadline " +
std :: to_string ( kUploadDeadlineMs ) + "ms). Last error: " + lastError ,
__FILE__ , __LINE__ );
ANS_DBG ( "AWSS3" , "%s: deadline exceeded after %d attempt(s), %lldms" ,
opName . c_str (), attempt , elapsedMs ());
// Ran out of time on transient errors — the caller should retry.
SetCallStatus ( kCallTransientError );
2026-04-17 07:03:03 +10:00
return false ;
}
if ( attempt < kUploadMaxAttempts ) {
_logger . LogDebug ( opName ,
"Attempt " + std :: to_string ( attempt ) + "/" +
2026-08-12 21:24:12 +10:00
std :: to_string ( kUploadMaxAttempts ) + " failed after " +
std :: to_string ( elapsedMs ()) + "ms: " + lastError +
2026-04-17 07:03:03 +10:00
" — retrying in " + std :: to_string ( kUploadRetryDelayMs ) + "ms" ,
__FILE__ , __LINE__ );
std :: this_thread :: sleep_for (
std :: chrono :: milliseconds ( kUploadRetryDelayMs ));
}
}
_logger . LogError ( opName ,
"Upload failed after " + std :: to_string ( kUploadMaxAttempts ) +
2026-08-12 21:24:12 +10:00
" attempts / " + std :: to_string ( elapsedMs ()) + "ms. Last error: " + lastError ,
2026-04-17 07:03:03 +10:00
__FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
// Every attempt hit a transient error — worth retrying from the caller.
SetCallStatus ( kCallTransientError );
2026-04-17 07:03:03 +10:00
return false ;
}
2026-03-28 16:54:11 +11:00
void ANSAWSS3 :: CheckLicense () {
// Note: caller (Initialize) already holds _configMutex
try {
// Check once globally
std :: call_once ( ansawss3LicenseOnceFlag , [ this ]() {
VerifyGlobalANSAWSS3License ( _licenseKey );
});
// Update this instance's local license flag
_isLicenseValid = ansawss3LicenceValid ;
}
catch ( const std :: exception & e ) {
this -> _logger . LogFatal ( "ANSAWSS3::CheckLicense. Error:" , e . what (), __FILE__ , __LINE__ );
}
}
void ANSAWSS3 :: CheckUnlockCode () {
// Note: caller (Initialize) already holds _configMutex
try {
CkGlobal glob ;
_unlockCode = "ANSDRC.CB1122026_MEQCIFwO1IFQCG0BhZwsXFO68QUU6mDB5uge4duOsqOJanEyAiAB67ahqnXin4SRy0vIegISgbFlpldmbuS5gbU21GYVqA==" ; // "ANSDRC.CB1082025_Ax6P3M7F8B3d";
_isUnlockCodeValid = glob . UnlockBundle ( _unlockCode . c_str ());
if ( ! _isUnlockCodeValid ) {
_logger . LogFatal ( "ANSAWSS3::CheckUnlockCode" , glob . lastErrorText (), __FILE__ , __LINE__ );
return ;
}
int status = glob . get_UnlockStatus ();
if ( status != 2 ) {
_logger . LogDebug ( "ANSAWSS3::CheckUnlockCode" , "Unlocked in trial mode." , __FILE__ , __LINE__ );
}
}
catch ( const std :: exception & e ) {
_logger . LogFatal ( "ANSAWSS3::CheckUnlockCode" , e . what (), __FILE__ , __LINE__ );
}
}
bool ANSAWSS3 :: Initialize ( const std :: string & licenseKey ) {
std :: lock_guard < std :: mutex > lock ( _configMutex );
try {
_licenseKey = licenseKey ;
CheckLicense ();
CheckUnlockCode ();
return _isLicenseValid && _isUnlockCodeValid ;
}
catch ( const std :: exception & e ) {
_logger . LogFatal ( "ANSAWSS3::Initialize" , e . what (), __FILE__ , __LINE__ );
return false ;
}
}
bool ANSAWSS3 :: SetServerProxy ( const std :: string & proxyHost , int proxyPort , const std :: string & proxyUsername , const std :: string & proxyPassword ) {
// Early validation checks
if ( ! _isLicenseValid || ! _isUnlockCodeValid ) {
_logger . LogError ( "ANSAWSS3::SetServerProxy" ,
! _isLicenseValid ? "Invalid license" : "Invalid unlock code" ,
__FILE__ , __LINE__ );
return false ;
}
std :: lock_guard < std :: mutex > lock ( _configMutex );
try {
_proxyHost = proxyHost ;
_proxyPort = proxyPort ;
_proxyUsername = proxyUsername ;
_proxyPassword = proxyPassword ;
// Simplified proxy validation logic
_bProxy = ! proxyHost . empty () && proxyPort > 0 ;
2026-08-12 21:24:12 +10:00
// Clear pool so new connections pick up proxy config. Bumping the
// generation also invalidates any connection still in flight
// elsewhere, which would otherwise be pooled after this clear.
_configGen . fetch_add ( 1 );
2026-03-28 16:54:11 +11:00
{
std :: lock_guard < std :: mutex > lk ( _poolMutex );
_pool . clear ();
}
return true ;
}
catch ( const std :: exception & e ) {
_logger . LogFatal ( "ANSAWSS3::SetServerProxy" , e . what (), __FILE__ , __LINE__ );
return false ;
}
}
// Returns: 1 = connected, 0 = failed (bad auth/URL), 2 = no internet (background retry started)
int ANSAWSS3 :: Connect ( const std :: string & baseDomain , const std :: string & bucketRegion , const std :: string & serviceName , int port , bool bTls , bool autoReconnect , bool & awsPath )
{
// Early validation checks
if ( ! _isLicenseValid || ! _isUnlockCodeValid ) {
_logger . LogError ( "ANSAWSS3::Connect" ,
! _isLicenseValid ? "Invalid license" : "Invalid unlock code" ,
__FILE__ , __LINE__ );
return 0 ;
}
// Stop any existing background retry
StopRetry ();
std :: lock_guard < std :: mutex > lock ( _configMutex );
try {
// Store parameters — strip http:// or https:// from baseDomain
{
std :: string domain = baseDomain . empty () ? "amazonaws.com" : baseDomain ;
if ( domain . rfind ( "https://" , 0 ) == 0 ) {
domain = domain . substr ( 8 );
bTls = true ; // caller said https, honour it
}
else if ( domain . rfind ( "http://" , 0 ) == 0 ) {
domain = domain . substr ( 7 );
}
// Remove trailing slash if any
if ( ! domain . empty () && domain . back () == '/' ) domain . pop_back ();
_baseDomain = domain ;
}
_bucketRegion = bucketRegion . empty () ? "us-east-1" : bucketRegion ;
_serviceName = serviceName . empty () ? "s3" : serviceName ;
_fullAWSURL = _serviceName + "." + _bucketRegion + "." + _baseDomain ;
_port = port ;
_bTls = bTls ;
_bAutoReconnect = autoReconnect ;
2026-08-12 21:24:12 +10:00
// Clear pool so new connections use updated config (see the
// generation note in SetServerProxy).
_configGen . fetch_add ( 1 );
2026-03-28 16:54:11 +11:00
{
std :: lock_guard < std :: mutex > lk ( _poolMutex );
_pool . clear ();
}
2026-05-01 11:16:47 +10:00
// Check if internet is available (fast DNS check). Try the AWS
// virtual-host form first; if that synthetic name doesn't exist
// (MinIO, Ceph, custom S3-compatible servers), fall back to the
// raw _baseDomain — that is the actual host the user gave us.
2026-03-28 16:54:11 +11:00
{
CkSocket dnsCheck ;
CkString dnsResult ;
2026-05-01 11:16:47 +10:00
bool resolved = dnsCheck . DnsLookup ( _fullAWSURL . c_str (), 3000 , dnsResult );
if ( ! resolved && _fullAWSURL != _baseDomain ) {
CkSocket dnsCheck2 ;
CkString dnsResult2 ;
resolved = dnsCheck2 . DnsLookup ( _baseDomain . c_str (), 3000 , dnsResult2 );
if ( resolved ) {
ANS_DBG ( "AWSS3" ,
"DNS for '%s' failed but '%s' resolved — proceeding with path-style/MinIO fallback" ,
_fullAWSURL . c_str (), _baseDomain . c_str ());
}
}
if ( ! resolved ) {
2026-03-28 16:54:11 +11:00
// No internet — start background retry, return immediately
_logger . LogDebug ( "ANSAWSS3::Connect" ,
2026-05-01 11:16:47 +10:00
"No internet available, starting background retry for '" + _fullAWSURL + "' / '" + _baseDomain + "'..." ,
2026-03-28 16:54:11 +11:00
__FILE__ , __LINE__ );
_stopRetry = false ;
_retryInProgress = true ;
_retryThread = std :: thread ( & ANSAWSS3 :: RetryLoop , this );
return 2 ; // no internet, retrying in background
}
}
// Internet is available — try real connection
if ( TryConnect ( awsPath )) {
return 1 ; // connected successfully
}
// Internet available but connection failed (bad auth, wrong URL, etc.)
return 0 ;
}
catch ( const std :: exception & e ) {
_logger . LogFatal ( "ANSAWSS3::Connect" , e . what (), __FILE__ , __LINE__ );
return 0 ;
}
}
bool ANSAWSS3 :: SetAuthentication ( const std :: string & accessKey , const std :: string & secretKey ) {
// Early validation checks
if ( ! _isLicenseValid || ! _isUnlockCodeValid ) {
_logger . LogError ( "ANSAWSS3::SetAuthentication" ,
! _isLicenseValid ? "Invalid license" : "Invalid unlock code" ,
__FILE__ , __LINE__ );
return false ;
}
2026-04-17 07:03:03 +10:00
// Write credentials + clear the pool under _configMutex. The lock is
// released before the auto-prewarm below so PrewarmConnectionPool
// (which also takes _configMutex) doesn't deadlock.
{
std :: lock_guard < std :: mutex > lock ( _configMutex );
try {
_accessKey = accessKey ;
_secretKey = secretKey ;
_authReady = true ;
2026-08-12 21:24:12 +10:00
// Clear pool so new connections pick up new credentials. The
// generation bump matters most here: the background RetryLoop
// can complete a connect at any moment, and a connection it
// built before _authReady was set carries no SigV4 state — if
// it landed in the pool after this clear, every upload using it
// would come back 403 (a permanent failure, so not retried).
_configGen . fetch_add ( 1 );
2026-03-28 16:54:11 +11:00
std :: lock_guard < std :: mutex > lk ( _poolMutex );
_pool . clear ();
}
2026-04-17 07:03:03 +10:00
catch ( const std :: exception & e ) {
_logger . LogFatal ( "ANSAWSS3::SetAuthentication" , e . what (), __FILE__ , __LINE__ );
return false ;
}
}
2026-03-28 16:54:11 +11:00
2026-04-17 07:03:03 +10:00
// Auto-prewarm the connection pool so the first N concurrent uploads
// don't serialize on TLS handshake. No-op if not connected yet.
static constexpr int kDefaultPrewarmCount = 8 ;
PrewarmConnectionPool ( kDefaultPrewarmCount );
return true ;
2026-03-28 16:54:11 +11:00
}
std :: vector < std :: string > ANSAWSS3 :: ListBuckets () {
// Early validation checks
if ( ! _isLicenseValid || ! _isUnlockCodeValid || ! _bConnected ) {
_logger . LogError ( "ANSAWSS3::ListBuckets" ,
! _isLicenseValid ? "Invalid license" : ! _isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress . load () ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)" ,
__FILE__ , __LINE__ );
return {};
}
try {
auto conn = AcquireConnection ();
if ( ! conn ) {
_logger . LogError ( "ListBuckets" , "Failed to acquire S3 connection" , __FILE__ , __LINE__ );
return {};
}
// Make request and capture response
conn -> rest . put_Host ( _fullAWSURL . c_str ());
CkStringBuilder sbResponse ;
if ( ! conn -> rest . FullRequestNoBodySb ( "GET" , "/" , sbResponse )) {
_logger . LogError ( "ANSAWSS3::ListBuckets - Request failed" ,
conn -> rest . lastErrorText (), __FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
DiscardConnection ( std :: move ( conn )); // transport failed — do not pool a suspect socket
2026-03-28 16:54:11 +11:00
return {};
}
// Check HTTP status code
int statusCode = conn -> rest . get_ResponseStatusCode ();
if ( statusCode != 200 ) {
_logger . LogError ( "ANSAWSS3::ListBuckets - HTTP " + std :: to_string ( statusCode ),
sbResponse . getAsString (), __FILE__ , __LINE__ );
ReleaseConnection ( std :: move ( conn ));
return {};
}
// Parse XML response
CkXml xml ;
if ( ! xml . LoadSb ( sbResponse , true )) {
_logger . LogError ( "ANSAWSS3::ListBuckets - XML parse error" ,
xml . lastErrorText (), __FILE__ , __LINE__ );
ReleaseConnection ( std :: move ( conn ));
return {};
}
// Extract bucket names
std :: vector < std :: string > bucketList ;
int bucketCount = xml . NumChildrenHavingTag ( "Buckets|Bucket" );
if ( bucketCount > 0 ) {
bucketList . reserve ( bucketCount );
for ( int i = 0 ; i < bucketCount ; ++ i ) {
xml . put_I ( i );
const char * name = xml . getChildContent ( "Buckets|Bucket[i]|Name" );
// Validate name before adding
if ( name != nullptr && name [ 0 ] != '\0' ) {
bucketList . emplace_back ( name );
}
}
}
ReleaseConnection ( std :: move ( conn ));
return bucketList ;
}
catch ( const std :: exception & e ) {
_logger . LogFatal ( "ANSAWSS3::ListBuckets" , e . what (), __FILE__ , __LINE__ );
return {};
}
}
std :: vector < std :: string > ANSAWSS3 :: ListBucketObjects ( const std :: string & bucketName ) {
// Early validation checks
if ( ! _isLicenseValid || ! _isUnlockCodeValid || ! _bConnected ) {
_logger . LogError ( "ANSAWSS3::ListBucketObjects" ,
! _isLicenseValid ? "Invalid license" : ! _isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress . load () ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)" ,
__FILE__ , __LINE__ );
return {};
}
if ( bucketName . empty ()) {
_logger . LogError ( "ANSAWSS3::ListBucketObjects" ,
"Bucket name is empty" ,
__FILE__ , __LINE__ );
return {};
}
std :: vector < std :: string > objectList ;
try {
auto conn = AcquireConnection ();
if ( ! conn ) {
_logger . LogError ( "ListBucketObjects" , "Failed to acquire S3 connection" , __FILE__ , __LINE__ );
return {};
}
// Set bucket-specific endpoint
if ( _bAwsPath ) {
conn -> rest . put_Host (( bucketName + "." + _fullAWSURL ). c_str ());
} else {
conn -> rest . put_Host ( _fullAWSURL . c_str ());
}
std :: string marker = "" ;
bool isTruncated = true ;
int pageCount = 0 ;
// Loop to handle pagination
while ( isTruncated ) {
pageCount ++ ;
// Build the request path with marker if available
std :: string basePath = _bAwsPath ? "/" : ( "/" + bucketName + "/" );
std :: string requestPath = basePath ;
if ( ! marker . empty ()) {
requestPath = basePath + "?marker=" + marker ;
}
CkStringBuilder sbResponse ;
bool success = conn -> rest . FullRequestNoBodySb ( "GET" , requestPath . c_str (), sbResponse );
if ( ! success ) {
_logger . LogError ( "ANSAWSS3::ListBucketObjects - Request failed on page " + std :: to_string ( pageCount ),
conn -> rest . lastErrorText (), __FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
DiscardConnection ( std :: move ( conn )); // transport failed — do not pool a suspect socket
2026-03-28 16:54:11 +11:00
return objectList ; // Return what we have so far
}
int statusCode = conn -> rest . get_ResponseStatusCode ();
if ( statusCode != 200 ) {
std :: string errorMsg = "HTTP " + std :: to_string ( statusCode );
std :: string response = sbResponse . getAsString ();
if ( ! response . empty ()) {
errorMsg += " - " + response ;
}
_logger . LogError ( "ANSAWSS3::ListBucketObjects - Page " + std :: to_string ( pageCount ),
errorMsg , __FILE__ , __LINE__ );
ReleaseConnection ( std :: move ( conn ));
return objectList ; // Return what we have so far
}
// Parse XML response
CkXml xml ;
bool loadSuccess = xml . LoadSb ( sbResponse , true );
if ( ! loadSuccess ) {
_logger . LogError ( "ANSAWSS3::ListBucketObjects" ,
"Failed to parse XML response on page " + std :: to_string ( pageCount ) + ": " + std :: string ( xml . lastErrorText ()),
__FILE__ , __LINE__ );
ReleaseConnection ( std :: move ( conn ));
return objectList ; // Return what we have so far
}
// Check if results are truncated (more pages to fetch)
const char * truncatedStr = xml . getChildContent ( "IsTruncated" );
isTruncated = ( truncatedStr != nullptr && std :: string ( truncatedStr ) == "true" );
// Get the marker for next page
if ( isTruncated ) {
// First try to get NextMarker
const char * nextMarker = xml . getChildContent ( "NextMarker" );
if ( nextMarker != nullptr && nextMarker [ 0 ] != '\0' ) {
marker = std :: string ( nextMarker );
}
else {
// If NextMarker is not present, use the last Key as marker
int count = xml . NumChildrenHavingTag ( "Contents" );
if ( count > 0 ) {
xml . put_I ( count - 1 ); // Last element
const char * lastKey = xml . getChildContent ( "Contents[i]|Key" );
if ( lastKey != nullptr && lastKey [ 0 ] != '\0' ) {
marker = std :: string ( lastKey );
}
else {
isTruncated = false ;
}
}
else {
isTruncated = false ;
}
}
}
// Iterate through all Contents elements in this page
int count = xml . NumChildrenHavingTag ( "Contents" );
for ( int i = 0 ; i < count ; i ++ ) {
xml . put_I ( i );
const char * key = xml . getChildContent ( "Contents[i]|Key" );
if ( key != nullptr && key [ 0 ] != '\0' ) {
objectList . emplace_back ( key );
}
}
_logger . LogDebug ( "ANSAWSS3::ListBucketObjects" ,
"Page " + std :: to_string ( pageCount ) + ": Retrieved " + std :: to_string ( count ) +
" objects. Total so far: " + std :: to_string ( objectList . size ()),
__FILE__ , __LINE__ );
}
_logger . LogDebug ( "ANSAWSS3::ListBucketObjects" ,
"Successfully listed " + std :: to_string ( objectList . size ()) +
" objects in bucket: " + bucketName + " (" + std :: to_string ( pageCount ) + " pages)" ,
__FILE__ , __LINE__ );
ReleaseConnection ( std :: move ( conn ));
}
catch ( const std :: exception & e ) {
_logger . LogFatal ( "ANSAWSS3::ListBucketObjects" ,
std :: string ( "Exception: " ) + e . what (),
__FILE__ , __LINE__ );
objectList . clear (); // Ensure we return empty list on exception
}
return objectList ;
}
std :: vector < std :: string > ANSAWSS3 :: ListBucketObjectsWithPrefix ( const std :: string & bucketName , const std :: string & prefix ) {
// Early validation checks
if ( ! _isLicenseValid || ! _isUnlockCodeValid || ! _bConnected ) {
_logger . LogError ( "ANSAWSS3::ListBucketObjectsWithPrefix" ,
! _isLicenseValid ? "Invalid license" : ! _isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress . load () ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)" ,
__FILE__ , __LINE__ );
return {};
}
if ( bucketName . empty ()) {
_logger . LogError ( "ANSAWSS3::ListBucketObjectsWithPrefix" ,
"Bucket name is empty" ,
__FILE__ , __LINE__ );
return {};
}
// prefix can be empty to list all objects
std :: vector < std :: string > objectList ;
try {
auto conn = AcquireConnection ();
if ( ! conn ) {
_logger . LogError ( "ListBucketObjectsWithPrefix" , "Failed to acquire S3 connection" , __FILE__ , __LINE__ );
return {};
}
// Set bucket-specific endpoint
if ( _bAwsPath ) {
conn -> rest . put_Host (( bucketName + "." + _fullAWSURL ). c_str ());
} else {
conn -> rest . put_Host ( _fullAWSURL . c_str ());
}
std :: string marker = "" ;
bool isTruncated = true ;
int pageCount = 0 ;
// Loop to handle pagination
while ( isTruncated ) {
pageCount ++ ;
// Build the request path with prefix and marker
std :: string basePath = _bAwsPath ? "/" : ( "/" + bucketName + "/" );
std :: string requestPath = basePath ;
bool hasParams = false ;
// Add prefix parameter if provided
if ( ! prefix . empty ()) {
requestPath += "?prefix=" + prefix ;
hasParams = true ;
}
// Add marker parameter if available
if ( ! marker . empty ()) {
if ( hasParams ) {
requestPath += "&marker=" + marker ;
}
else {
requestPath += "?marker=" + marker ;
hasParams = true ;
}
}
CkStringBuilder sbResponse ;
bool success = conn -> rest . FullRequestNoBodySb ( "GET" , requestPath . c_str (), sbResponse );
if ( ! success ) {
_logger . LogError ( "ANSAWSS3::ListBucketObjectsWithPrefix - Request failed on page " + std :: to_string ( pageCount ),
conn -> rest . lastErrorText (), __FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
DiscardConnection ( std :: move ( conn )); // transport failed — do not pool a suspect socket
2026-03-28 16:54:11 +11:00
return objectList ; // Return what we have so far
}
int statusCode = conn -> rest . get_ResponseStatusCode ();
if ( statusCode != 200 ) {
std :: string errorMsg = "HTTP " + std :: to_string ( statusCode );
std :: string response = sbResponse . getAsString ();
if ( ! response . empty ()) {
errorMsg += " - " + response ;
}
_logger . LogError ( "ANSAWSS3::ListBucketObjectsWithPrefix - Page " + std :: to_string ( pageCount ),
errorMsg , __FILE__ , __LINE__ );
ReleaseConnection ( std :: move ( conn ));
return objectList ; // Return what we have so far
}
// Parse XML response
CkXml xml ;
bool loadSuccess = xml . LoadSb ( sbResponse , true );
if ( ! loadSuccess ) {
_logger . LogError ( "ANSAWSS3::ListBucketObjectsWithPrefix" ,
"Failed to parse XML response on page " + std :: to_string ( pageCount ) + ": " + std :: string ( xml . lastErrorText ()),
__FILE__ , __LINE__ );
ReleaseConnection ( std :: move ( conn ));
return objectList ; // Return what we have so far
}
// Check if results are truncated (more pages to fetch)
const char * truncatedStr = xml . getChildContent ( "IsTruncated" );
isTruncated = ( truncatedStr != nullptr && std :: string ( truncatedStr ) == "true" );
// Get the marker for next page
if ( isTruncated ) {
// First try to get NextMarker
const char * nextMarker = xml . getChildContent ( "NextMarker" );
if ( nextMarker != nullptr && nextMarker [ 0 ] != '\0' ) {
marker = std :: string ( nextMarker );
}
else {
// If NextMarker is not present, use the last Key as marker
int count = xml . NumChildrenHavingTag ( "Contents" );
if ( count > 0 ) {
xml . put_I ( count - 1 ); // Last element
const char * lastKey = xml . getChildContent ( "Contents[i]|Key" );
if ( lastKey != nullptr && lastKey [ 0 ] != '\0' ) {
marker = std :: string ( lastKey );
}
else {
isTruncated = false ;
}
}
else {
isTruncated = false ;
}
}
}
// Iterate through all Contents elements in this page
int count = xml . NumChildrenHavingTag ( "Contents" );
for ( int i = 0 ; i < count ; i ++ ) {
xml . put_I ( i );
const char * key = xml . getChildContent ( "Contents[i]|Key" );
if ( key != nullptr && key [ 0 ] != '\0' ) {
objectList . emplace_back ( key );
}
}
std :: string prefixInfo = prefix . empty () ? "all objects" : "prefix '" + prefix + "'" ;
_logger . LogDebug ( "ANSAWSS3::ListBucketObjectsWithPrefix" ,
"Page " + std :: to_string ( pageCount ) + ": Retrieved " + std :: to_string ( count ) +
" objects for " + prefixInfo + ". Total so far: " + std :: to_string ( objectList . size ()),
__FILE__ , __LINE__ );
}
std :: string prefixInfo = prefix . empty () ? "" : " with prefix '" + prefix + "'" ;
_logger . LogDebug ( "ANSAWSS3::ListBucketObjectsWithPrefix" ,
"Successfully listed " + std :: to_string ( objectList . size ()) +
" objects in bucket: " + bucketName + prefixInfo + " (" + std :: to_string ( pageCount ) + " pages)" ,
__FILE__ , __LINE__ );
ReleaseConnection ( std :: move ( conn ));
}
catch ( const std :: exception & e ) {
_logger . LogFatal ( "ANSAWSS3::ListBucketObjectsWithPrefix" ,
std :: string ( "Exception: " ) + e . what (),
__FILE__ , __LINE__ );
objectList . clear (); // Ensure we return empty list on exception
}
return objectList ;
}
bool ANSAWSS3 :: CreateBucket ( const std :: string & bucketName ) {
// Early validation checks
if ( ! _isLicenseValid || ! _isUnlockCodeValid || ! _bConnected ) {
_logger . LogError ( "ANSAWSS3::CreateBucket" ,
! _isLicenseValid ? "Invalid license" : ! _isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress . load () ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)" ,
__FILE__ , __LINE__ );
return false ;
}
// Validate input parameters
if ( bucketName . empty ()) {
_logger . LogError ( "ANSAWSS3::CreateBucket" ,
"Bucket name cannot be empty" ,
__FILE__ , __LINE__ );
return false ;
}
bool createSuccess = false ;
try {
auto conn = AcquireConnection ();
if ( ! conn ) {
_logger . LogError ( "CreateBucket" , "Failed to acquire S3 connection" , __FILE__ , __LINE__ );
return false ;
}
// Build global endpoint
std :: string globalEndpoint = _serviceName + "." + _baseDomain ;
CkStringBuilder sbBucketRegion ;
sbBucketRegion . Append ( _bucketRegion . c_str ());
// We only need to specify the LocationConstraint if the bucket's region is NOT us-east-1
CkXml xml ;
if ( ! sbBucketRegion . ContentsEqual ( "us-east-1" , true )) {
xml . put_Tag ( "CreateBucketConfiguration" );
xml . AddAttribute ( "xmlns" , "http://s3.amazonaws.com/doc/2006-03-01/" );
xml . UpdateChildContent ( "LocationConstraint" , _bucketRegion . c_str ());
}
// Set bucket-specific endpoint
if ( _bAwsPath ) {
conn -> rest . put_Host (( bucketName + "." + globalEndpoint ). c_str ());
} else {
conn -> rest . put_Host ( _fullAWSURL . c_str ());
}
// Make the call to create the bucket
std :: string createPath = _bAwsPath ? "/" : ( "/" + bucketName );
const char * responseStr = nullptr ;
if ( ! sbBucketRegion . ContentsEqual ( "us-east-1" , true )) {
responseStr = conn -> rest . fullRequestString ( "PUT" , createPath . c_str (), xml . getXml ());
}
else {
// If the bucket is to be created in the us-east-1 region (the default region)
// just send a PUT with no body
responseStr = conn -> rest . fullRequestNoBody ( "PUT" , createPath . c_str ());
}
if ( conn -> rest . get_LastMethodSuccess () != true ) {
_logger . LogError ( "ANSAWSS3::CreateBucket - Request failed" ,
conn -> rest . lastErrorText (), __FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
DiscardConnection ( std :: move ( conn )); // transport failed — do not pool a suspect socket
2026-03-28 16:54:11 +11:00
return false ;
}
int statusCode = conn -> rest . get_ResponseStatusCode ();
if ( statusCode == 200 ) {
_logger . LogDebug ( "ANSAWSS3::CreateBucket" ,
"Successfully created bucket: " + bucketName + " in region: " + _bucketRegion ,
__FILE__ , __LINE__ );
createSuccess = true ;
}
else {
std :: string errorMsg = "HTTP " + std :: to_string ( statusCode );
if ( responseStr != nullptr && responseStr [ 0 ] != '\0' ) {
CkXml responseXml ;
if ( responseXml . LoadXml ( responseStr )) {
const char * errorCode = responseXml . getChildContent ( "Code" );
const char * errorMessage = responseXml . getChildContent ( "Message" );
if ( errorCode != nullptr ) {
errorMsg += " - " + std :: string ( errorCode );
}
if ( errorMessage != nullptr ) {
errorMsg += ": " + std :: string ( errorMessage );
}
}
else {
errorMsg += " \n " + std :: string ( responseStr );
}
}
if ( statusCode == 409 ) {
_logger . LogError ( "ANSAWSS3::CreateBucket" , errorMsg , __FILE__ , __LINE__ );
}
else {
_logger . LogError ( "ANSAWSS3::CreateBucket" , errorMsg , __FILE__ , __LINE__ );
}
}
ReleaseConnection ( std :: move ( conn ));
}
catch ( const std :: exception & e ) {
_logger . LogFatal ( "ANSAWSS3::CreateBucket" ,
std :: string ( "Exception: " ) + e . what (),
__FILE__ , __LINE__ );
}
return createSuccess ;
}
bool ANSAWSS3 :: DeleteBucket ( const std :: string & bucketName ) {
// Early validation checks
if ( ! _isLicenseValid || ! _isUnlockCodeValid || ! _bConnected ) {
_logger . LogError ( "ANSAWSS3::DeleteBucket" ,
! _isLicenseValid ? "Invalid license" : ! _isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress . load () ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)" ,
__FILE__ , __LINE__ );
return false ;
}
if ( bucketName . empty ()) {
_logger . LogError ( "ANSAWSS3::DeleteBucket" ,
"Bucket name cannot be empty" ,
__FILE__ , __LINE__ );
return false ;
}
bool deleteSuccess = false ;
try {
auto conn = AcquireConnection ();
if ( ! conn ) {
_logger . LogError ( "DeleteBucket" , "Failed to acquire S3 connection" , __FILE__ , __LINE__ );
return false ;
}
// Configure AWS authentication
conn -> rest . ClearAuth ();
conn -> authAws . put_AccessKey ( _accessKey . c_str ());
conn -> authAws . put_SecretKey ( _secretKey . c_str ());
conn -> authAws . put_ServiceName ( _serviceName . c_str ());
bool authSuccess = conn -> rest . SetAuthAws ( conn -> authAws );
if ( ! authSuccess ) {
_logger . LogError ( "ANSAWSS3::DeleteBucket" ,
"Failed to set AWS authentication: " + std :: string ( conn -> rest . lastErrorText ()),
__FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
DiscardConnection ( std :: move ( conn )); // transport failed — do not pool a suspect socket
2026-03-28 16:54:11 +11:00
return false ;
}
// Set bucket-specific endpoint
std :: string globalEndpoint = _serviceName + "." + _baseDomain ;
if ( _bAwsPath ) {
conn -> rest . put_Host (( bucketName + "." + globalEndpoint ). c_str ());
} else {
conn -> rest . put_Host ( _fullAWSURL . c_str ());
}
std :: string deletePath = _bAwsPath ? "/" : ( "/" + bucketName );
CkStringBuilder sbResponse ;
bool success = conn -> rest . FullRequestNoBodySb ( "DELETE" , deletePath . c_str (), sbResponse );
if ( ! success ) {
_logger . LogError ( "ANSAWSS3::DeleteBucket" ,
"Failed to send DELETE request: " + std :: string ( conn -> rest . lastErrorText ()),
__FILE__ , __LINE__ );
}
else {
int statusCode = conn -> rest . get_ResponseStatusCode ();
// S3 returns 204 (No Content) for successful bucket deletion
if ( statusCode == 204 ) {
_logger . LogDebug ( "ANSAWSS3::DeleteBucket" ,
"Successfully deleted bucket: " + bucketName ,
__FILE__ , __LINE__ );
deleteSuccess = true ;
}
else if ( statusCode == 404 ) {
_logger . LogWarn ( "ANSAWSS3::DeleteBucket" ,
"Bucket not found (already deleted?): " + bucketName ,
__FILE__ , __LINE__ );
}
else if ( statusCode == 409 ) {
// Bucket not empty
std :: string errorMsg = "HTTP 409 - Bucket is not empty and cannot be deleted" ;
std :: string response = sbResponse . getAsString ();
if ( ! response . empty ()) {
CkXml responseXml ;
if ( responseXml . LoadXml ( response . c_str ())) {
const char * errorCode = responseXml . getChildContent ( "Code" );
const char * errorMessage = responseXml . getChildContent ( "Message" );
if ( errorCode != nullptr ) {
errorMsg += " - " + std :: string ( errorCode );
}
if ( errorMessage != nullptr ) {
errorMsg += ": " + std :: string ( errorMessage );
}
}
}
_logger . LogError ( "ANSAWSS3::DeleteBucket" , errorMsg , __FILE__ , __LINE__ );
}
else {
std :: string errorMsg = "HTTP " + std :: to_string ( statusCode );
std :: string response = sbResponse . getAsString ();
if ( ! response . empty ()) {
errorMsg += " - " + response ;
}
_logger . LogError ( "ANSAWSS3::DeleteBucket" , errorMsg , __FILE__ , __LINE__ );
}
}
ReleaseConnection ( std :: move ( conn ));
}
catch ( const std :: exception & e ) {
_logger . LogFatal ( "ANSAWSS3::DeleteBucket" ,
std :: string ( "Exception: " ) + e . what (),
__FILE__ , __LINE__ );
}
return deleteSuccess ;
}
bool ANSAWSS3 :: CreateFolder ( const std :: string & bucketName , const std :: string & prefix ) {
// Early validation checks
if ( ! _isLicenseValid || ! _isUnlockCodeValid || ! _bConnected ) {
_logger . LogError ( "ANSAWSS3::CreateFolder" ,
! _isLicenseValid ? "Invalid license" : ! _isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress . load () ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)" ,
__FILE__ , __LINE__ );
return false ;
}
if ( bucketName . empty () || prefix . empty ()) {
_logger . LogError ( "ANSAWSS3::CreateFolder" ,
"Bucket name or folder path is empty" ,
__FILE__ , __LINE__ );
return false ;
}
bool createSuccess = false ;
try {
auto conn = AcquireConnection ();
if ( ! conn ) {
_logger . LogError ( "CreateFolder" , "Failed to acquire S3 connection" , __FILE__ , __LINE__ );
return false ;
}
// Ensure folder path ends with "/"
std :: string normalizedPath = prefix ;
if ( normalizedPath . back () != '/' ) {
normalizedPath += '/' ;
}
// Set bucket-specific endpoint
if ( _bAwsPath ) {
conn -> rest . put_Host (( bucketName + "." + _fullAWSURL ). c_str ());
} else {
conn -> rest . put_Host ( _fullAWSURL . c_str ());
}
// Object path is the folder name with trailing slash
std :: string objectPath = _bAwsPath ? ( "/" + normalizedPath ) : ( "/" + bucketName + "/" + normalizedPath );
// Create a zero-byte object to represent the folder
// Use PUT with empty body
const char * responseStr = conn -> rest . fullRequestNoBody ( "PUT" , objectPath . c_str ());
if ( conn -> rest . get_LastMethodSuccess () != true ) {
_logger . LogError ( "ANSAWSS3::CreateFolder - Request failed" ,
conn -> rest . lastErrorText (), __FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
DiscardConnection ( std :: move ( conn )); // transport failed — do not pool a suspect socket
2026-03-28 16:54:11 +11:00
return false ;
}
int statusCode = conn -> rest . get_ResponseStatusCode ();
if ( statusCode == 200 ) {
_logger . LogDebug ( "ANSAWSS3::CreateFolder" ,
"Successfully created folder: " + normalizedPath + " in bucket: " + bucketName ,
__FILE__ , __LINE__ );
createSuccess = true ;
}
else {
std :: string errorMsg = "HTTP " + std :: to_string ( statusCode );
if ( responseStr != nullptr && responseStr [ 0 ] != '\0' ) {
CkXml responseXml ;
if ( responseXml . LoadXml ( responseStr )) {
const char * errorCode = responseXml . getChildContent ( "Code" );
const char * errorMessage = responseXml . getChildContent ( "Message" );
if ( errorCode != nullptr ) {
errorMsg += " - " + std :: string ( errorCode );
}
if ( errorMessage != nullptr ) {
errorMsg += ": " + std :: string ( errorMessage );
}
}
else {
errorMsg += " - " + std :: string ( responseStr );
}
}
_logger . LogError ( "ANSAWSS3::CreateFolder" , errorMsg , __FILE__ , __LINE__ );
}
ReleaseConnection ( std :: move ( conn ));
}
catch ( const std :: exception & e ) {
_logger . LogFatal ( "ANSAWSS3::CreateFolder" ,
std :: string ( "Exception: " ) + e . what (),
__FILE__ , __LINE__ );
}
return createSuccess ;
}
bool ANSAWSS3 :: DeleteFolder ( const std :: string & bucketName , const std :: string & prefix ) {
// Early validation checks
if ( ! _isLicenseValid || ! _isUnlockCodeValid || ! _bConnected ) {
_logger . LogError ( "ANSAWSS3::DeleteFolder" ,
! _isLicenseValid ? "Invalid license" : ! _isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress . load () ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)" ,
__FILE__ , __LINE__ );
return false ;
}
if ( bucketName . empty () || prefix . empty ()) {
_logger . LogError ( "ANSAWSS3::DeleteFolder" ,
"Bucket name or folder path is empty" ,
__FILE__ , __LINE__ );
return false ;
}
bool deleteSuccess = false ;
try {
auto conn = AcquireConnection ();
if ( ! conn ) {
_logger . LogError ( "DeleteFolder" , "Failed to acquire S3 connection" , __FILE__ , __LINE__ );
return false ;
}
// Ensure folder path ends with "/"
std :: string normalizedPath = prefix ;
if ( normalizedPath . back () != '/' ) {
normalizedPath += '/' ;
}
// Set bucket-specific endpoint
if ( _bAwsPath ) {
conn -> rest . put_Host (( bucketName + "." + _fullAWSURL ). c_str ());
} else {
conn -> rest . put_Host ( _fullAWSURL . c_str ());
}
std :: string objectPath = _bAwsPath ? ( "/" + normalizedPath ) : ( "/" + bucketName + "/" + normalizedPath );
CkStringBuilder sbResponse ;
bool success = conn -> rest . FullRequestNoBodySb ( "DELETE" , objectPath . c_str (), sbResponse );
if ( ! success ) {
_logger . LogError ( "ANSAWSS3::DeleteFolder" ,
"Failed to send DELETE request: " + std :: string ( conn -> rest . lastErrorText ()),
__FILE__ , __LINE__ );
}
else {
int statusCode = conn -> rest . get_ResponseStatusCode ();
// S3 returns 204 (No Content) for successful deletion
if ( statusCode == 204 || statusCode == 200 ) {
_logger . LogDebug ( "ANSAWSS3::DeleteFolder" ,
"Successfully deleted folder marker: " + normalizedPath + " from bucket: " + bucketName ,
__FILE__ , __LINE__ );
deleteSuccess = true ;
}
else if ( statusCode == 404 ) {
_logger . LogWarn ( "ANSAWSS3::DeleteFolder" ,
"Folder marker not found (already deleted?): " + normalizedPath ,
__FILE__ , __LINE__ );
}
else {
std :: string errorMsg = "HTTP " + std :: to_string ( statusCode );
std :: string response = sbResponse . getAsString ();
if ( ! response . empty ()) {
errorMsg += " - " + response ;
}
_logger . LogError ( "ANSAWSS3::DeleteFolder" , errorMsg , __FILE__ , __LINE__ );
}
}
ReleaseConnection ( std :: move ( conn ));
}
catch ( const std :: exception & e ) {
_logger . LogFatal ( "ANSAWSS3::DeleteFolder" ,
std :: string ( "Exception: " ) + e . what (),
__FILE__ , __LINE__ );
}
return deleteSuccess ;
}
std :: string ANSAWSS3 :: GetBucketRegion ( const std :: string & bucketName ) {
// Early validation checks
if ( ! _isLicenseValid || ! _isUnlockCodeValid || ! _bConnected ) {
_logger . LogError ( "ANSAWSS3::GetBucketRegion" ,
! _isLicenseValid ? "Invalid license" : ! _isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress . load () ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)" ,
__FILE__ , __LINE__ );
return "" ;
}
if ( bucketName . empty ()) {
_logger . LogError ( "ANSAWSS3::GetBucketRegion" ,
"Bucket name is empty" ,
__FILE__ , __LINE__ );
return "" ;
}
std :: string region = "" ;
try {
auto conn = AcquireConnection ();
if ( ! conn ) {
_logger . LogError ( "GetBucketRegion" , "Failed to acquire S3 connection" , __FILE__ , __LINE__ );
return "" ;
}
// Set bucket-specific endpoint
if ( _bAwsPath ) {
conn -> rest . put_Host (( bucketName + "." + _fullAWSURL ). c_str ());
} else {
conn -> rest . put_Host ( _fullAWSURL . c_str ());
}
// Send the GET request to query the bucket location
std :: string locationPath = _bAwsPath ? "/?location" : ( "/" + bucketName + "?location" );
const char * strResult = conn -> rest . fullRequestNoBody ( "GET" , locationPath . c_str ());
if ( conn -> rest . get_LastMethodSuccess () != true ) {
_logger . LogError ( "ANSAWSS3::GetBucketRegion - Request failed" ,
conn -> rest . lastErrorText (), __FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
DiscardConnection ( std :: move ( conn )); // transport failed — do not pool a suspect socket
2026-03-28 16:54:11 +11:00
return "" ;
}
int responseStatusCode = conn -> rest . get_ResponseStatusCode ();
if ( responseStatusCode != 200 ) {
std :: string errorMsg = "HTTP " + std :: to_string ( responseStatusCode );
if ( strResult != nullptr && strResult [ 0 ] != '\0' ) {
errorMsg += " - " + std :: string ( strResult );
}
_logger . LogError ( "ANSAWSS3::GetBucketRegion" , errorMsg , __FILE__ , __LINE__ );
ReleaseConnection ( std :: move ( conn ));
return "" ;
}
// Parse XML response
CkXml xml ;
bool success = xml . LoadXml ( strResult );
if ( ! success ) {
_logger . LogError ( "ANSAWSS3::GetBucketRegion - Failed to parse XML" ,
xml . lastErrorText (), __FILE__ , __LINE__ );
ReleaseConnection ( std :: move ( conn ));
return "" ;
}
// Get the region from XML content
const char * regionContent = xml . content ();
if ( regionContent != nullptr && regionContent [ 0 ] != '\0' ) {
region = std :: string ( regionContent );
// AWS returns empty string for us-east-1 (classic region)
// Some implementations may return "null" or empty
if ( region . empty () || region == "null" ) {
region = "us-east-1" ;
_logger . LogDebug ( "ANSAWSS3::GetBucketRegion" ,
"Bucket '" + bucketName + "' is in default region: us-east-1" ,
__FILE__ , __LINE__ );
}
else {
_logger . LogDebug ( "ANSAWSS3::GetBucketRegion" ,
"Bucket '" + bucketName + "' is in region: " + region ,
__FILE__ , __LINE__ );
}
}
else {
// Empty response typically means us-east-1
region = "us-east-1" ;
_logger . LogDebug ( "ANSAWSS3::GetBucketRegion" ,
"Empty response - assuming default region: us-east-1" ,
__FILE__ , __LINE__ );
}
ReleaseConnection ( std :: move ( conn ));
}
catch ( const std :: exception & e ) {
_logger . LogFatal ( "ANSAWSS3::GetBucketRegion" ,
std :: string ( "Exception: " ) + e . what (),
__FILE__ , __LINE__ );
}
return region ;
}
// Uploads text data from a file to the specified S3 bucket
bool ANSAWSS3 :: UploadTextData ( const std :: string & bucketName , const std :: string & textFilePath , std :: string & uploadedFilePath ) {
2026-04-17 07:03:03 +10:00
const std :: string kOp = "ANSAWSS3::UploadTextData" ;
2026-04-17 20:59:21 +10:00
// _logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__);
2026-04-17 07:03:03 +10:00
// Early validation (permanent — do NOT retry)
2026-08-12 21:24:12 +10:00
if ( ! CheckUploadPreconditions ( kOp )) return false ;
2026-03-28 16:54:11 +11:00
if ( bucketName . empty () || textFilePath . empty ()) {
2026-04-17 07:03:03 +10:00
_logger . LogError ( kOp , "Bucket name or file path is empty" , __FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
SetCallStatus ( kCallPermanentError ); // caller bug — retrying cannot help
2026-03-28 16:54:11 +11:00
return false ;
}
2026-04-17 07:03:03 +10:00
// Filename and content type are attempt-independent.
const std :: string fileName = ExtractFileName ( textFilePath );
if ( fileName . empty ()) {
_logger . LogError ( kOp ,
"Failed to extract filename from path: " + textFilePath ,
2026-03-28 16:54:11 +11:00
__FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
SetCallStatus ( kCallPermanentError ); // bad input path — retrying cannot help
2026-04-17 07:03:03 +10:00
return false ;
}
const std :: string contentType = GetContentType ( textFilePath );
const std :: string objectPath =
_bAwsPath ? ( "/" + fileName ) : ( "/" + bucketName + "/" + fileName );
2026-03-28 16:54:11 +11:00
2026-04-17 07:03:03 +10:00
return UploadWithRetry ( kOp ,
[ & ]( std :: string & lastError ) -> AttemptResult {
auto conn = AcquireConnection ();
if ( ! conn ) {
lastError = "Failed to acquire S3 connection" ;
return AttemptResult :: Transient ;
2026-03-28 16:54:11 +11:00
}
2026-04-17 07:03:03 +10:00
if ( _bAwsPath ) {
conn -> rest . put_Host (( bucketName + "." + _fullAWSURL ). c_str ());
} else {
conn -> rest . put_Host ( _fullAWSURL . c_str ());
}
// Read the file on each attempt (cheap; keeps the body fresh).
CkFileAccess fac ;
const char * fileContents = fac . readEntireTextFile ( textFilePath . c_str (), "utf-8" );
if ( ! fac . get_LastMethodSuccess ()) {
// File read failure is permanent (disk/IO, not network).
lastError = std :: string ( "Failed to read file: " ) + fac . lastErrorText ();
ReleaseConnection ( std :: move ( conn ));
return AttemptResult :: Permanent ;
}
if ( fileContents == nullptr || fileContents [ 0 ] == '\0' ) {
lastError = "File is empty: " + textFilePath ;
ReleaseConnection ( std :: move ( conn ));
return AttemptResult :: Permanent ;
}
conn -> rest . AddHeader ( "Content-Type" , contentType . c_str ());
conn -> rest . AddHeader ( "Content-Encoding" , "gzip" );
conn -> rest . AddHeader ( "Expect" , "100-continue" );
const char * responseBodyStr =
conn -> rest . fullRequestString ( "PUT" , objectPath . c_str (), fileContents );
if ( ! conn -> rest . get_LastMethodSuccess ()) {
2026-08-12 21:24:12 +10:00
return ClassifyFailedRequest ( std :: move ( conn ), lastError );
2026-04-17 07:03:03 +10:00
}
int statusCode = conn -> rest . get_ResponseStatusCode ();
if ( statusCode == 200 ) {
std :: string scheme = _bTls ? "https://" : "http://" ;
2026-05-01 11:16:47 +10:00
uploadedFilePath = scheme + ( _bAwsPath ? _fullAWSURL : _baseDomain ) + "/" + bucketName + "/" + fileName ;
2026-04-17 07:03:03 +10:00
_logger . LogDebug ( kOp ,
"Successfully uploaded: " + fileName + " | URL: " + uploadedFilePath ,
__FILE__ , __LINE__ );
ReleaseConnection ( std :: move ( conn ));
return AttemptResult :: Success ;
}
lastError = "HTTP " + std :: to_string ( statusCode );
if ( responseBodyStr != nullptr && responseBodyStr [ 0 ] != '\0' ) {
lastError += " - " + std :: string ( responseBodyStr );
}
2026-03-28 16:54:11 +11:00
ReleaseConnection ( std :: move ( conn ));
2026-04-17 07:03:03 +10:00
bool retryable = ( statusCode >= 500 && statusCode <= 599 )
|| statusCode == 408 || statusCode == 429 ;
return retryable ? AttemptResult :: Transient : AttemptResult :: Permanent ;
});
2026-03-28 16:54:11 +11:00
}
bool ANSAWSS3 :: UploadBinaryData ( const std :: string & bucketName , const std :: string & dataFilePath , std :: string & uploadedFilePath ) {
2026-04-17 07:03:03 +10:00
const std :: string kOp = "ANSAWSS3::UploadBinaryData" ;
2026-04-17 20:59:21 +10:00
// _logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__);
2026-04-17 07:03:03 +10:00
// Early validation (permanent — do NOT retry)
2026-08-12 21:24:12 +10:00
if ( ! CheckUploadPreconditions ( kOp )) return false ;
2026-03-28 16:54:11 +11:00
if ( bucketName . empty () || dataFilePath . empty ()) {
2026-04-17 07:03:03 +10:00
_logger . LogError ( kOp , "Bucket name or file path is empty" , __FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
SetCallStatus ( kCallPermanentError ); // caller bug — retrying cannot help
2026-03-28 16:54:11 +11:00
return false ;
}
2026-04-17 07:03:03 +10:00
const std :: string fileName = ExtractFileName ( dataFilePath );
if ( fileName . empty ()) {
_logger . LogError ( kOp ,
"Failed to extract filename from path: " + dataFilePath ,
2026-03-28 16:54:11 +11:00
__FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
SetCallStatus ( kCallPermanentError ); // bad input path — retrying cannot help
2026-04-17 07:03:03 +10:00
return false ;
}
const std :: string contentType = GetContentType ( dataFilePath );
const std :: string objectPath =
_bAwsPath ? ( "/" + fileName ) : ( "/" + bucketName + "/" + fileName );
2026-03-28 16:54:11 +11:00
2026-04-17 07:03:03 +10:00
return UploadWithRetry ( kOp ,
[ & ]( std :: string & lastError ) -> AttemptResult {
auto conn = AcquireConnection ();
if ( ! conn ) {
lastError = "Failed to acquire S3 connection" ;
return AttemptResult :: Transient ;
2026-03-28 16:54:11 +11:00
}
2026-04-17 07:03:03 +10:00
if ( _bAwsPath ) {
conn -> rest . put_Host (( bucketName + "." + _fullAWSURL ). c_str ());
} else {
conn -> rest . put_Host ( _fullAWSURL . c_str ());
}
CkBinData binData ;
if ( ! binData . LoadFile ( dataFilePath . c_str ())) {
lastError = std :: string ( "Failed to load file: " ) + binData . lastErrorText ();
ReleaseConnection ( std :: move ( conn ));
return AttemptResult :: Permanent ; // disk/IO, not network
}
conn -> rest . AddHeader ( "Content-Type" , contentType . c_str ());
conn -> rest . AddHeader ( "Expect" , "100-continue" );
CkStringBuilder sbResponse ;
if ( ! conn -> rest . FullRequestBd ( "PUT" , objectPath . c_str (), binData , sbResponse )) {
2026-08-12 21:24:12 +10:00
return ClassifyFailedRequest ( std :: move ( conn ), lastError );
2026-04-17 07:03:03 +10:00
}
int statusCode = conn -> rest . get_ResponseStatusCode ();
if ( statusCode == 200 ) {
std :: string scheme = _bTls ? "https://" : "http://" ;
2026-05-01 11:16:47 +10:00
uploadedFilePath = scheme + ( _bAwsPath ? _fullAWSURL : _baseDomain ) + "/" + bucketName + "/" + fileName ;
2026-04-17 07:03:03 +10:00
_logger . LogDebug ( kOp ,
"Successfully uploaded: " + fileName + " | URL: " + uploadedFilePath ,
__FILE__ , __LINE__ );
ReleaseConnection ( std :: move ( conn ));
return AttemptResult :: Success ;
}
lastError = "HTTP " + std :: to_string ( statusCode );
std :: string body = sbResponse . getAsString ();
if ( ! body . empty ()) lastError += " - " + body ;
2026-03-28 16:54:11 +11:00
ReleaseConnection ( std :: move ( conn ));
2026-04-17 07:03:03 +10:00
bool retryable = ( statusCode >= 500 && statusCode <= 599 )
|| statusCode == 408 || statusCode == 429 ;
return retryable ? AttemptResult :: Transient : AttemptResult :: Permanent ;
});
2026-03-28 16:54:11 +11:00
}
bool ANSAWSS3 :: UploadPrefixBinaryData ( const std :: string & bucketName , const std :: string & prefix , const std :: string & dataFilePath , const std :: string & objectName , std :: string & uploadedFilePath ) {
2026-04-17 07:03:03 +10:00
const std :: string kOp = "ANSAWSS3::UploadPrefixBinaryData" ;
2026-04-17 20:59:21 +10:00
//_logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__);
2026-04-17 07:03:03 +10:00
// Early validation (permanent — do NOT retry)
2026-08-12 21:24:12 +10:00
if ( ! CheckUploadPreconditions ( kOp )) return false ;
2026-03-28 16:54:11 +11:00
if ( bucketName . empty () || dataFilePath . empty ()) {
2026-04-17 07:03:03 +10:00
_logger . LogError ( kOp , "Bucket name or file path is empty" , __FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
SetCallStatus ( kCallPermanentError ); // caller bug — retrying cannot help
2026-03-28 16:54:11 +11:00
return false ;
}
2026-04-17 07:03:03 +10:00
// Resolve filename (from objectName, fall back to path).
std :: string fileName = objectName ;
if ( fileName . empty ()) {
fileName = ExtractFileName ( dataFilePath );
2026-03-28 16:54:11 +11:00
if ( fileName . empty ()) {
2026-04-17 07:03:03 +10:00
_logger . LogError ( kOp ,
"Failed to extract filename from path: " + dataFilePath ,
__FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
SetCallStatus ( kCallPermanentError ); // bad input path — retrying cannot help
2026-04-17 07:03:03 +10:00
return false ;
}
}
// Build once: the object key and the HTTP request path. Keeping
// them separate avoids the path-style bucket-duplication bug in
// the returned URL.
std :: string objectKey ; // "<prefix>/<fileName>" — no leading '/'
if ( ! prefix . empty ()) {
std :: string normalizedPrefix = prefix ;
if ( ! normalizedPrefix . empty () && normalizedPrefix . front () == '/' ) {
normalizedPrefix = normalizedPrefix . substr ( 1 );
}
if ( ! normalizedPrefix . empty () && normalizedPrefix . back () != '/' ) {
normalizedPrefix += '/' ;
}
objectKey = normalizedPrefix ;
}
objectKey += fileName ;
const std :: string objectPath = _bAwsPath
? ( "/" + objectKey )
: ( "/" + bucketName + "/" + objectKey );
const std :: string contentType = GetContentType ( dataFilePath );
return UploadWithRetry ( kOp ,
[ & ]( std :: string & lastError ) -> AttemptResult {
auto conn = AcquireConnection ();
if ( ! conn ) {
lastError = "Failed to acquire S3 connection" ;
return AttemptResult :: Transient ;
}
if ( _bAwsPath ) {
conn -> rest . put_Host (( bucketName + "." + _fullAWSURL ). c_str ());
} else {
conn -> rest . put_Host ( _fullAWSURL . c_str ());
}
CkBinData binData ;
if ( ! binData . LoadFile ( dataFilePath . c_str ())) {
lastError = std :: string ( "Failed to load file: " ) + binData . lastErrorText ();
ReleaseConnection ( std :: move ( conn ));
return AttemptResult :: Permanent ;
}
conn -> rest . AddHeader ( "Content-Type" , contentType . c_str ());
conn -> rest . AddHeader ( "Expect" , "100-continue" );
CkStringBuilder sbResponse ;
if ( ! conn -> rest . FullRequestBd ( "PUT" , objectPath . c_str (), binData , sbResponse )) {
2026-08-12 21:24:12 +10:00
return ClassifyFailedRequest ( std :: move ( conn ), lastError );
2026-04-17 07:03:03 +10:00
}
int statusCode = conn -> rest . get_ResponseStatusCode ();
if ( statusCode == 200 ) {
// Build canonical path-style URL from parts — independent
// of the addressing style we actually used.
std :: string scheme = _bTls ? "https://" : "http://" ;
2026-05-01 11:16:47 +10:00
uploadedFilePath = scheme + ( _bAwsPath ? _fullAWSURL : _baseDomain ) + "/" + bucketName + "/" + objectKey ;
2026-04-17 07:03:03 +10:00
_logger . LogDebug ( kOp ,
"Successfully uploaded to: " + objectKey + " | URL: " + uploadedFilePath ,
2026-03-28 16:54:11 +11:00
__FILE__ , __LINE__ );
ReleaseConnection ( std :: move ( conn ));
2026-04-17 07:03:03 +10:00
return AttemptResult :: Success ;
2026-03-28 16:54:11 +11:00
}
2026-04-17 07:03:03 +10:00
lastError = "HTTP " + std :: to_string ( statusCode );
std :: string body = sbResponse . getAsString ();
if ( ! body . empty ()) lastError += " - " + body ;
2026-03-28 16:54:11 +11:00
ReleaseConnection ( std :: move ( conn ));
2026-04-17 07:03:03 +10:00
bool retryable = ( statusCode >= 500 && statusCode <= 599 )
|| statusCode == 408 || statusCode == 429 ;
return retryable ? AttemptResult :: Transient : AttemptResult :: Permanent ;
});
2026-03-28 16:54:11 +11:00
}
bool ANSAWSS3 :: UploadFileStream ( const std :: string & bucketName , const std :: string & dataFilePath , std :: string & uploadedFilePath ) {
2026-04-17 07:03:03 +10:00
const std :: string kOp = "ANSAWSS3::UploadFileStream" ;
2026-04-17 20:59:21 +10:00
//_logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__);
2026-04-17 07:03:03 +10:00
// Early validation (permanent — do NOT retry)
2026-08-12 21:24:12 +10:00
if ( ! CheckUploadPreconditions ( kOp )) return false ;
2026-03-28 16:54:11 +11:00
if ( bucketName . empty () || dataFilePath . empty ()) {
2026-04-17 07:03:03 +10:00
_logger . LogError ( kOp , "Bucket name or file path is empty" , __FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
SetCallStatus ( kCallPermanentError ); // caller bug — retrying cannot help
2026-03-28 16:54:11 +11:00
return false ;
}
2026-04-17 07:03:03 +10:00
const std :: string fileName = ExtractFileName ( dataFilePath );
const std :: string objectPath =
_bAwsPath ? ( "/" + fileName ) : ( "/" + bucketName + "/" + fileName );
2026-03-28 16:54:11 +11:00
2026-04-17 07:03:03 +10:00
return UploadWithRetry ( kOp ,
[ & ]( std :: string & lastError ) -> AttemptResult {
auto conn = AcquireConnection ();
if ( ! conn ) {
lastError = "Failed to acquire S3 connection" ;
return AttemptResult :: Transient ;
2026-03-28 16:54:11 +11:00
}
2026-04-17 07:03:03 +10:00
if ( _bAwsPath ) {
conn -> rest . put_Host (( bucketName + "." + _fullAWSURL ). c_str ());
} else {
conn -> rest . put_Host ( _fullAWSURL . c_str ());
}
// Fresh stream per attempt (it holds OS file handle state).
CkStream fileStream ;
fileStream . put_SourceFile ( dataFilePath . c_str ());
const char * responseStr =
conn -> rest . fullRequestStream ( "PUT" , objectPath . c_str (), fileStream );
if ( conn -> rest . get_LastMethodSuccess () != true ) {
2026-08-12 21:24:12 +10:00
return ClassifyFailedRequest ( std :: move ( conn ), lastError );
2026-04-17 07:03:03 +10:00
}
int statusCode = conn -> rest . get_ResponseStatusCode ();
if ( statusCode == 200 ) {
2026-03-28 16:54:11 +11:00
std :: string scheme = _bTls ? "https://" : "http://" ;
2026-05-01 11:16:47 +10:00
uploadedFilePath = scheme + ( _bAwsPath ? _fullAWSURL : _baseDomain ) + "/" + bucketName + "/" + fileName ;
2026-04-17 07:03:03 +10:00
_logger . LogDebug ( kOp ,
"Successfully uploaded file: " + fileName + " | URL: " + uploadedFilePath ,
2026-03-28 16:54:11 +11:00
__FILE__ , __LINE__ );
ReleaseConnection ( std :: move ( conn ));
2026-04-17 07:03:03 +10:00
return AttemptResult :: Success ;
2026-03-28 16:54:11 +11:00
}
2026-04-17 07:03:03 +10:00
lastError = "HTTP " + std :: to_string ( statusCode );
if ( responseStr != nullptr && responseStr [ 0 ] != '\0' ) {
lastError += " - " + std :: string ( responseStr );
2026-03-28 16:54:11 +11:00
}
ReleaseConnection ( std :: move ( conn ));
2026-04-17 07:03:03 +10:00
bool retryable = ( statusCode >= 500 && statusCode <= 599 )
|| statusCode == 408 || statusCode == 429 ;
return retryable ? AttemptResult :: Transient : AttemptResult :: Permanent ;
});
}
bool ANSAWSS3 :: UploadMultipartData ( const std :: string & bucketName , const std :: string & dataFilePath , std :: string & uploadedFilePath , int partSize ) {
const std :: string kOp = "ANSAWSS3::UploadMultipartData" ;
2026-04-17 20:59:21 +10:00
//_logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__);
2026-04-17 07:03:03 +10:00
// Early validation (permanent — do NOT retry)
2026-08-12 21:24:12 +10:00
if ( ! CheckUploadPreconditions ( kOp )) return false ;
2026-04-17 07:03:03 +10:00
if ( bucketName . empty () || dataFilePath . empty ()) {
_logger . LogError ( kOp , "Bucket name or file path is empty" , __FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
SetCallStatus ( kCallPermanentError ); // caller bug — retrying cannot help
2026-04-17 07:03:03 +10:00
return false ;
}
if ( partSize < 5242880 ) {
_logger . LogError ( kOp ,
"Part size must be at least 5MB (5242880 bytes)" ,
__FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
SetCallStatus ( kCallPermanentError ); // caller bug — retrying cannot help
2026-04-17 07:03:03 +10:00
return false ;
}
{
CkFileAccess fac ;
if ( ! fac . FileExists ( dataFilePath . c_str ())) {
_logger . LogError ( kOp , "File not found: " + dataFilePath , __FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
SetCallStatus ( kCallPermanentError ); // bad input path — retrying cannot help
2026-03-28 16:54:11 +11:00
return false ;
}
2026-04-17 07:03:03 +10:00
}
2026-03-28 16:54:11 +11:00
2026-04-17 07:03:03 +10:00
const std :: string fileName = ExtractFileName ( dataFilePath );
if ( fileName . empty ()) {
_logger . LogError ( kOp ,
"Failed to extract filename from path: " + dataFilePath ,
__FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
SetCallStatus ( kCallPermanentError ); // bad input path — retrying cannot help
2026-04-17 07:03:03 +10:00
return false ;
}
const std :: string objectPath =
_bAwsPath ? ( "/" + fileName ) : ( "/" + bucketName + "/" + fileName );
const std :: string fileFolderPath =
dataFilePath . substr ( 0 , dataFilePath . find_last_of ( "/ \\ " ));
// Fix 2: include a per-call unique token in the scratch-file name so
// two parallel multipart uploads of the same local file don't share
// the same partsList XML (which would race on create/delete).
const std :: string partsListFilePath =
fileFolderPath + "/partsList_" + fileName + "_" +
MakeUniqueMultipartToken ( this ) + ".xml" ;
// Note on retry semantics for multipart:
// Each attempt performs a fresh Initiate → Parts → Complete cycle. We
// reset the on-disk partsList XML at the start of each attempt because
// ETags returned by a prior attempt are bound to that attempt's
// UploadId and are not valid for a new Initiate. Parts uploaded during
// a failed attempt therefore become orphans on S3 until the service's
// multipart-lifecycle rule cleans them up (default 7 days, configurable).
return UploadWithRetry ( kOp ,
[ & ]( std :: string & lastError ) -> AttemptResult {
CkFileAccess fac ;
// Ensure no stale parts list from a prior attempt.
if ( fac . FileExists ( partsListFilePath . c_str ())) {
fac . FileDelete ( partsListFilePath . c_str ());
2026-03-28 16:54:11 +11:00
}
2026-04-17 07:03:03 +10:00
auto conn = AcquireConnection ();
if ( ! conn ) {
lastError = "Failed to acquire S3 connection" ;
return AttemptResult :: Transient ;
2026-03-28 16:54:11 +11:00
}
2026-04-17 07:03:03 +10:00
if ( _bAwsPath ) {
conn -> rest . put_Host (( bucketName + "." + _fullAWSURL ). c_str ());
} else {
conn -> rest . put_Host ( _fullAWSURL . c_str ());
}
2026-03-28 16:54:11 +11:00
2026-04-17 07:03:03 +10:00
// ---- STEP 1: INITIATE ----
conn -> rest . ClearAllQueryParams ();
conn -> rest . AddQueryParam ( "uploads" , "" );
const char * responseXml = conn -> rest . fullRequestNoBody ( "POST" , objectPath . c_str ());
if ( ! conn -> rest . get_LastMethodSuccess ()) {
2026-08-12 21:24:12 +10:00
{ AttemptResult r = ClassifyFailedRequest ( std :: move ( conn ), lastError );
lastError = "Initiate: " + lastError ; return r ; }
2026-04-17 07:03:03 +10:00
}
int initStatus = conn -> rest . get_ResponseStatusCode ();
if ( initStatus != 200 ) {
lastError = "Initiate HTTP " + std :: to_string ( initStatus );
if ( responseXml != nullptr && responseXml [ 0 ] != '\0' ) lastError += " - " + std :: string ( responseXml );
ReleaseConnection ( std :: move ( conn ));
bool retryable = ( initStatus >= 500 && initStatus <= 599 ) || initStatus == 408 || initStatus == 429 ;
return retryable ? AttemptResult :: Transient : AttemptResult :: Permanent ;
}
CkXml xmlInit ;
if ( ! xmlInit . LoadXml ( responseXml )) {
lastError = std :: string ( "Initiate XML parse error: " ) + xmlInit . lastErrorText ();
ReleaseConnection ( std :: move ( conn ));
return AttemptResult :: Transient ;
}
const char * uploadId = xmlInit . getChildContent ( "UploadId" );
if ( uploadId == nullptr || uploadId [ 0 ] == '\0' ) {
lastError = "UploadId not found in Initiate response" ;
ReleaseConnection ( std :: move ( conn ));
return AttemptResult :: Transient ;
}
_logger . LogDebug ( kOp ,
"Multipart initiated. UploadId: " + std :: string ( uploadId ),
2026-03-28 16:54:11 +11:00
__FILE__ , __LINE__ );
2026-04-17 07:03:03 +10:00
// ---- STEP 2: UPLOAD PARTS ----
fac . OpenForRead ( dataFilePath . c_str ());
int numParts = fac . GetNumBlocks ( partSize );
fac . FileClose ();
2026-03-28 16:54:11 +11:00
2026-04-17 07:03:03 +10:00
CkXml partsListXml ;
partsListXml . put_Tag ( "CompleteMultipartUpload" );
2026-03-28 16:54:11 +11:00
2026-04-17 07:03:03 +10:00
CkStringBuilder sbPartNumber ;
for ( int partNumber = 1 ; partNumber <= numParts ; ++ partNumber ) {
sbPartNumber . Clear ();
sbPartNumber . AppendInt ( partNumber );
2026-03-28 16:54:11 +11:00
CkStream fileStream ;
fileStream . put_SourceFile ( dataFilePath . c_str ());
fileStream . put_SourceFilePartSize ( partSize );
2026-04-17 07:03:03 +10:00
fileStream . put_SourceFilePart ( partNumber - 1 );
2026-03-28 16:54:11 +11:00
conn -> rest . ClearAllQueryParams ();
conn -> rest . AddQueryParam ( "partNumber" , sbPartNumber . getAsString ());
conn -> rest . AddQueryParam ( "uploadId" , uploadId );
const char * responseStr = conn -> rest . fullRequestStream ( "PUT" , objectPath . c_str (), fileStream );
if ( ! conn -> rest . get_LastMethodSuccess ()) {
2026-08-12 21:24:12 +10:00
{ AttemptResult r = ClassifyFailedRequest ( std :: move ( conn ), lastError );
lastError = "Part " + std :: to_string ( partNumber ) + ": " + lastError ; return r ; }
2026-03-28 16:54:11 +11:00
}
2026-04-17 07:03:03 +10:00
int partStatus = conn -> rest . get_ResponseStatusCode ();
if ( partStatus != 200 ) {
lastError = "Part " + std :: to_string ( partNumber ) + " HTTP " + std :: to_string ( partStatus );
if ( responseStr != nullptr && responseStr [ 0 ] != '\0' ) lastError += " - " + std :: string ( responseStr );
2026-03-28 16:54:11 +11:00
ReleaseConnection ( std :: move ( conn ));
2026-04-17 07:03:03 +10:00
bool retryable = ( partStatus >= 500 && partStatus <= 599 ) || partStatus == 408 || partStatus == 429 ;
return retryable ? AttemptResult :: Transient : AttemptResult :: Permanent ;
2026-03-28 16:54:11 +11:00
}
const char * etag = conn -> rest . responseHdrByName ( "ETag" );
if ( ! conn -> rest . get_LastMethodSuccess () || etag == nullptr || etag [ 0 ] == '\0' ) {
2026-04-17 07:03:03 +10:00
lastError = "Part " + std :: to_string ( partNumber ) + ": ETag missing from response" ;
2026-08-12 21:24:12 +10:00
DiscardConnection ( std :: move ( conn )); // transport failed — do not pool a suspect socket
2026-04-17 07:03:03 +10:00
return AttemptResult :: Transient ;
2026-03-28 16:54:11 +11:00
}
CkXml * xPart = partsListXml . NewChild ( "Part" , "" );
xPart -> NewChildInt2 ( "PartNumber" , partNumber );
xPart -> NewChild2 ( "ETag" , etag );
delete xPart ;
2026-04-17 07:03:03 +10:00
// Persist partsList for diagnostics; not used for resume in this attempt.
partsListXml . SaveXml ( partsListFilePath . c_str ());
2026-03-28 16:54:11 +11:00
}
2026-04-17 07:03:03 +10:00
// ---- STEP 3: COMPLETE ----
conn -> rest . ClearAllQueryParams ();
conn -> rest . AddQueryParam ( "uploadId" , uploadId );
responseXml = conn -> rest . fullRequestString ( "POST" , objectPath . c_str (), partsListXml . getXml ());
2026-03-28 16:54:11 +11:00
2026-04-17 07:03:03 +10:00
if ( ! conn -> rest . get_LastMethodSuccess ()) {
2026-08-12 21:24:12 +10:00
{ AttemptResult r = ClassifyFailedRequest ( std :: move ( conn ), lastError );
lastError = "Complete: " + lastError ; return r ; }
2026-04-17 07:03:03 +10:00
}
int completeStatus = conn -> rest . get_ResponseStatusCode ();
if ( completeStatus != 200 ) {
lastError = "Complete HTTP " + std :: to_string ( completeStatus );
if ( responseXml != nullptr && responseXml [ 0 ] != '\0' ) lastError += " - " + std :: string ( responseXml );
ReleaseConnection ( std :: move ( conn ));
bool retryable = ( completeStatus >= 500 && completeStatus <= 599 ) || completeStatus == 408 || completeStatus == 429 ;
return retryable ? AttemptResult :: Transient : AttemptResult :: Permanent ;
}
2026-03-28 16:54:11 +11:00
2026-04-17 07:03:03 +10:00
// Cleanup parts list on success.
if ( fac . FileExists ( partsListFilePath . c_str ())) {
fac . FileDelete ( partsListFilePath . c_str ());
}
2026-03-28 16:54:11 +11:00
2026-04-17 07:03:03 +10:00
std :: string scheme = _bTls ? "https://" : "http://" ;
2026-05-01 11:16:47 +10:00
uploadedFilePath = scheme + ( _bAwsPath ? _fullAWSURL : _baseDomain ) + "/" + bucketName + "/" + fileName ;
2026-04-17 07:03:03 +10:00
_logger . LogDebug ( kOp ,
"Multipart upload completed for: " + fileName + " | URL: " + uploadedFilePath ,
__FILE__ , __LINE__ );
2026-03-28 16:54:11 +11:00
ReleaseConnection ( std :: move ( conn ));
2026-04-17 07:03:03 +10:00
return AttemptResult :: Success ;
});
}
bool ANSAWSS3 :: UploadPrefixMultipartData ( const std :: string & bucketName , const std :: string & prefix , const std :: string & dataFilePath , const std :: string & objectName , std :: string & uploadedFilePath , int partSize ) {
const std :: string kOp = "ANSAWSS3::UploadPrefixMultipartData" ;
2026-04-17 20:59:21 +10:00
//_logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__);
2026-03-28 16:54:11 +11:00
2026-04-17 07:03:03 +10:00
// Early validation (permanent — do NOT retry)
2026-08-12 21:24:12 +10:00
if ( ! CheckUploadPreconditions ( kOp )) return false ;
2026-04-17 07:03:03 +10:00
if ( bucketName . empty () || dataFilePath . empty ()) {
_logger . LogError ( kOp , "Bucket name or file path is empty" , __FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
SetCallStatus ( kCallPermanentError ); // caller bug — retrying cannot help
2026-04-17 07:03:03 +10:00
return false ;
}
if ( partSize < 5242880 ) {
_logger . LogError ( kOp ,
"Part size must be at least 5MB (5242880 bytes)" ,
__FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
SetCallStatus ( kCallPermanentError ); // caller bug — retrying cannot help
2026-04-17 07:03:03 +10:00
return false ;
}
{
CkFileAccess fac ;
if ( ! fac . FileExists ( dataFilePath . c_str ())) {
_logger . LogError ( kOp , "File not found: " + dataFilePath , __FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
SetCallStatus ( kCallPermanentError ); // bad input path — retrying cannot help
2026-04-17 07:03:03 +10:00
return false ;
}
}
std :: string fileName = objectName ;
if ( fileName . empty ()) {
fileName = ExtractFileName ( dataFilePath );
if ( fileName . empty ()) {
_logger . LogError ( kOp ,
"Failed to extract filename from path: " + dataFilePath ,
__FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
SetCallStatus ( kCallPermanentError ); // bad input path — retrying cannot help
2026-04-17 07:03:03 +10:00
return false ;
}
}
// Build once: the object key and the HTTP request path. Keeping
// them separate avoids the path-style bucket-duplication bug in
// the returned URL.
std :: string objectKey ; // "<prefix>/<fileName>" — no leading '/'
if ( ! prefix . empty ()) {
std :: string normalizedPrefix = prefix ;
if ( ! normalizedPrefix . empty () && normalizedPrefix . front () == '/' ) {
normalizedPrefix = normalizedPrefix . substr ( 1 );
}
if ( ! normalizedPrefix . empty () && normalizedPrefix . back () != '/' ) {
normalizedPrefix += '/' ;
}
objectKey = normalizedPrefix ;
}
objectKey += fileName ;
const std :: string objectPath = _bAwsPath
? ( "/" + objectKey )
: ( "/" + bucketName + "/" + objectKey );
// Parts-list file uses a sanitized filename for safe local storage.
// Fix 2: also append a per-call unique token so parallel multipart
// uploads of the same local file don't share the same scratch file.
const std :: string fileFolderPath =
dataFilePath . substr ( 0 , dataFilePath . find_last_of ( "/ \\ " ));
std :: string safeFileName = fileName ;
std :: replace ( safeFileName . begin (), safeFileName . end (), '/' , '_' );
std :: replace ( safeFileName . begin (), safeFileName . end (), '\\' , '_' );
const std :: string partsListFilePath =
fileFolderPath + "/partsList_" + safeFileName + "_" +
MakeUniqueMultipartToken ( this ) + ".xml" ;
// See note in UploadMultipartData regarding multipart retry semantics.
return UploadWithRetry ( kOp ,
[ & ]( std :: string & lastError ) -> AttemptResult {
CkFileAccess fac ;
if ( fac . FileExists ( partsListFilePath . c_str ())) {
fac . FileDelete ( partsListFilePath . c_str ());
}
auto conn = AcquireConnection ();
if ( ! conn ) {
lastError = "Failed to acquire S3 connection" ;
return AttemptResult :: Transient ;
}
if ( _bAwsPath ) {
conn -> rest . put_Host (( bucketName + "." + _fullAWSURL ). c_str ());
} else {
conn -> rest . put_Host ( _fullAWSURL . c_str ());
}
// ---- STEP 1: INITIATE ----
conn -> rest . ClearAllQueryParams ();
conn -> rest . AddQueryParam ( "uploads" , "" );
const char * responseXml = conn -> rest . fullRequestNoBody ( "POST" , objectPath . c_str ());
if ( ! conn -> rest . get_LastMethodSuccess ()) {
2026-08-12 21:24:12 +10:00
{ AttemptResult r = ClassifyFailedRequest ( std :: move ( conn ), lastError );
lastError = "Initiate: " + lastError ; return r ; }
2026-04-17 07:03:03 +10:00
}
int initStatus = conn -> rest . get_ResponseStatusCode ();
if ( initStatus != 200 ) {
lastError = "Initiate HTTP " + std :: to_string ( initStatus );
if ( responseXml != nullptr && responseXml [ 0 ] != '\0' ) lastError += " - " + std :: string ( responseXml );
ReleaseConnection ( std :: move ( conn ));
bool retryable = ( initStatus >= 500 && initStatus <= 599 ) || initStatus == 408 || initStatus == 429 ;
return retryable ? AttemptResult :: Transient : AttemptResult :: Permanent ;
}
CkXml xmlInit ;
if ( ! xmlInit . LoadXml ( responseXml )) {
lastError = std :: string ( "Initiate XML parse error: " ) + xmlInit . lastErrorText ();
ReleaseConnection ( std :: move ( conn ));
return AttemptResult :: Transient ;
}
const char * uploadId = xmlInit . getChildContent ( "UploadId" );
if ( uploadId == nullptr || uploadId [ 0 ] == '\0' ) {
lastError = "UploadId not found in Initiate response" ;
ReleaseConnection ( std :: move ( conn ));
return AttemptResult :: Transient ;
}
_logger . LogDebug ( kOp ,
"Multipart initiated. UploadId: " + std :: string ( uploadId ),
__FILE__ , __LINE__ );
// ---- STEP 2: UPLOAD PARTS ----
fac . OpenForRead ( dataFilePath . c_str ());
int numParts = fac . GetNumBlocks ( partSize );
fac . FileClose ();
CkXml partsListXml ;
partsListXml . put_Tag ( "CompleteMultipartUpload" );
CkStringBuilder sbPartNumber ;
for ( int partNumber = 1 ; partNumber <= numParts ; ++ partNumber ) {
sbPartNumber . Clear ();
sbPartNumber . AppendInt ( partNumber );
CkStream fileStream ;
fileStream . put_SourceFile ( dataFilePath . c_str ());
fileStream . put_SourceFilePartSize ( partSize );
fileStream . put_SourceFilePart ( partNumber - 1 );
conn -> rest . ClearAllQueryParams ();
conn -> rest . AddQueryParam ( "partNumber" , sbPartNumber . getAsString ());
conn -> rest . AddQueryParam ( "uploadId" , uploadId );
const char * responseStr = conn -> rest . fullRequestStream ( "PUT" , objectPath . c_str (), fileStream );
if ( ! conn -> rest . get_LastMethodSuccess ()) {
2026-08-12 21:24:12 +10:00
{ AttemptResult r = ClassifyFailedRequest ( std :: move ( conn ), lastError );
lastError = "Part " + std :: to_string ( partNumber ) + ": " + lastError ; return r ; }
2026-04-17 07:03:03 +10:00
}
int partStatus = conn -> rest . get_ResponseStatusCode ();
if ( partStatus != 200 ) {
lastError = "Part " + std :: to_string ( partNumber ) + " HTTP " + std :: to_string ( partStatus );
if ( responseStr != nullptr && responseStr [ 0 ] != '\0' ) lastError += " - " + std :: string ( responseStr );
ReleaseConnection ( std :: move ( conn ));
bool retryable = ( partStatus >= 500 && partStatus <= 599 ) || partStatus == 408 || partStatus == 429 ;
return retryable ? AttemptResult :: Transient : AttemptResult :: Permanent ;
}
const char * etag = conn -> rest . responseHdrByName ( "ETag" );
if ( ! conn -> rest . get_LastMethodSuccess () || etag == nullptr || etag [ 0 ] == '\0' ) {
lastError = "Part " + std :: to_string ( partNumber ) + ": ETag missing from response" ;
2026-08-12 21:24:12 +10:00
DiscardConnection ( std :: move ( conn )); // transport failed — do not pool a suspect socket
2026-04-17 07:03:03 +10:00
return AttemptResult :: Transient ;
}
CkXml * xPart = partsListXml . NewChild ( "Part" , "" );
xPart -> NewChildInt2 ( "PartNumber" , partNumber );
xPart -> NewChild2 ( "ETag" , etag );
delete xPart ;
partsListXml . SaveXml ( partsListFilePath . c_str ());
}
// ---- STEP 3: COMPLETE ----
conn -> rest . ClearAllQueryParams ();
conn -> rest . AddQueryParam ( "uploadId" , uploadId );
responseXml = conn -> rest . fullRequestString ( "POST" , objectPath . c_str (), partsListXml . getXml ());
if ( ! conn -> rest . get_LastMethodSuccess ()) {
2026-08-12 21:24:12 +10:00
{ AttemptResult r = ClassifyFailedRequest ( std :: move ( conn ), lastError );
lastError = "Complete: " + lastError ; return r ; }
2026-04-17 07:03:03 +10:00
}
int completeStatus = conn -> rest . get_ResponseStatusCode ();
if ( completeStatus != 200 ) {
lastError = "Complete HTTP " + std :: to_string ( completeStatus );
if ( responseXml != nullptr && responseXml [ 0 ] != '\0' ) lastError += " - " + std :: string ( responseXml );
ReleaseConnection ( std :: move ( conn ));
bool retryable = ( completeStatus >= 500 && completeStatus <= 599 ) || completeStatus == 408 || completeStatus == 429 ;
return retryable ? AttemptResult :: Transient : AttemptResult :: Permanent ;
}
if ( fac . FileExists ( partsListFilePath . c_str ())) {
fac . FileDelete ( partsListFilePath . c_str ());
}
// Build canonical path-style URL from parts — independent
// of the addressing style we actually used.
std :: string scheme = _bTls ? "https://" : "http://" ;
2026-05-01 11:16:47 +10:00
uploadedFilePath = scheme + ( _bAwsPath ? _fullAWSURL : _baseDomain ) + "/" + bucketName + "/" + objectKey ;
2026-04-17 07:03:03 +10:00
_logger . LogDebug ( kOp ,
"Multipart upload completed. S3 key: " + objectKey + " | URL: " + uploadedFilePath ,
__FILE__ , __LINE__ );
ReleaseConnection ( std :: move ( conn ));
return AttemptResult :: Success ;
});
2026-03-28 16:54:11 +11:00
}
// Upload jpeg data
bool ANSAWSS3 :: UploadJpegImage ( const std :: string & bucketName , unsigned char * jpeg_string , int32 bufferLength , const std :: string & fileName , std :: string & uploadedFilePath ) {
2026-04-17 07:03:03 +10:00
const std :: string kOp = "ANSAWSS3::UploadJpegImage" ;
2026-04-17 20:59:21 +10:00
// _logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__);
2026-04-17 07:03:03 +10:00
// Early validation checks (permanent — do NOT retry)
2026-08-12 21:24:12 +10:00
if ( ! CheckUploadPreconditions ( kOp )) return false ;
2026-03-28 16:54:11 +11:00
if ( bucketName . empty () || bufferLength <= 0 ) {
2026-04-17 07:03:03 +10:00
_logger . LogError ( kOp ,
2026-03-28 16:54:11 +11:00
"Bucket name or buffer length is invalid" ,
__FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
SetCallStatus ( kCallPermanentError ); // caller bug — retrying cannot help
2026-03-28 16:54:11 +11:00
return false ;
}
2026-04-17 07:03:03 +10:00
// Object path is attempt-independent.
const std :: string objectPath =
_bAwsPath ? ( "/" + fileName ) : ( "/" + bucketName + "/" + fileName );
2026-03-28 16:54:11 +11:00
2026-04-17 07:03:03 +10:00
return UploadWithRetry ( kOp ,
[ & ]( std :: string & lastError ) -> AttemptResult {
auto conn = AcquireConnection ();
if ( ! conn ) {
lastError = "Failed to acquire S3 connection" ;
return AttemptResult :: Transient ;
2026-03-28 16:54:11 +11:00
}
2026-04-17 07:03:03 +10:00
if ( _bAwsPath ) {
conn -> rest . put_Host (( bucketName + "." + _fullAWSURL ). c_str ());
} else {
conn -> rest . put_Host ( _fullAWSURL . c_str ());
}
CkBinData binData ;
CkByteData jpegBytes ;
jpegBytes . append2 ( jpeg_string , static_cast < unsigned long > ( bufferLength ));
binData . AppendBinary ( jpegBytes );
conn -> rest . AddHeader ( "Content-Type" , "image/jpeg" );
conn -> rest . AddHeader ( "Expect" , "100-continue" );
CkStringBuilder sbResponse ;
if ( ! conn -> rest . FullRequestBd ( "PUT" , objectPath . c_str (), binData , sbResponse )) {
2026-08-12 21:24:12 +10:00
return ClassifyFailedRequest ( std :: move ( conn ), lastError );
2026-04-17 07:03:03 +10:00
}
int statusCode = conn -> rest . get_ResponseStatusCode ();
if ( statusCode == 200 ) {
std :: string scheme = _bTls ? "https://" : "http://" ;
2026-05-01 11:16:47 +10:00
uploadedFilePath = scheme + ( _bAwsPath ? _fullAWSURL : _baseDomain ) + "/" + bucketName + "/" + fileName ;
2026-04-17 07:03:03 +10:00
_logger . LogDebug ( kOp ,
"Successfully uploaded: " + fileName + " | URL: " + uploadedFilePath ,
__FILE__ , __LINE__ );
ReleaseConnection ( std :: move ( conn ));
return AttemptResult :: Success ;
}
lastError = "HTTP " + std :: to_string ( statusCode );
std :: string body = sbResponse . getAsString ();
if ( ! body . empty ()) lastError += " - " + body ;
2026-03-28 16:54:11 +11:00
ReleaseConnection ( std :: move ( conn ));
2026-04-17 07:03:03 +10:00
bool retryable = ( statusCode >= 500 && statusCode <= 599 )
|| statusCode == 408 || statusCode == 429 ;
return retryable ? AttemptResult :: Transient : AttemptResult :: Permanent ;
});
2026-03-28 16:54:11 +11:00
}
bool ANSAWSS3 :: UploadPrefixJpegImage ( const std :: string & bucketName , const std :: string & prefix , unsigned char * jpeg_string , int32 bufferLength , const std :: string & fileName , std :: string & uploadedFilePath ) {
2026-04-17 07:03:03 +10:00
const std :: string kOp = "ANSAWSS3::UploadPrefixJpegImage" ;
2026-04-17 20:59:21 +10:00
//_logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__);
2026-04-17 07:03:03 +10:00
// Early validation checks (permanent failures — do NOT retry)
2026-08-12 21:24:12 +10:00
if ( ! CheckUploadPreconditions ( kOp )) return false ;
2026-03-28 16:54:11 +11:00
if ( bucketName . empty () || bufferLength <= 0 || fileName . empty ()) {
2026-04-17 07:03:03 +10:00
_logger . LogError ( kOp ,
2026-03-28 16:54:11 +11:00
"Bucket name, buffer length, or filename is invalid" ,
__FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
SetCallStatus ( kCallPermanentError ); // caller bug — retrying cannot help
2026-03-28 16:54:11 +11:00
return false ;
}
2026-04-17 07:03:03 +10:00
// Build once: the object key (path within the bucket) and the HTTP
// request path (which depends on addressing style). Keeping these
// separate avoids the path-style bucket-duplication bug in the
// returned URL.
std :: string objectKey ; // "<prefix>/<fileName>" — no leading '/'
if ( ! prefix . empty ()) {
std :: string normalizedPrefix = prefix ;
if ( ! normalizedPrefix . empty () && normalizedPrefix . front () == '/' ) {
normalizedPrefix = normalizedPrefix . substr ( 1 );
2026-03-28 16:54:11 +11:00
}
2026-04-17 07:03:03 +10:00
if ( ! normalizedPrefix . empty () && normalizedPrefix . back () != '/' ) {
normalizedPrefix += '/' ;
2026-03-28 16:54:11 +11:00
}
2026-04-17 07:03:03 +10:00
objectKey = normalizedPrefix ;
2026-03-28 16:54:11 +11:00
}
2026-04-17 07:03:03 +10:00
objectKey += fileName ;
const std :: string objectPath = _bAwsPath
? ( "/" + objectKey )
: ( "/" + bucketName + "/" + objectKey );
return UploadWithRetry ( kOp ,
[ & ]( std :: string & lastError ) -> AttemptResult {
auto conn = AcquireConnection ();
if ( ! conn ) {
lastError = "Failed to acquire S3 connection" ;
return AttemptResult :: Transient ;
}
if ( _bAwsPath ) {
conn -> rest . put_Host (( bucketName + "." + _fullAWSURL ). c_str ());
} else {
conn -> rest . put_Host ( _fullAWSURL . c_str ());
}
CkBinData binData ;
CkByteData jpegBytes ;
jpegBytes . append2 ( jpeg_string , static_cast < unsigned long > ( bufferLength ));
binData . AppendBinary ( jpegBytes );
conn -> rest . AddHeader ( "Content-Type" , "image/jpeg" );
conn -> rest . AddHeader ( "Expect" , "100-continue" );
CkStringBuilder sbResponse ;
if ( ! conn -> rest . FullRequestBd ( "PUT" , objectPath . c_str (), binData , sbResponse )) {
2026-08-12 21:24:12 +10:00
return ClassifyFailedRequest ( std :: move ( conn ), lastError );
2026-04-17 07:03:03 +10:00
}
int statusCode = conn -> rest . get_ResponseStatusCode ();
if ( statusCode == 200 ) {
// Build the returned URL from parts (independent of
// addressing style) — always the canonical path-style URL.
std :: string scheme = _bTls ? "https://" : "http://" ;
2026-05-01 11:16:47 +10:00
uploadedFilePath = scheme + ( _bAwsPath ? _fullAWSURL : _baseDomain ) + "/" + bucketName + "/" + objectKey ;
2026-04-19 23:15:50 +10:00
_logger . LogDebug ( kOp ,
2026-04-17 07:03:03 +10:00
"Successfully uploaded: " + objectKey + " (" + std :: to_string ( bufferLength ) + " bytes) | URL: " + uploadedFilePath ,
__FILE__ , __LINE__ );
ReleaseConnection ( std :: move ( conn ));
return AttemptResult :: Success ;
}
lastError = "HTTP " + std :: to_string ( statusCode );
std :: string body = sbResponse . getAsString ();
if ( ! body . empty ()) lastError += " - " + body ;
ReleaseConnection ( std :: move ( conn ));
bool retryable = ( statusCode >= 500 && statusCode <= 599 )
|| statusCode == 408 || statusCode == 429 ;
return retryable ? AttemptResult :: Transient : AttemptResult :: Permanent ;
});
2026-03-28 16:54:11 +11:00
}
2026-05-19 10:37:46 +10:00
// Generates a temporary pre-signed URL (AWS Signature V4) for an object
// already stored in S3. Public users can open the URL — without any AWS
// credentials — until it expires. SigV4 pre-signing is a purely local
// operation: it needs valid credentials and a known endpoint, but performs
// no network I/O, so it succeeds even when not connected.
bool ANSAWSS3 :: GeneratePresignedURL ( const std :: string & bucketName ,
const std :: string & prefix ,
const std :: string & objectName ,
int validDays ,
std :: string & presignedURL ) {
const std :: string kOp = "ANSAWSS3::GeneratePresignedURL" ;
// Permanent failures — do NOT proceed.
if ( ! _isLicenseValid || ! _isUnlockCodeValid ) {
_logger . LogError ( kOp ,
! _isLicenseValid ? "Invalid license" : "Invalid unlock code" ,
__FILE__ , __LINE__ );
ANS_DBG ( "AWSS3" , "GeneratePresignedURL: invalid license/unlock code" );
return false ;
}
if ( bucketName . empty () || objectName . empty ()) {
_logger . LogError ( kOp , "Bucket name or object name is empty" , __FILE__ , __LINE__ );
ANS_DBG ( "AWSS3" , "GeneratePresignedURL: empty bucket/object name" );
return false ;
}
// AWS SigV4 caps pre-signed URL validity at 7 days.
if ( validDays < 1 || validDays > 7 ) {
_logger . LogError ( kOp ,
"validDays must be between 1 and 7 (got " + std :: to_string ( validDays ) + ")" ,
__FILE__ , __LINE__ );
ANS_DBG ( "AWSS3" , "GeneratePresignedURL: validDays out of range (%d)" , validDays );
return false ;
}
// Snapshot the config/auth attributes under the config mutex.
std :: string accessKey , secretKey , region , endpoint , service ;
bool useHttps ;
{
std :: lock_guard < std :: mutex > lock ( _configMutex );
accessKey = _accessKey ;
secretKey = _secretKey ;
region = _bucketRegion ;
endpoint = _fullAWSURL ;
service = _serviceName ;
useHttps = _bTls ;
}
if ( accessKey . empty () || secretKey . empty ()) {
_logger . LogError ( kOp ,
"AWS credentials not set (call SetAuthentication first)" ,
__FILE__ , __LINE__ );
ANS_DBG ( "AWSS3" , "GeneratePresignedURL: missing credentials" );
return false ;
}
if ( endpoint . empty () || region . empty () || service . empty ()) {
_logger . LogError ( kOp ,
"S3 endpoint/region/service not configured (call Connect first)" ,
__FILE__ , __LINE__ );
ANS_DBG ( "AWSS3" , "GeneratePresignedURL: endpoint/region/service not configured" );
return false ;
}
// Build the object key the same way UploadPrefixJpegImage does, so the
// URL is generated for exactly the key that was uploaded.
std :: string objectKey ;
if ( ! prefix . empty ()) {
std :: string normalizedPrefix = prefix ;
if ( normalizedPrefix . front () == '/' ) {
normalizedPrefix = normalizedPrefix . substr ( 1 );
}
if ( ! normalizedPrefix . empty () && normalizedPrefix . back () != '/' ) {
normalizedPrefix += '/' ;
}
objectKey = normalizedPrefix ;
}
objectKey += objectName ;
// Configure a CkHttp with the same credentials/endpoint the upload
// path uses so the signed host matches the stored object's host.
CkHttp http ;
http . put_AwsAccessKey ( accessKey . c_str ());
http . put_AwsSecretKey ( secretKey . c_str ());
http . put_AwsRegion ( region . c_str ());
http . put_AwsEndpoint ( endpoint . c_str ());
const int numSecondsValid = validDays * 86400 ;
const char * url = http . s3_GenPresignedUrl ( "GET" , useHttps ,
bucketName . c_str (), objectKey . c_str (), numSecondsValid , service . c_str ());
if ( ! http . get_LastMethodSuccess ()) {
_logger . LogError ( kOp ,
std :: string ( "Pre-signed URL generation failed: " ) + http . lastErrorText (),
__FILE__ , __LINE__ );
ANS_DBG ( "AWSS3" , "GeneratePresignedURL FAILED: key=%s err=%s" ,
objectKey . c_str (), http . lastErrorText ());
return false ;
}
presignedURL = ( url != nullptr ) ? url : "" ;
_logger . LogDebug ( kOp ,
"Generated pre-signed URL for '" + objectKey + "' valid " +
std :: to_string ( validDays ) + " day(s): " + presignedURL ,
__FILE__ , __LINE__ );
ANS_DBG ( "AWSS3" , "GeneratePresignedURL OK: key=%s validDays=%d url=%s" ,
objectKey . c_str (), validDays , presignedURL . c_str ());
return true ;
}
2026-03-28 16:54:11 +11:00
// Downloads
bool ANSAWSS3 :: DownloadFile ( const std :: string & bucketName ,
const std :: string & objectName ,
const std :: string & saveFilePath ) {
// Early validation checks
if ( ! _isLicenseValid || ! _isUnlockCodeValid || ! _bConnected ) {
_logger . LogError ( "ANSAWSS3::DownloadFile" ,
! _isLicenseValid ? "Invalid license" : ! _isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress . load () ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)" ,
__FILE__ , __LINE__ );
return false ;
}
if ( bucketName . empty () || objectName . empty () || saveFilePath . empty ()) {
_logger . LogError ( "ANSAWSS3::DownloadFile" ,
"Bucket name, object name, or save path is empty" ,
__FILE__ , __LINE__ );
return false ;
}
bool downloadSuccess = false ;
try {
auto conn = AcquireConnection ();
if ( ! conn ) {
_logger . LogError ( "DownloadFile" , "Failed to acquire S3 connection" , __FILE__ , __LINE__ );
return false ;
}
// Set bucket-specific endpoint
if ( _bAwsPath ) {
conn -> rest . put_Host (( bucketName + "." + _fullAWSURL ). c_str ());
} else {
conn -> rest . put_Host ( _fullAWSURL . c_str ());
}
// Normalize object name (remove leading slash if present)
std :: string normalizedObjectName = objectName ;
if ( ! normalizedObjectName . empty () && normalizedObjectName [ 0 ] == '/' ) {
normalizedObjectName = normalizedObjectName . substr ( 1 );
}
std :: string objectPath = _bAwsPath ? ( "/" + normalizedObjectName ) : ( "/" + bucketName + "/" + normalizedObjectName );
_logger . LogDebug ( "ANSAWSS3::DownloadFile" ,
"Downloading: " + normalizedObjectName + " from bucket: " + bucketName ,
__FILE__ , __LINE__ );
// Send GET request
if ( ! conn -> rest . SendReqNoBody ( "GET" , objectPath . c_str ())) {
_logger . LogError ( "ANSAWSS3::DownloadFile - Request failed" ,
conn -> rest . lastErrorText (), __FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
DiscardConnection ( std :: move ( conn )); // transport failed — do not pool a suspect socket
2026-03-28 16:54:11 +11:00
return false ;
}
// Read the response header
int responseStatusCode = conn -> rest . ReadResponseHeader ();
if ( responseStatusCode < 0 ) {
_logger . LogError ( "ANSAWSS3::DownloadFile - Failed to read response header" ,
conn -> rest . lastErrorText (), __FILE__ , __LINE__ );
2026-08-12 21:24:12 +10:00
DiscardConnection ( std :: move ( conn )); // transport failed — do not pool a suspect socket
2026-03-28 16:54:11 +11:00
return false ;
}
// We expect a 200 response status if the data is coming
if ( responseStatusCode == 200 ) {
// Determine the final file path
std :: string finalFilePath ;
// Check if saveFilePath is a directory or a file
CkFileAccess fac ;
bool isDirectory = false ;
// If path exists and is a directory, or ends with path separator
if ( fac . DirExists ( saveFilePath . c_str ())) {
isDirectory = true ;
}
else if ( ! saveFilePath . empty () &&
( saveFilePath . back () == '\\' || saveFilePath . back () == '/' )) {
isDirectory = true ;
}
if ( isDirectory ) {
// Save path is a directory - append object name
finalFilePath = saveFilePath ;
// Ensure path ends with separator
if ( ! finalFilePath . empty () &&
finalFilePath . back () != '\\' &&
finalFilePath . back () != '/' ) {
finalFilePath += " \\ " ;
}
// Extract just the filename from object name (in case it has path separators)
std :: string filename = normalizedObjectName ;
size_t lastSlash = filename . find_last_of ( "/ \\ " );
if ( lastSlash != std :: string :: npos ) {
filename = filename . substr ( lastSlash + 1 );
}
finalFilePath += filename ;
}
else {
// Save path is the complete file path
finalFilePath = saveFilePath ;
}
// Ensure the directory exists
size_t lastSeparator = finalFilePath . find_last_of ( "/ \\ " );
if ( lastSeparator != std :: string :: npos ) {
std :: string directory = finalFilePath . substr ( 0 , lastSeparator );
// Create directory if it doesn't exist
if ( ! fac . DirExists ( directory . c_str ())) {
if ( ! fac . DirCreate ( directory . c_str ())) {
_logger . LogError ( "ANSAWSS3::DownloadFile - Failed to create directory" ,
"Directory: " + directory + " - " + fac . lastErrorText (),
__FILE__ , __LINE__ );
ReleaseConnection ( std :: move ( conn ));
return false ;
}
_logger . LogDebug ( "ANSAWSS3::DownloadFile" ,
"Created directory: " + directory ,
__FILE__ , __LINE__ );
}
}
// Setup stream to write to file
CkStream bodyStream ;
bodyStream . put_SinkFile ( finalFilePath . c_str ());
// Read the response body to the stream
if ( ! conn -> rest . ReadRespBodyStream ( bodyStream , true )) {
_logger . LogError ( "ANSAWSS3::DownloadFile - Failed to read response body" ,
conn -> rest . lastErrorText (), __FILE__ , __LINE__ );
}
else {
_logger . LogDebug ( "ANSAWSS3::DownloadFile" ,
"Successfully downloaded: " + normalizedObjectName + " to: " + finalFilePath ,
__FILE__ , __LINE__ );
downloadSuccess = true ;
}
}
else {
// Handle non-200 response
const char * errResponse = conn -> rest . readRespBodyString ();
std :: string errorMsg = "HTTP " + std :: to_string ( responseStatusCode );
if ( ! conn -> rest . get_LastMethodSuccess ()) {
errorMsg += " - Failed to read error response: " + std :: string ( conn -> rest . lastErrorText ());
}
else {
if ( errResponse != nullptr && errResponse [ 0 ] != '\0' ) {
errorMsg += " - " + std :: string ( errResponse );
}
}
_logger . LogError ( "ANSAWSS3::DownloadFile" , errorMsg , __FILE__ , __LINE__ );
}
ReleaseConnection ( std :: move ( conn ));
}
catch ( const std :: exception & e ) {
_logger . LogFatal ( "ANSAWSS3::DownloadFile" ,
std :: string ( "Exception: " ) + e . what (),
__FILE__ , __LINE__ );
}
return downloadSuccess ;
}
// Delete
bool ANSAWSS3 :: DeleteBucketObject ( const std :: string & bucketName , const std :: string & objectName ) {
// Early validation checks
if ( ! _isLicenseValid || ! _isUnlockCodeValid || ! _bConnected ) {
_logger . LogError ( "ANSAWSS3::DeleteBucketObject" ,
! _isLicenseValid ? "Invalid license" : ! _isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress . load () ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)" ,
__FILE__ , __LINE__ );
return false ;
}
if ( bucketName . empty () || objectName . empty ()) {
_logger . LogError ( "ANSAWSS3::DeleteBucketObject" ,
"Bucket name or object name is empty" ,
__FILE__ , __LINE__ );
return false ;
}
bool deleteSuccess = false ;
try {
auto conn = AcquireConnection ();
if ( ! conn ) {
_logger . LogError ( "DeleteBucketObject" , "Failed to acquire S3 connection" , __FILE__ , __LINE__ );
return false ;
}
// Set bucket-specific endpoint
if ( _bAwsPath ) {
conn -> rest . put_Host (( bucketName + "." + _fullAWSURL ). c_str ());
} else {
conn -> rest . put_Host ( _fullAWSURL . c_str ());
}
std :: string objectPath = _bAwsPath ? ( "/" + objectName ) : ( "/" + bucketName + "/" + objectName );
CkStringBuilder sbResponse ;
bool success = conn -> rest . FullRequestNoBodySb ( "DELETE" , objectPath . c_str (), sbResponse );
if ( ! success ) {
_logger . LogError ( "ANSAWSS3::DeleteBucketObject" ,
"Failed to send DELETE request: " + std :: string ( conn -> rest . lastErrorText ()),
__FILE__ , __LINE__ );
}
else {
int statusCode = conn -> rest . get_ResponseStatusCode ();
// S3 returns 204 (No Content) for successful deletion
// Also accept 200 as success
if ( statusCode == 204 || statusCode == 200 ) {
_logger . LogDebug ( "ANSAWSS3::DeleteBucketObject" ,
"Successfully deleted object: " + objectName + " from bucket: " + bucketName ,
__FILE__ , __LINE__ );
deleteSuccess = true ;
}
else if ( statusCode == 404 ) {
// Object doesn't exist - you may want to treat this as success or failure
// depending on your use case
// Uncomment the next line if you want to treat "not found" as success:
deleteSuccess = true ;
}
else {
std :: string errorMsg = "HTTP " + std :: to_string ( statusCode );
std :: string response = sbResponse . getAsString ();
if ( ! response . empty ()) {
errorMsg += " - " + response ;
}
_logger . LogError ( "ANSAWSS3::DeleteBucketObject" , errorMsg , __FILE__ , __LINE__ );
}
}
ReleaseConnection ( std :: move ( conn ));
}
catch ( const std :: exception & e ) {
_logger . LogFatal ( "ANSAWSS3::DeleteBucketObject" ,
std :: string ( "Exception: " ) + e . what (),
__FILE__ , __LINE__ );
}
return deleteSuccess ;
}
}