Files
ANSCORE/modules/ANSUtilities/ANSAWSS3.cpp
T
nghianguyenandClaude Opus 5 cb964907f0 feat(ANSLS): know ANSMIND as 1009, and stop filing unknown products under it
The gateway's product table already runs to 1009 ANSMIND and 1010 ANSGW.
The licensing library stopped at 1008, so a real ANSMIND licence could not
be named, deactivated or listed here, while every product name it did not
recognise quietly became product 1009. With 1009 now a real product, that
fallback would mint an ANSMIND trial for a typo.

ANSPRODUCTS gains ANSMIND = 1009 and ANSGW = 1010, and both are added
wherever the product table is spelled out: ActivateTrialLicense,
DeactivateLicense, ActivateLicense, ActivateLicenseWithCustomHWID and
ListLicenses, which now reports an ANSMIND and an ANSGW row like the
gateway does.

Unknown names no longer map to any id. ActivateTrialLicense returns 0, its
existing failure value, instead of generating a key. ListLicenses skips
the row. ActivateLicense and ActivateLicenseWithCustomHWID keep passing
the name through as the registration name: they never assigned an id, the
key's own validation data rejects a name it was not issued for, and the
gateway resolves an unknown product the same way.

The ANSVIS master-licence rule in LicenseVerification is untouched.

Separately, the fallback loops in ANSUtilities, ANSAWSS3 and ANSLLM asked
for product 1008 under the registration name "ANSSRT". The product is
ANSTS, so a genuine ANSTS licence could never satisfy them. Only that
string changes; the loops still stop at 1008.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-13 10:38:31 +10:00

3179 lines
142 KiB
C++

#include "ANSUtilities.h"
#include <iostream>
#include <CkFileAccess.h>
#include <CkAuthGoogle.h>
#include <CkSocket.h>
#include <CkGlobal.h>
#include <CkHttp.h>
#include <CkMailMan.h>
#include <CkEmail.h>
#include <CkMht.h>
#include <vector>
#include <CkBinData.h>
#include <CkStringBuilder.h>
#include <CkStream.h>
#include <fstream>
#include <chrono>
#include <thread>
#include <sstream>
#include <atomic>
static bool ansawss3LicenceValid = false;
static std::once_flag ansawss3LicenseOnceFlag;
namespace ANSCENTER
{
static void VerifyGlobalANSAWSS3License(const std::string& licenseKey) {
try {
static const std::vector<std::pair<int, std::string>> licenseChecks = {
{1000, "ANNHUB-LV"},
{1001, "DLHUB-LV"},
{1002, "ODHUB-LV"},
{1003, "ANSVIS"},
{1004, "ANSFR"},
{1005, "ANSOCR"},
{1006, "ANSALPR"},
{1007, "ANSCV"},
{1008, "ANSTS"}
};
ansawss3LicenceValid = false;
for (const auto& [productId, productName] : licenseChecks) {
if (ANSCENTER::ANSLicenseHelper::LicenseVerification(licenseKey, productId, productName)) {
ansawss3LicenceValid = true;
break; // Stop at the first valid license
}
}
}
catch (const std::exception& e) {
ansawss3LicenceValid = false;
}
}
// Builds a per-call unique suffix for scratch file names used by
// multipart uploads. Combining thread id, instance address, and a
// monotonically-increasing counter guarantees no collision between
// parallel uploads — including two threads uploading the same source
// file at the same time (Fix 2).
static std::string MakeUniqueMultipartToken(const void* instancePtr) {
static std::atomic<uint64_t> counter{0};
std::ostringstream oss;
oss << std::hex
<< std::hash<std::thread::id>{}(std::this_thread::get_id()) << "_"
<< reinterpret_cast<uintptr_t>(instancePtr) << "_"
<< counter.fetch_add(1, std::memory_order_relaxed);
return oss.str();
}
// ────────────────────────────────────────────────────────────────────────
// SummarizeChilkatError: turn Chilkat's verbose multi-line lastErrorText()
// into a single diagnostic line suitable for _logger.LogError().
//
// Chilkat's log looks like:
// ChilkatLog: Connect: DllDate: ... ChilkatVersion: ... UnlockStatus: ...
// restConnect: domain_or_ip: s3.us-east-1.amazonaws.com
// socket2Connect: connect2: connectImplicitSsl: connectSocket_v2:
// connect_domain: ckDnsResolveDomainIPv4_n: ...
// clientHandshake2: readHandshakeMessages:
// Failed to read beginning of SSL/TLS record.
// readTlsRecord: Socket operation timeout.
// See https://cknotes.com/...
// ConnectFailReason: 103
// Failed.
//
// Diagnostics we care about:
// • the deepest concrete failure reason (the non-label leaf line)
// • ConnectFailReason: <N> if present
// • the target (domain_or_ip / ip_or_domain + port)
// Rest (DllDate, ChilkatVersion, Architecture, VerboseLogging, ...) is noise.
//
// Returns e.g.:
// "Connect failed [reason=103] s3.us-east-1.amazonaws.com:443 — Socket operation timeout (TLS handshake). See https://cknotes.com/failed-to-read-beginning-of-ssl-tls-record-..."
// ────────────────────────────────────────────────────────────────────────
static std::string SummarizeChilkatError(const char* raw) {
if (!raw || !*raw) return "(empty Chilkat log)";
std::string s(raw);
// Split into trimmed lines.
std::vector<std::string> lines;
{
std::istringstream iss(s);
std::string ln;
while (std::getline(iss, ln)) {
// Trim leading whitespace (Chilkat indents by level).
size_t a = ln.find_first_not_of(" \t\r\n");
if (a == std::string::npos) continue;
size_t b = ln.find_last_not_of(" \t\r\n");
lines.push_back(ln.substr(a, b - a + 1));
}
}
if (lines.empty()) return "(empty Chilkat log)";
// Lines that are pure labels ("foo:" or "--foo") are scaffolding.
auto isLabelLine = [](const std::string& ln) {
if (ln.empty()) return true;
if (ln.rfind("--", 0) == 0) return true; // "--foo" close-marker
// "label: value" — keep only if value is non-empty and line doesn't
// look like pure call-stack breadcrumb (trailing ':').
if (!ln.empty() && ln.back() == ':') return true;
return false;
};
// Fields we want to extract.
std::string target, failReason, tlsHint, leafError, seeUrl;
for (auto& ln : lines) {
if (ln.rfind("domain_or_ip:", 0) == 0 || ln.rfind("ip_or_domain:", 0) == 0) {
auto p = ln.find(':');
if (p != std::string::npos) target = ln.substr(p + 1);
} else if (ln.rfind("ConnectFailReason:", 0) == 0) {
auto p = ln.find(':');
if (p != std::string::npos) failReason = ln.substr(p + 1);
} else if (ln.rfind("port:", 0) == 0) {
auto p = ln.find(':');
if (p != std::string::npos) {
// Append port to target if we have one.
std::string port = ln.substr(p + 1);
// Trim leading space.
size_t a = port.find_first_not_of(" \t");
if (a != std::string::npos) port = port.substr(a);
if (!target.empty()) target += ":" + port;
}
} else if (ln.rfind("See http", 0) == 0 || ln.rfind("See https", 0) == 0) {
seeUrl = ln.substr(4); // drop "See "
} else if (ln.rfind("Failed to read beginning of SSL/TLS record", 0) == 0) {
tlsHint = "TLS handshake — server sent nothing (firewall/AV TLS inspection, proxy, or packet loss)";
} else if (!isLabelLine(ln)) {
// Candidate concrete message. Keep the LATEST one that's not a
// stack label; Chilkat's deepest concrete error is usually the
// most specific explanation.
leafError = ln;
}
}
// Trim leading space on captured fields.
auto trim = [](std::string& v) {
size_t a = v.find_first_not_of(" \t");
if (a == std::string::npos) { v.clear(); return; }
size_t b = v.find_last_not_of(" \t");
v = v.substr(a, b - a + 1);
};
trim(target); trim(failReason); trim(leafError); trim(seeUrl);
// Compose.
std::ostringstream out;
out << "Chilkat Connect failed";
if (!failReason.empty()) out << " [reason=" << failReason << "]";
if (!target.empty()) out << " " << target;
if (!leafError.empty()) out << " — " << leafError;
if (!tlsHint.empty()) out << " (" << tlsHint << ")";
if (!seeUrl.empty()) out << " (" << seeUrl << ")";
return out.str();
}
// ── Per-thread call status ───────────────────────────────────────────────
// Written by every upload entry point, read by the C exports in dllmain so
// the caller learns WHY a call failed (and therefore whether to retry)
// without any change to the exported signatures. Thread-local because a
// single ANSAWSS3 handle is routinely shared by many worker threads — a
// per-instance field would report another worker's outcome.
static thread_local int t_callStatus = ANSAWSS3::kCallPermanentError;
void ANSAWSS3::SetCallStatus(int status) { t_callStatus = status; }
int ANSAWSS3::LastCallStatus() { return t_callStatus; }
// ────────────────────────────────────────────────────────────────────────
// SummarizeChilkatRequestError: one line out of a request-level Chilkat log.
//
// The raw log is ~2 KB of call-stack scaffolding per failure. Written once
// per retry, per worker, it buries the one thing an operator needs: what S3
// actually said. S3 returns its reason as XML in the response body, which
// Chilkat echoes into the log:
// <Error><Code>NoSuchBucket</Code><Message>The specified bucket does
// not exist</Message>...
// Pull that out, prefix the status code, and drop the rest.
// ────────────────────────────────────────────────────────────────────────
static std::string SummarizeChilkatRequestError(const char* raw, int statusCode) {
std::ostringstream out;
if (statusCode > 0) out << "HTTP " << statusCode;
auto tagValue = [](const std::string& s, const char* tag) -> std::string {
const std::string open = std::string("<") + tag + ">";
const std::string close = std::string("</") + tag + ">";
const size_t a = s.find(open);
if (a == std::string::npos) return {};
const size_t b = s.find(close, a + open.size());
if (b == std::string::npos) return {};
return s.substr(a + open.size(), b - a - open.size());
};
if (raw && *raw) {
const std::string s(raw);
const std::string code = tagValue(s, "Code");
const std::string msg = tagValue(s, "Message");
if (!code.empty()) {
if (statusCode > 0) out << " ";
out << code;
if (!msg.empty()) out << ": " << msg;
}
else {
// No S3 error document — fall back to the deepest concrete line
// (same idea as SummarizeChilkatError, without the connect
// specific fields).
std::istringstream iss(s);
std::string ln, leaf;
while (std::getline(iss, ln)) {
const size_t a = ln.find_first_not_of(" \t\r\n");
if (a == std::string::npos) continue;
const size_t b = ln.find_last_not_of(" \t\r\n");
ln = ln.substr(a, b - a + 1);
if (ln.rfind("--", 0) == 0) continue; // close marker
if (!ln.empty() && ln.back() == ':') continue; // stack label
if (ln == "Failed.") continue;
leaf = ln;
}
if (!leaf.empty()) {
if (statusCode > 0) out << " — ";
out << leaf;
}
}
}
std::string result = out.str();
return result.empty() ? "(no detail in Chilkat log)" : result;
}
// Private helper function to extract file name from a path
// Helper function to extract filename from path
std::string ANSAWSS3::ExtractFileName(const std::string& filePath) {
size_t pos = filePath.find_last_of("/\\");
if (pos != std::string::npos) {
return filePath.substr(pos + 1);
}
return filePath;
}
// Helper function to determine content type from file extension
std::string ANSAWSS3::GetContentType(const std::string& filePath) {
// Default to application/octet-stream for unknown types
std::string contentType = "application/octet-stream";
size_t extPos = filePath.find_last_of('.');
if (extPos == std::string::npos) {
return contentType;
}
// Extract and convert extension to lowercase
std::string fileExt = filePath.substr(extPos + 1);
std::transform(fileExt.begin(), fileExt.end(), fileExt.begin(), ::tolower);
// Image types
if (fileExt == "jpg" || fileExt == "jpeg") {
contentType = "image/jpeg";
}
else if (fileExt == "png") {
contentType = "image/png";
}
else if (fileExt == "gif") {
contentType = "image/gif";
}
else if (fileExt == "bmp") {
contentType = "image/bmp";
}
else if (fileExt == "webp") {
contentType = "image/webp";
}
else if (fileExt == "svg") {
contentType = "image/svg+xml";
}
else if (fileExt == "ico") {
contentType = "image/x-icon";
}
// Video types
else if (fileExt == "mp4") {
contentType = "video/mp4";
}
else if (fileExt == "avi") {
contentType = "video/x-msvideo";
}
else if (fileExt == "mov") {
contentType = "video/quicktime";
}
// Document types
else if (fileExt == "pdf") {
contentType = "application/pdf";
}
else if (fileExt == "json") {
contentType = "application/json";
}
else if (fileExt == "xml") {
contentType = "application/xml";
}
else if (fileExt == "txt") {
contentType = "text/plain";
}
return contentType;
}
ANSAWSS3::ANSAWSS3() {
_unlockCode = "ANSDRC.CB1122026_MEQCIFwO1IFQCG0BhZwsXFO68QUU6mDB5uge4duOsqOJanEyAiAB67ahqnXin4SRy0vIegISgbFlpldmbuS5gbU21GYVqA==";// "ANSDRC.CB1082025_Ax6P3M7F8B3d";//
_proxyHost = "";
_proxyPort = 0;
_proxyUsername = "";
_proxyPassword = "";
_bProxy = false;
_serviceName = "s3";
}
ANSAWSS3::~ANSAWSS3() {
StopRetry();
std::lock_guard<std::mutex> lk(_poolMutex);
_pool.clear(); // CkRest destructors handle disconnect
}
void ANSAWSS3::StopRetry() {
_stopRetry = true;
if (_retryThread.joinable()) {
_retryThread.join();
}
_retryInProgress = false;
}
// TryConnect — actual connection attempt (caller must hold _configMutex)
//
// First attempts the AWS virtual-host name (s3.<region>.<base>); for
// S3-compatible servers like MinIO/Ceph that hostname does not exist, so
// we fall back to the raw _baseDomain (path-style). The first attempt is
// therefore *expected* to fail on MinIO — we suppress its error-level log
// (logOnFailure=false) so the operator only sees errors when the fallback
// also fails.
bool ANSAWSS3::TryConnect(bool& awsPath) {
const bool fallbackPossible = (_fullAWSURL != _baseDomain);
auto testConn = CreateConnection(/*logOnFailure*/ !fallbackPossible);
if (!testConn) {
// Fallback: connect directly to baseDomain (MinIO, Ceph, etc.)
std::string savedFullURL = _fullAWSURL;
_fullAWSURL = _baseDomain;
testConn = CreateConnection(/*logOnFailure*/ true);
if (!testConn) {
_fullAWSURL = savedFullURL; // restore on failure
return false;
}
awsPath = false;
_bAwsPath = false;
}
else {
awsPath = true;
_bAwsPath = true;
}
_bConnected = true;
// Return test connection to pool for reuse
{
std::lock_guard<std::mutex> lk(_poolMutex);
_pool.push_back(std::move(testConn));
}
return true;
}
// Background connect loop, started by Connect() when the endpoint could not
// be reached at all (return code 2).
//
// This loop NEVER gives up on its own. The caller — a LabVIEW VI, an
// unattended AIBOX — calls Connect() exactly once at startup, so if this
// thread returned while still unconnected the handle would stay dead for
// the life of the process and every upload would fail instantly. Backoff
// grows 3 s -> 30 s so a long outage costs little, and the loop only exits
// once connected or when StopRetry() is called.
void ANSAWSS3::RetryLoop() {
int attempt = 0;
int backoffSec = 3;
constexpr int kMaxBackoffSec = 30;
while (!_stopRetry) {
// Sleep in 1-second increments so we can respond to stop quickly.
for (int i = 0; i < backoffSec && !_stopRetry; ++i) {
std::this_thread::sleep_for(std::chrono::seconds(1));
}
if (_stopRetry) break;
attempt++;
// Quick DNS check — is internet available? Try AWS virtual-host
// form first, then fall back to raw _baseDomain for MinIO/Ceph.
{
CkSocket dnsCheck;
CkString dnsResult;
bool resolved = dnsCheck.DnsLookup(_fullAWSURL.c_str(), 3000, dnsResult);
if (!resolved && _fullAWSURL != _baseDomain) {
CkSocket dnsCheck2;
CkString dnsResult2;
resolved = dnsCheck2.DnsLookup(_baseDomain.c_str(), 3000, dnsResult2);
}
if (!resolved) {
_logger.LogDebug("ANSAWSS3::RetryLoop",
"Retry #" + std::to_string(attempt) + " - no internet, retrying in " +
std::to_string(backoffSec) + "s",
__FILE__, __LINE__);
ANS_DBG("AWSS3", "RetryLoop #%d: DNS still failing, next try in %ds",
attempt, backoffSec);
backoffSec = (std::min)(kMaxBackoffSec, backoffSec * 2);
continue;
}
}
// Internet is available — attempt real connection to validate auth/URL
{
std::lock_guard<std::mutex> lk(_configMutex);
bool awsPath = true;
if (TryConnect(awsPath)) {
_logger.LogDebug("ANSAWSS3::RetryLoop",
"Connected successfully after " + std::to_string(attempt) + " retries",
__FILE__, __LINE__);
ANS_DBG("AWSS3", "RetryLoop: connected after %d attempt(s)", attempt);
_retryInProgress = false;
return;
}
}
// DNS resolves but the connection failed — bad credentials, wrong
// URL, or the endpoint is refusing us. Keep trying anyway: the
// caller has no other way back, and this state is often temporary
// (endpoint rebooting, TLS interception clearing, clock skew being
// corrected). Backoff keeps the cost negligible.
_logger.LogError("ANSAWSS3::RetryLoop",
"Retry #" + std::to_string(attempt) +
" - endpoint resolves but connection failed (check authentication or URL). "
"Retrying in " + std::to_string(backoffSec) + "s",
__FILE__, __LINE__);
ANS_DBG("AWSS3", "RetryLoop #%d: connect failed, next try in %ds", attempt, backoffSec);
backoffSec = (std::min)(kMaxBackoffSec, backoffSec * 2);
}
_retryInProgress = false;
}
// ── Connection pool helpers ──
// logOnFailure=false: caller knows this attempt may legitimately fail
// (e.g. TryConnect's first probe of the AWS virtual-host name on a MinIO
// server). Failures are still emitted to ANS_DBG; just not as error-level
// log entries that would scare ops dashboards.
std::unique_ptr<S3Connection> ANSAWSS3::CreateConnection(bool logOnFailure) {
auto conn = std::make_unique<S3Connection>();
// Stamp the config generation this connection is built against so a
// later Connect / SetAuthentication / SetServerProxy can invalidate it
// even if it lands in the pool after that call cleared it.
conn->configGen = _configGen.load();
// Apply request timeouts BEFORE Connect() so ConnectTimeoutMs takes
// effect on this very handshake. IdleTimeoutMs is sticky and applies
// to every subsequent request on this CkRest. A brand-new connection
// gets the FULL idle timeout — it is known-good, so it should be given
// every chance to complete; only reuse gets the short leash.
conn->rest.put_ConnectTimeoutMs(_connectTimeoutMs.load());
conn->rest.put_IdleTimeoutMs(_idleTimeoutMs.load());
// Connect
if (!conn->rest.Connect(_fullAWSURL.c_str(), _port, _bTls, _bAutoReconnect)) {
const std::string summary = SummarizeChilkatError(conn->rest.lastErrorText());
if (logOnFailure) {
_logger.LogError("ANSAWSS3::CreateConnection", summary, __FILE__, __LINE__);
}
ANS_DBG("AWSS3", "Connect failed%s: %s",
logOnFailure ? "" : " (probe, retry expected)", summary.c_str());
return nullptr;
}
// Apply proxy if configured
if (_bProxy) {
conn->socket.put_HttpProxyHostname(_proxyHost.c_str());
conn->socket.put_HttpProxyPort(_proxyPort);
conn->socket.put_HttpProxyUsername(_proxyUsername.c_str());
conn->socket.put_HttpProxyPassword(_proxyPassword.c_str());
conn->socket.put_HttpProxyForHttp(_bProxy);
if (!conn->rest.UseConnection(conn->socket, true)) {
const std::string summary = SummarizeChilkatError(conn->rest.lastErrorText());
if (logOnFailure) {
_logger.LogError("ANSAWSS3::CreateConnection - Proxy error", summary, __FILE__, __LINE__);
}
ANS_DBG("AWSS3", "Proxy error%s: %s",
logOnFailure ? "" : " (probe, retry expected)", summary.c_str());
return nullptr;
}
}
// Apply auth
if (_authReady) {
conn->authAws.put_AccessKey(_accessKey.c_str());
conn->authAws.put_SecretKey(_secretKey.c_str());
conn->authAws.put_ServiceName(_serviceName.c_str());
if (_bucketRegion != "us-east-1") {
if (!_bucketRegion.empty()) conn->authAws.put_Region(_bucketRegion.c_str());
}
if (!conn->rest.SetAuthAws(conn->authAws)) {
const std::string summary = SummarizeChilkatError(conn->rest.lastErrorText());
if (logOnFailure) {
_logger.LogError("ANSAWSS3::CreateConnection - Auth error", summary, __FILE__, __LINE__);
}
ANS_DBG("AWSS3", "Auth error%s: %s",
logOnFailure ? "" : " (probe, retry expected)", summary.c_str());
return nullptr;
}
}
return conn;
}
// Take a connection from the pool, skipping any that are stale (idle too
// long) or built against superseded config. Falls through to a fresh
// connection when nothing reusable is left.
std::unique_ptr<S3Connection> ANSAWSS3::AcquireConnection() {
const uint64_t gen = _configGen.load();
const auto now = std::chrono::steady_clock::now();
for (;;) {
std::unique_ptr<S3Connection> conn;
{
std::lock_guard<std::mutex> lk(_poolMutex);
if (_pool.empty()) break;
conn = std::move(_pool.back());
_pool.pop_back();
}
if (conn->configGen != gen) {
ANS_DBG("AWSS3", "Acquire: dropping pooled conn from older config gen %llu (now %llu)",
(unsigned long long)conn->configGen, (unsigned long long)gen);
DiscardConnection(std::move(conn)); // pool lock released first
continue;
}
const long long idleMs = std::chrono::duration_cast<std::chrono::milliseconds>(
now - conn->lastUsed).count();
if (idleMs > kMaxPooledIdleMs) {
ANS_DBG("AWSS3", "Acquire: dropping pooled conn idle for %lldms (> %dms)",
idleMs, kMaxPooledIdleMs);
DiscardConnection(std::move(conn));
continue;
}
// Clear per-request state left behind by whoever used this
// connection last. Multipart uploads set uploadId / partNumber
// query params; a plain PUT that inherited them would target the
// wrong resource. Content-Encoding is set by UploadTextData only,
// and a JPEG that inherited "gzip" would be stored with metadata
// saying it is compressed — browsers would then fail to display it.
// NOTE: do NOT call ClearAllHeaders here — it wipes Chilkat's SigV4
// state (see commit fcc985e). Headers that every upload sets
// explicitly (Content-Type, Expect) are overwritten by AddHeader
// and need no clearing.
conn->rest.ClearAllQueryParams();
conn->rest.RemoveHeader("Content-Encoding");
// Short leash on reuse: if this socket died while idle, fail fast
// and let the retry run on a fresh connection.
conn->rest.put_IdleTimeoutMs(_reusedIdleTimeoutMs.load());
return conn;
}
// Pool empty (or everything in it was stale) — create a new connection.
std::lock_guard<std::mutex> cfgLk(_configMutex);
return CreateConnection();
}
void ANSAWSS3::ReleaseConnection(std::unique_ptr<S3Connection> conn) {
if (!conn) return;
// Mark as fresh and restore the full idle timeout; AcquireConnection
// re-applies the short one when it hands the connection out again.
conn->lastUsed = std::chrono::steady_clock::now();
conn->rest.put_IdleTimeoutMs(_idleTimeoutMs.load());
std::lock_guard<std::mutex> lk(_poolMutex);
_pool.push_back(std::move(conn));
}
// Close a suspect connection instead of pooling it. Disconnect is given a
// small budget only — the point is to release the socket, not to wait on a
// peer that has already stopped answering.
void ANSAWSS3::DiscardConnection(std::unique_ptr<S3Connection> conn) {
if (!conn) return;
conn->rest.Disconnect(100);
ANS_DBG("AWSS3", "Discarded connection (not returned to pool)");
// unique_ptr destructor tears the CkRest down here.
}
// Pre-populate the pool with `count` ready-to-use connections. Each
// connection performs a TLS handshake; doing `count` of them in parallel
// (rather than serially as AcquireConnection would on a cold pool) cuts
// the first-upload latency for workloads that fan out right after
// Connect() / SetAuthentication().
int ANSAWSS3::PrewarmConnectionPool(int count) {
if (count <= 0) return 0;
if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) {
_logger.LogError("ANSAWSS3::PrewarmConnectionPool",
"Not ready: call Connect() and SetAuthentication() first",
__FILE__, __LINE__);
return 0;
}
// Hold _configMutex once across the whole prewarm. It blocks config
// writes for the duration, but does NOT block individual worker
// threads since CreateConnection() itself does not re-acquire it.
std::lock_guard<std::mutex> cfgLk(_configMutex);
std::vector<std::unique_ptr<S3Connection>> built(count);
std::vector<std::thread> workers;
workers.reserve(count);
for (int i = 0; i < count; ++i) {
workers.emplace_back([this, i, &built]() {
try {
built[i] = CreateConnection();
} catch (...) {
built[i] = nullptr;
}
});
}
for (auto& t : workers) t.join();
int added = 0;
{
std::lock_guard<std::mutex> lk(_poolMutex);
for (auto& conn : built) {
if (conn) {
_pool.push_back(std::move(conn));
++added;
}
}
}
_logger.LogDebug("ANSAWSS3::PrewarmConnectionPool",
"Added " + std::to_string(added) + "/" + std::to_string(count) + " connections to pool",
__FILE__, __LINE__);
return added;
}
// Update the per-connection request timeouts. New values are applied
// immediately to all pooled connections (IdleTimeoutMs is picked up on
// the next request; ConnectTimeoutMs is harmless on already-connected
// CkRest instances but takes effect if they ever reconnect) and baked
// into the defaults used by future CreateConnection() calls.
bool ANSAWSS3::SetTimeouts(int connectMs, int idleMs) {
if (connectMs < 1000 || idleMs < 1000) {
_logger.LogError("ANSAWSS3::SetTimeouts",
"Timeout values must be >= 1000 ms (got connect=" +
std::to_string(connectMs) + ", idle=" + std::to_string(idleMs) + ")",
__FILE__, __LINE__);
return false;
}
_connectTimeoutMs.store(connectMs);
_idleTimeoutMs.store(idleMs);
// Reuse timeout scales with the configured idle timeout (a quarter of
// it, clamped to a sane floor) so a deployment on a slow link that
// raises idleMs does not end up with an over-aggressive leash on
// pooled sockets.
// Never longer than the configured idle timeout itself (idleMs can be
// as low as 1000), and never so short that a healthy but busy server
// gets cut off.
const int reusedMs = (std::min)(idleMs, (std::max)(2000, idleMs / 4));
_reusedIdleTimeoutMs.store(reusedMs);
// Propagate to existing pooled connections so already-warm workers
// start using the new values on their next request.
{
std::lock_guard<std::mutex> lk(_poolMutex);
for (auto& conn : _pool) {
if (conn) {
conn->rest.put_ConnectTimeoutMs(connectMs);
conn->rest.put_IdleTimeoutMs(idleMs);
}
}
}
_logger.LogDebug("ANSAWSS3::SetTimeouts",
"connect=" + std::to_string(connectMs) + "ms, idle=" + std::to_string(idleMs) +
"ms, idle-on-reuse=" + std::to_string(reusedMs) + "ms",
__FILE__, __LINE__);
return true;
}
// Shared precondition check for every upload entry point. The distinction
// matters to the caller: a licence problem needs an operator, whereas "not
// connected" clears itself once the background RetryLoop gets through — so
// the two report different status codes.
bool ANSAWSS3::CheckUploadPreconditions(const std::string& opName) {
if (!_isLicenseValid || !_isUnlockCodeValid) {
_logger.LogError(opName,
!_isLicenseValid ? "Invalid license" : "Invalid unlock code",
__FILE__, __LINE__);
SetCallStatus(kCallPermanentError);
return false;
}
if (!_bConnected) {
const bool retrying = _retryInProgress.load();
_logger.LogError(opName,
retrying ? "Not connected (waiting for internet, retrying in background)"
: "Not connected (no internet or connection not established)",
__FILE__, __LINE__);
ANS_DBG("AWSS3", "%s: not connected (background retry %s) — caller should retry later",
opName.c_str(), retrying ? "active" : "inactive");
SetCallStatus(kCallNotConnected);
return false;
}
return true;
}
// See the declaration comment in ANSUtilities.h for why a failed Chilkat
// request is not automatically a network problem.
ANSAWSS3::AttemptResult ANSAWSS3::ClassifyFailedRequest(
std::unique_ptr<S3Connection> conn, std::string& lastError) {
const int statusCode = conn ? conn->rest.get_ResponseStatusCode() : 0;
lastError = SummarizeChilkatRequestError(
conn ? conn->rest.lastErrorText() : nullptr, statusCode);
// The server answered, and its answer will not change on a retry:
// wrong bucket, no permission, bad key. 408 and 429 are the exceptions
// — those explicitly ask the client to try again.
if (statusCode >= 400 && statusCode <= 499 &&
statusCode != 408 && statusCode != 429) {
ANS_DBG("AWSS3", "Request failed permanently: %s", lastError.c_str());
ReleaseConnection(std::move(conn)); // the socket itself is fine
return AttemptResult::Permanent;
}
ANS_DBG("AWSS3", "Request failed transiently: %s", lastError.c_str());
DiscardConnection(std::move(conn)); // suspect socket — do not pool
return AttemptResult::Transient;
}
// Drives a retry loop for a single upload operation. See the declaration
// comment in ANSUtilities.h for the AttemptResult contract.
//
// Retry policy (shared across all upload entry points):
// - kUploadMaxAttempts attempts total, AND kUploadDeadlineMs wall-clock
// (whichever comes first — the attempt count alone does not bound how
// long the caller blocks, since one attempt can sit on a timeout)
// - kUploadRetryDelayMs between attempts (no delay after the last one)
// - std::exception thrown inside attemptFn is treated as Transient
// - Any other exception is treated as Permanent (bail immediately)
bool ANSAWSS3::UploadWithRetry(
const std::string& opName,
const std::function<AttemptResult(std::string& lastError)>& attemptFn) {
std::string lastError;
const auto started = std::chrono::steady_clock::now();
auto elapsedMs = [&started]() {
return std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now() - started).count();
};
for (int attempt = 1; attempt <= kUploadMaxAttempts; ++attempt) {
AttemptResult result = AttemptResult::Transient;
try {
result = attemptFn(lastError);
}
catch (const std::exception& e) {
lastError = std::string("Exception: ") + e.what();
result = AttemptResult::Transient;
}
catch (...) {
lastError = "Unknown exception";
result = AttemptResult::Permanent;
}
if (result == AttemptResult::Success) {
SetCallStatus(kCallSuccess);
return true;
}
if (result == AttemptResult::Permanent) {
_logger.LogError(opName,
"Permanent failure (no retry): " + lastError,
__FILE__, __LINE__);
SetCallStatus(kCallPermanentError);
return false;
}
// Transient — retry if both budgets allow it.
if (elapsedMs() >= kUploadDeadlineMs) {
_logger.LogError(opName,
"Upload gave up after " + std::to_string(attempt) + " attempt(s) / " +
std::to_string(elapsedMs()) + "ms (deadline " +
std::to_string(kUploadDeadlineMs) + "ms). Last error: " + lastError,
__FILE__, __LINE__);
ANS_DBG("AWSS3", "%s: deadline exceeded after %d attempt(s), %lldms",
opName.c_str(), attempt, elapsedMs());
// Ran out of time on transient errors — the caller should retry.
SetCallStatus(kCallTransientError);
return false;
}
if (attempt < kUploadMaxAttempts) {
_logger.LogDebug(opName,
"Attempt " + std::to_string(attempt) + "/" +
std::to_string(kUploadMaxAttempts) + " failed after " +
std::to_string(elapsedMs()) + "ms: " + lastError +
" — retrying in " + std::to_string(kUploadRetryDelayMs) + "ms",
__FILE__, __LINE__);
std::this_thread::sleep_for(
std::chrono::milliseconds(kUploadRetryDelayMs));
}
}
_logger.LogError(opName,
"Upload failed after " + std::to_string(kUploadMaxAttempts) +
" attempts / " + std::to_string(elapsedMs()) + "ms. Last error: " + lastError,
__FILE__, __LINE__);
// Every attempt hit a transient error — worth retrying from the caller.
SetCallStatus(kCallTransientError);
return false;
}
void ANSAWSS3::CheckLicense() {
// Note: caller (Initialize) already holds _configMutex
try {
// Check once globally
std::call_once(ansawss3LicenseOnceFlag, [this]() {
VerifyGlobalANSAWSS3License(_licenseKey);
});
// Update this instance's local license flag
_isLicenseValid = ansawss3LicenceValid;
}
catch (const std::exception& e) {
this->_logger.LogFatal("ANSAWSS3::CheckLicense. Error:", e.what(), __FILE__, __LINE__);
}
}
void ANSAWSS3::CheckUnlockCode() {
// Note: caller (Initialize) already holds _configMutex
try {
CkGlobal glob;
_unlockCode = "ANSDRC.CB1122026_MEQCIFwO1IFQCG0BhZwsXFO68QUU6mDB5uge4duOsqOJanEyAiAB67ahqnXin4SRy0vIegISgbFlpldmbuS5gbU21GYVqA==";// "ANSDRC.CB1082025_Ax6P3M7F8B3d";
_isUnlockCodeValid = glob.UnlockBundle(_unlockCode.c_str());
if (!_isUnlockCodeValid) {
_logger.LogFatal("ANSAWSS3::CheckUnlockCode", glob.lastErrorText(), __FILE__, __LINE__);
return;
}
int status = glob.get_UnlockStatus();
if (status != 2) {
_logger.LogDebug("ANSAWSS3::CheckUnlockCode", "Unlocked in trial mode.", __FILE__, __LINE__);
}
}
catch (const std::exception& e) {
_logger.LogFatal("ANSAWSS3::CheckUnlockCode", e.what(), __FILE__, __LINE__);
}
}
bool ANSAWSS3::Initialize(const std::string& licenseKey) {
std::lock_guard<std::mutex> lock(_configMutex);
try {
_licenseKey = licenseKey;
CheckLicense();
CheckUnlockCode();
return _isLicenseValid && _isUnlockCodeValid;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSAWSS3::Initialize", e.what(), __FILE__, __LINE__);
return false;
}
}
bool ANSAWSS3::SetServerProxy(const std::string& proxyHost, int proxyPort,const std::string& proxyUsername,const std::string& proxyPassword) {
// Early validation checks
if (!_isLicenseValid || !_isUnlockCodeValid) {
_logger.LogError("ANSAWSS3::SetServerProxy",
!_isLicenseValid ? "Invalid license" : "Invalid unlock code",
__FILE__, __LINE__);
return false;
}
std::lock_guard<std::mutex> lock(_configMutex);
try {
_proxyHost = proxyHost;
_proxyPort = proxyPort;
_proxyUsername = proxyUsername;
_proxyPassword = proxyPassword;
// Simplified proxy validation logic
_bProxy = !proxyHost.empty() && proxyPort > 0;
// Clear pool so new connections pick up proxy config. Bumping the
// generation also invalidates any connection still in flight
// elsewhere, which would otherwise be pooled after this clear.
_configGen.fetch_add(1);
{
std::lock_guard<std::mutex> lk(_poolMutex);
_pool.clear();
}
return true;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSAWSS3::SetServerProxy", e.what(), __FILE__, __LINE__);
return false;
}
}
// Returns: 1 = connected, 0 = failed (bad auth/URL), 2 = no internet (background retry started)
int ANSAWSS3::Connect(const std::string& baseDomain, const std::string& bucketRegion, const std::string& serviceName, int port, bool bTls, bool autoReconnect, bool &awsPath)
{
// Early validation checks
if (!_isLicenseValid || !_isUnlockCodeValid) {
_logger.LogError("ANSAWSS3::Connect",
!_isLicenseValid ? "Invalid license" : "Invalid unlock code",
__FILE__, __LINE__);
return 0;
}
// Stop any existing background retry
StopRetry();
std::lock_guard<std::mutex> lock(_configMutex);
try {
// Store parameters — strip http:// or https:// from baseDomain
{
std::string domain = baseDomain.empty() ? "amazonaws.com" : baseDomain;
if (domain.rfind("https://", 0) == 0) {
domain = domain.substr(8);
bTls = true; // caller said https, honour it
}
else if (domain.rfind("http://", 0) == 0) {
domain = domain.substr(7);
}
// Remove trailing slash if any
if (!domain.empty() && domain.back() == '/') domain.pop_back();
_baseDomain = domain;
}
_bucketRegion = bucketRegion.empty() ? "us-east-1" : bucketRegion;
_serviceName = serviceName.empty() ? "s3" : serviceName;
_fullAWSURL = _serviceName + "." + _bucketRegion + "." + _baseDomain;
_port = port;
_bTls = bTls;
_bAutoReconnect = autoReconnect;
// Clear pool so new connections use updated config (see the
// generation note in SetServerProxy).
_configGen.fetch_add(1);
{
std::lock_guard<std::mutex> lk(_poolMutex);
_pool.clear();
}
// Check if internet is available (fast DNS check). Try the AWS
// virtual-host form first; if that synthetic name doesn't exist
// (MinIO, Ceph, custom S3-compatible servers), fall back to the
// raw _baseDomain — that is the actual host the user gave us.
{
CkSocket dnsCheck;
CkString dnsResult;
bool resolved = dnsCheck.DnsLookup(_fullAWSURL.c_str(), 3000, dnsResult);
if (!resolved && _fullAWSURL != _baseDomain) {
CkSocket dnsCheck2;
CkString dnsResult2;
resolved = dnsCheck2.DnsLookup(_baseDomain.c_str(), 3000, dnsResult2);
if (resolved) {
ANS_DBG("AWSS3",
"DNS for '%s' failed but '%s' resolved — proceeding with path-style/MinIO fallback",
_fullAWSURL.c_str(), _baseDomain.c_str());
}
}
if (!resolved) {
// No internet — start background retry, return immediately
_logger.LogDebug("ANSAWSS3::Connect",
"No internet available, starting background retry for '" + _fullAWSURL + "' / '" + _baseDomain + "'...",
__FILE__, __LINE__);
_stopRetry = false;
_retryInProgress = true;
_retryThread = std::thread(&ANSAWSS3::RetryLoop, this);
return 2; // no internet, retrying in background
}
}
// Internet is available — try real connection
if (TryConnect(awsPath)) {
return 1; // connected successfully
}
// Internet available but connection failed (bad auth, wrong URL, etc.)
return 0;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSAWSS3::Connect", e.what(), __FILE__, __LINE__);
return 0;
}
}
bool ANSAWSS3::SetAuthentication(const std::string& accessKey,const std::string& secretKey) {
// Early validation checks
if (!_isLicenseValid || !_isUnlockCodeValid) {
_logger.LogError("ANSAWSS3::SetAuthentication",
!_isLicenseValid ? "Invalid license" : "Invalid unlock code",
__FILE__, __LINE__);
return false;
}
// Write credentials + clear the pool under _configMutex. The lock is
// released before the auto-prewarm below so PrewarmConnectionPool
// (which also takes _configMutex) doesn't deadlock.
{
std::lock_guard<std::mutex> lock(_configMutex);
try {
_accessKey = accessKey;
_secretKey = secretKey;
_authReady = true;
// Clear pool so new connections pick up new credentials. The
// generation bump matters most here: the background RetryLoop
// can complete a connect at any moment, and a connection it
// built before _authReady was set carries no SigV4 state — if
// it landed in the pool after this clear, every upload using it
// would come back 403 (a permanent failure, so not retried).
_configGen.fetch_add(1);
std::lock_guard<std::mutex> lk(_poolMutex);
_pool.clear();
}
catch (const std::exception& e) {
_logger.LogFatal("ANSAWSS3::SetAuthentication", e.what(), __FILE__, __LINE__);
return false;
}
}
// Auto-prewarm the connection pool so the first N concurrent uploads
// don't serialize on TLS handshake. No-op if not connected yet.
static constexpr int kDefaultPrewarmCount = 8;
PrewarmConnectionPool(kDefaultPrewarmCount);
return true;
}
std::vector<std::string> ANSAWSS3::ListBuckets() {
// Early validation checks
if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) {
_logger.LogError("ANSAWSS3::ListBuckets",
!_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)",
__FILE__, __LINE__);
return {};
}
try {
auto conn = AcquireConnection();
if (!conn) {
_logger.LogError("ListBuckets", "Failed to acquire S3 connection", __FILE__, __LINE__);
return {};
}
// Make request and capture response
conn->rest.put_Host(_fullAWSURL.c_str());
CkStringBuilder sbResponse;
if (!conn->rest.FullRequestNoBodySb("GET", "/", sbResponse)) {
_logger.LogError("ANSAWSS3::ListBuckets - Request failed",
conn->rest.lastErrorText(), __FILE__, __LINE__);
DiscardConnection(std::move(conn)); // transport failed — do not pool a suspect socket
return {};
}
// Check HTTP status code
int statusCode = conn->rest.get_ResponseStatusCode();
if (statusCode != 200) {
_logger.LogError("ANSAWSS3::ListBuckets - HTTP " + std::to_string(statusCode),
sbResponse.getAsString(), __FILE__, __LINE__);
ReleaseConnection(std::move(conn));
return {};
}
// Parse XML response
CkXml xml;
if (!xml.LoadSb(sbResponse, true)) {
_logger.LogError("ANSAWSS3::ListBuckets - XML parse error",
xml.lastErrorText(), __FILE__, __LINE__);
ReleaseConnection(std::move(conn));
return {};
}
// Extract bucket names
std::vector<std::string> bucketList;
int bucketCount = xml.NumChildrenHavingTag("Buckets|Bucket");
if (bucketCount > 0) {
bucketList.reserve(bucketCount);
for (int i = 0; i < bucketCount; ++i) {
xml.put_I(i);
const char* name = xml.getChildContent("Buckets|Bucket[i]|Name");
// Validate name before adding
if (name != nullptr && name[0] != '\0') {
bucketList.emplace_back(name);
}
}
}
ReleaseConnection(std::move(conn));
return bucketList;
}
catch (const std::exception& e) {
_logger.LogFatal("ANSAWSS3::ListBuckets", e.what(), __FILE__, __LINE__);
return {};
}
}
std::vector<std::string> ANSAWSS3::ListBucketObjects(const std::string& bucketName) {
// Early validation checks
if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) {
_logger.LogError("ANSAWSS3::ListBucketObjects",
!_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)",
__FILE__, __LINE__);
return {};
}
if (bucketName.empty()) {
_logger.LogError("ANSAWSS3::ListBucketObjects",
"Bucket name is empty",
__FILE__, __LINE__);
return {};
}
std::vector<std::string> objectList;
try {
auto conn = AcquireConnection();
if (!conn) {
_logger.LogError("ListBucketObjects", "Failed to acquire S3 connection", __FILE__, __LINE__);
return {};
}
// Set bucket-specific endpoint
if (_bAwsPath) {
conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str());
} else {
conn->rest.put_Host(_fullAWSURL.c_str());
}
std::string marker = "";
bool isTruncated = true;
int pageCount = 0;
// Loop to handle pagination
while (isTruncated) {
pageCount++;
// Build the request path with marker if available
std::string basePath = _bAwsPath ? "/" : ("/" + bucketName + "/");
std::string requestPath = basePath;
if (!marker.empty()) {
requestPath = basePath + "?marker=" + marker;
}
CkStringBuilder sbResponse;
bool success = conn->rest.FullRequestNoBodySb("GET", requestPath.c_str(), sbResponse);
if (!success) {
_logger.LogError("ANSAWSS3::ListBucketObjects - Request failed on page " + std::to_string(pageCount),
conn->rest.lastErrorText(), __FILE__, __LINE__);
DiscardConnection(std::move(conn)); // transport failed — do not pool a suspect socket
return objectList; // Return what we have so far
}
int statusCode = conn->rest.get_ResponseStatusCode();
if (statusCode != 200) {
std::string errorMsg = "HTTP " + std::to_string(statusCode);
std::string response = sbResponse.getAsString();
if (!response.empty()) {
errorMsg += " - " + response;
}
_logger.LogError("ANSAWSS3::ListBucketObjects - Page " + std::to_string(pageCount),
errorMsg, __FILE__, __LINE__);
ReleaseConnection(std::move(conn));
return objectList; // Return what we have so far
}
// Parse XML response
CkXml xml;
bool loadSuccess = xml.LoadSb(sbResponse, true);
if (!loadSuccess) {
_logger.LogError("ANSAWSS3::ListBucketObjects",
"Failed to parse XML response on page " + std::to_string(pageCount) + ": " + std::string(xml.lastErrorText()),
__FILE__, __LINE__);
ReleaseConnection(std::move(conn));
return objectList; // Return what we have so far
}
// Check if results are truncated (more pages to fetch)
const char* truncatedStr = xml.getChildContent("IsTruncated");
isTruncated = (truncatedStr != nullptr && std::string(truncatedStr) == "true");
// Get the marker for next page
if (isTruncated) {
// First try to get NextMarker
const char* nextMarker = xml.getChildContent("NextMarker");
if (nextMarker != nullptr && nextMarker[0] != '\0') {
marker = std::string(nextMarker);
}
else {
// If NextMarker is not present, use the last Key as marker
int count = xml.NumChildrenHavingTag("Contents");
if (count > 0) {
xml.put_I(count - 1); // Last element
const char* lastKey = xml.getChildContent("Contents[i]|Key");
if (lastKey != nullptr && lastKey[0] != '\0') {
marker = std::string(lastKey);
}
else {
isTruncated = false;
}
}
else {
isTruncated = false;
}
}
}
// Iterate through all Contents elements in this page
int count = xml.NumChildrenHavingTag("Contents");
for (int i = 0; i < count; i++) {
xml.put_I(i);
const char* key = xml.getChildContent("Contents[i]|Key");
if (key != nullptr && key[0] != '\0') {
objectList.emplace_back(key);
}
}
_logger.LogDebug("ANSAWSS3::ListBucketObjects",
"Page " + std::to_string(pageCount) + ": Retrieved " + std::to_string(count) +
" objects. Total so far: " + std::to_string(objectList.size()),
__FILE__, __LINE__);
}
_logger.LogDebug("ANSAWSS3::ListBucketObjects",
"Successfully listed " + std::to_string(objectList.size()) +
" objects in bucket: " + bucketName + " (" + std::to_string(pageCount) + " pages)",
__FILE__, __LINE__);
ReleaseConnection(std::move(conn));
}
catch (const std::exception& e) {
_logger.LogFatal("ANSAWSS3::ListBucketObjects",
std::string("Exception: ") + e.what(),
__FILE__, __LINE__);
objectList.clear(); // Ensure we return empty list on exception
}
return objectList;
}
std::vector<std::string> ANSAWSS3::ListBucketObjectsWithPrefix(const std::string& bucketName, const std::string& prefix) {
// Early validation checks
if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) {
_logger.LogError("ANSAWSS3::ListBucketObjectsWithPrefix",
!_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)",
__FILE__, __LINE__);
return {};
}
if (bucketName.empty()) {
_logger.LogError("ANSAWSS3::ListBucketObjectsWithPrefix",
"Bucket name is empty",
__FILE__, __LINE__);
return {};
}
// prefix can be empty to list all objects
std::vector<std::string> objectList;
try {
auto conn = AcquireConnection();
if (!conn) {
_logger.LogError("ListBucketObjectsWithPrefix", "Failed to acquire S3 connection", __FILE__, __LINE__);
return {};
}
// Set bucket-specific endpoint
if (_bAwsPath) {
conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str());
} else {
conn->rest.put_Host(_fullAWSURL.c_str());
}
std::string marker = "";
bool isTruncated = true;
int pageCount = 0;
// Loop to handle pagination
while (isTruncated) {
pageCount++;
// Build the request path with prefix and marker
std::string basePath = _bAwsPath ? "/" : ("/" + bucketName + "/");
std::string requestPath = basePath;
bool hasParams = false;
// Add prefix parameter if provided
if (!prefix.empty()) {
requestPath += "?prefix=" + prefix;
hasParams = true;
}
// Add marker parameter if available
if (!marker.empty()) {
if (hasParams) {
requestPath += "&marker=" + marker;
}
else {
requestPath += "?marker=" + marker;
hasParams = true;
}
}
CkStringBuilder sbResponse;
bool success = conn->rest.FullRequestNoBodySb("GET", requestPath.c_str(), sbResponse);
if (!success) {
_logger.LogError("ANSAWSS3::ListBucketObjectsWithPrefix - Request failed on page " + std::to_string(pageCount),
conn->rest.lastErrorText(), __FILE__, __LINE__);
DiscardConnection(std::move(conn)); // transport failed — do not pool a suspect socket
return objectList; // Return what we have so far
}
int statusCode = conn->rest.get_ResponseStatusCode();
if (statusCode != 200) {
std::string errorMsg = "HTTP " + std::to_string(statusCode);
std::string response = sbResponse.getAsString();
if (!response.empty()) {
errorMsg += " - " + response;
}
_logger.LogError("ANSAWSS3::ListBucketObjectsWithPrefix - Page " + std::to_string(pageCount),
errorMsg, __FILE__, __LINE__);
ReleaseConnection(std::move(conn));
return objectList; // Return what we have so far
}
// Parse XML response
CkXml xml;
bool loadSuccess = xml.LoadSb(sbResponse, true);
if (!loadSuccess) {
_logger.LogError("ANSAWSS3::ListBucketObjectsWithPrefix",
"Failed to parse XML response on page " + std::to_string(pageCount) + ": " + std::string(xml.lastErrorText()),
__FILE__, __LINE__);
ReleaseConnection(std::move(conn));
return objectList; // Return what we have so far
}
// Check if results are truncated (more pages to fetch)
const char* truncatedStr = xml.getChildContent("IsTruncated");
isTruncated = (truncatedStr != nullptr && std::string(truncatedStr) == "true");
// Get the marker for next page
if (isTruncated) {
// First try to get NextMarker
const char* nextMarker = xml.getChildContent("NextMarker");
if (nextMarker != nullptr && nextMarker[0] != '\0') {
marker = std::string(nextMarker);
}
else {
// If NextMarker is not present, use the last Key as marker
int count = xml.NumChildrenHavingTag("Contents");
if (count > 0) {
xml.put_I(count - 1); // Last element
const char* lastKey = xml.getChildContent("Contents[i]|Key");
if (lastKey != nullptr && lastKey[0] != '\0') {
marker = std::string(lastKey);
}
else {
isTruncated = false;
}
}
else {
isTruncated = false;
}
}
}
// Iterate through all Contents elements in this page
int count = xml.NumChildrenHavingTag("Contents");
for (int i = 0; i < count; i++) {
xml.put_I(i);
const char* key = xml.getChildContent("Contents[i]|Key");
if (key != nullptr && key[0] != '\0') {
objectList.emplace_back(key);
}
}
std::string prefixInfo = prefix.empty() ? "all objects" : "prefix '" + prefix + "'";
_logger.LogDebug("ANSAWSS3::ListBucketObjectsWithPrefix",
"Page " + std::to_string(pageCount) + ": Retrieved " + std::to_string(count) +
" objects for " + prefixInfo + ". Total so far: " + std::to_string(objectList.size()),
__FILE__, __LINE__);
}
std::string prefixInfo = prefix.empty() ? "" : " with prefix '" + prefix + "'";
_logger.LogDebug("ANSAWSS3::ListBucketObjectsWithPrefix",
"Successfully listed " + std::to_string(objectList.size()) +
" objects in bucket: " + bucketName + prefixInfo + " (" + std::to_string(pageCount) + " pages)",
__FILE__, __LINE__);
ReleaseConnection(std::move(conn));
}
catch (const std::exception& e) {
_logger.LogFatal("ANSAWSS3::ListBucketObjectsWithPrefix",
std::string("Exception: ") + e.what(),
__FILE__, __LINE__);
objectList.clear(); // Ensure we return empty list on exception
}
return objectList;
}
bool ANSAWSS3::CreateBucket(const std::string& bucketName) {
// Early validation checks
if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) {
_logger.LogError("ANSAWSS3::CreateBucket",
!_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)",
__FILE__, __LINE__);
return false;
}
// Validate input parameters
if (bucketName.empty()) {
_logger.LogError("ANSAWSS3::CreateBucket",
"Bucket name cannot be empty",
__FILE__, __LINE__);
return false;
}
bool createSuccess = false;
try {
auto conn = AcquireConnection();
if (!conn) {
_logger.LogError("CreateBucket", "Failed to acquire S3 connection", __FILE__, __LINE__);
return false;
}
// Build global endpoint
std::string globalEndpoint = _serviceName + "." + _baseDomain;
CkStringBuilder sbBucketRegion;
sbBucketRegion.Append(_bucketRegion.c_str());
// We only need to specify the LocationConstraint if the bucket's region is NOT us-east-1
CkXml xml;
if (!sbBucketRegion.ContentsEqual("us-east-1", true)) {
xml.put_Tag("CreateBucketConfiguration");
xml.AddAttribute("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/");
xml.UpdateChildContent("LocationConstraint", _bucketRegion.c_str());
}
// Set bucket-specific endpoint
if (_bAwsPath) {
conn->rest.put_Host((bucketName + "." + globalEndpoint).c_str());
} else {
conn->rest.put_Host(_fullAWSURL.c_str());
}
// Make the call to create the bucket
std::string createPath = _bAwsPath ? "/" : ("/" + bucketName);
const char* responseStr = nullptr;
if (!sbBucketRegion.ContentsEqual("us-east-1", true)) {
responseStr = conn->rest.fullRequestString("PUT", createPath.c_str(), xml.getXml());
}
else {
// If the bucket is to be created in the us-east-1 region (the default region)
// just send a PUT with no body
responseStr = conn->rest.fullRequestNoBody("PUT", createPath.c_str());
}
if (conn->rest.get_LastMethodSuccess() != true) {
_logger.LogError("ANSAWSS3::CreateBucket - Request failed",
conn->rest.lastErrorText(), __FILE__, __LINE__);
DiscardConnection(std::move(conn)); // transport failed — do not pool a suspect socket
return false;
}
int statusCode = conn->rest.get_ResponseStatusCode();
if (statusCode == 200) {
_logger.LogDebug("ANSAWSS3::CreateBucket",
"Successfully created bucket: " + bucketName + " in region: " + _bucketRegion,
__FILE__, __LINE__);
createSuccess = true;
}
else {
std::string errorMsg = "HTTP " + std::to_string(statusCode);
if (responseStr != nullptr && responseStr[0] != '\0') {
CkXml responseXml;
if (responseXml.LoadXml(responseStr)) {
const char* errorCode = responseXml.getChildContent("Code");
const char* errorMessage = responseXml.getChildContent("Message");
if (errorCode != nullptr) {
errorMsg += " - " + std::string(errorCode);
}
if (errorMessage != nullptr) {
errorMsg += ": " + std::string(errorMessage);
}
}
else {
errorMsg += "\n" + std::string(responseStr);
}
}
if (statusCode == 409) {
_logger.LogError("ANSAWSS3::CreateBucket", errorMsg, __FILE__, __LINE__);
}
else {
_logger.LogError("ANSAWSS3::CreateBucket", errorMsg, __FILE__, __LINE__);
}
}
ReleaseConnection(std::move(conn));
}
catch (const std::exception& e) {
_logger.LogFatal("ANSAWSS3::CreateBucket",
std::string("Exception: ") + e.what(),
__FILE__, __LINE__);
}
return createSuccess;
}
bool ANSAWSS3::DeleteBucket(const std::string& bucketName) {
// Early validation checks
if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) {
_logger.LogError("ANSAWSS3::DeleteBucket",
!_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)",
__FILE__, __LINE__);
return false;
}
if (bucketName.empty()) {
_logger.LogError("ANSAWSS3::DeleteBucket",
"Bucket name cannot be empty",
__FILE__, __LINE__);
return false;
}
bool deleteSuccess = false;
try {
auto conn = AcquireConnection();
if (!conn) {
_logger.LogError("DeleteBucket", "Failed to acquire S3 connection", __FILE__, __LINE__);
return false;
}
// Configure AWS authentication
conn->rest.ClearAuth();
conn->authAws.put_AccessKey(_accessKey.c_str());
conn->authAws.put_SecretKey(_secretKey.c_str());
conn->authAws.put_ServiceName(_serviceName.c_str());
bool authSuccess = conn->rest.SetAuthAws(conn->authAws);
if (!authSuccess) {
_logger.LogError("ANSAWSS3::DeleteBucket",
"Failed to set AWS authentication: " + std::string(conn->rest.lastErrorText()),
__FILE__, __LINE__);
DiscardConnection(std::move(conn)); // transport failed — do not pool a suspect socket
return false;
}
// Set bucket-specific endpoint
std::string globalEndpoint = _serviceName + "." + _baseDomain;
if (_bAwsPath) {
conn->rest.put_Host((bucketName + "." + globalEndpoint).c_str());
} else {
conn->rest.put_Host(_fullAWSURL.c_str());
}
std::string deletePath = _bAwsPath ? "/" : ("/" + bucketName);
CkStringBuilder sbResponse;
bool success = conn->rest.FullRequestNoBodySb("DELETE", deletePath.c_str(), sbResponse);
if (!success) {
_logger.LogError("ANSAWSS3::DeleteBucket",
"Failed to send DELETE request: " + std::string(conn->rest.lastErrorText()),
__FILE__, __LINE__);
}
else {
int statusCode = conn->rest.get_ResponseStatusCode();
// S3 returns 204 (No Content) for successful bucket deletion
if (statusCode == 204) {
_logger.LogDebug("ANSAWSS3::DeleteBucket",
"Successfully deleted bucket: " + bucketName,
__FILE__, __LINE__);
deleteSuccess = true;
}
else if (statusCode == 404) {
_logger.LogWarn("ANSAWSS3::DeleteBucket",
"Bucket not found (already deleted?): " + bucketName,
__FILE__, __LINE__);
}
else if (statusCode == 409) {
// Bucket not empty
std::string errorMsg = "HTTP 409 - Bucket is not empty and cannot be deleted";
std::string response = sbResponse.getAsString();
if (!response.empty()) {
CkXml responseXml;
if (responseXml.LoadXml(response.c_str())) {
const char* errorCode = responseXml.getChildContent("Code");
const char* errorMessage = responseXml.getChildContent("Message");
if (errorCode != nullptr) {
errorMsg += " - " + std::string(errorCode);
}
if (errorMessage != nullptr) {
errorMsg += ": " + std::string(errorMessage);
}
}
}
_logger.LogError("ANSAWSS3::DeleteBucket", errorMsg, __FILE__, __LINE__);
}
else {
std::string errorMsg = "HTTP " + std::to_string(statusCode);
std::string response = sbResponse.getAsString();
if (!response.empty()) {
errorMsg += " - " + response;
}
_logger.LogError("ANSAWSS3::DeleteBucket", errorMsg, __FILE__, __LINE__);
}
}
ReleaseConnection(std::move(conn));
}
catch (const std::exception& e) {
_logger.LogFatal("ANSAWSS3::DeleteBucket",
std::string("Exception: ") + e.what(),
__FILE__, __LINE__);
}
return deleteSuccess;
}
bool ANSAWSS3::CreateFolder(const std::string& bucketName, const std::string& prefix) {
// Early validation checks
if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) {
_logger.LogError("ANSAWSS3::CreateFolder",
!_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)",
__FILE__, __LINE__);
return false;
}
if (bucketName.empty() || prefix.empty()) {
_logger.LogError("ANSAWSS3::CreateFolder",
"Bucket name or folder path is empty",
__FILE__, __LINE__);
return false;
}
bool createSuccess = false;
try {
auto conn = AcquireConnection();
if (!conn) {
_logger.LogError("CreateFolder", "Failed to acquire S3 connection", __FILE__, __LINE__);
return false;
}
// Ensure folder path ends with "/"
std::string normalizedPath = prefix;
if (normalizedPath.back() != '/') {
normalizedPath += '/';
}
// Set bucket-specific endpoint
if (_bAwsPath) {
conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str());
} else {
conn->rest.put_Host(_fullAWSURL.c_str());
}
// Object path is the folder name with trailing slash
std::string objectPath = _bAwsPath ? ("/" + normalizedPath) : ("/" + bucketName + "/" + normalizedPath);
// Create a zero-byte object to represent the folder
// Use PUT with empty body
const char* responseStr = conn->rest.fullRequestNoBody("PUT", objectPath.c_str());
if (conn->rest.get_LastMethodSuccess() != true) {
_logger.LogError("ANSAWSS3::CreateFolder - Request failed",
conn->rest.lastErrorText(), __FILE__, __LINE__);
DiscardConnection(std::move(conn)); // transport failed — do not pool a suspect socket
return false;
}
int statusCode = conn->rest.get_ResponseStatusCode();
if (statusCode == 200) {
_logger.LogDebug("ANSAWSS3::CreateFolder",
"Successfully created folder: " + normalizedPath + " in bucket: " + bucketName,
__FILE__, __LINE__);
createSuccess = true;
}
else {
std::string errorMsg = "HTTP " + std::to_string(statusCode);
if (responseStr != nullptr && responseStr[0] != '\0') {
CkXml responseXml;
if (responseXml.LoadXml(responseStr)) {
const char* errorCode = responseXml.getChildContent("Code");
const char* errorMessage = responseXml.getChildContent("Message");
if (errorCode != nullptr) {
errorMsg += " - " + std::string(errorCode);
}
if (errorMessage != nullptr) {
errorMsg += ": " + std::string(errorMessage);
}
}
else {
errorMsg += " - " + std::string(responseStr);
}
}
_logger.LogError("ANSAWSS3::CreateFolder", errorMsg, __FILE__, __LINE__);
}
ReleaseConnection(std::move(conn));
}
catch (const std::exception& e) {
_logger.LogFatal("ANSAWSS3::CreateFolder",
std::string("Exception: ") + e.what(),
__FILE__, __LINE__);
}
return createSuccess;
}
bool ANSAWSS3::DeleteFolder(const std::string& bucketName, const std::string& prefix) {
// Early validation checks
if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) {
_logger.LogError("ANSAWSS3::DeleteFolder",
!_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)",
__FILE__, __LINE__);
return false;
}
if (bucketName.empty() || prefix.empty()) {
_logger.LogError("ANSAWSS3::DeleteFolder",
"Bucket name or folder path is empty",
__FILE__, __LINE__);
return false;
}
bool deleteSuccess = false;
try {
auto conn = AcquireConnection();
if (!conn) {
_logger.LogError("DeleteFolder", "Failed to acquire S3 connection", __FILE__, __LINE__);
return false;
}
// Ensure folder path ends with "/"
std::string normalizedPath = prefix;
if (normalizedPath.back() != '/') {
normalizedPath += '/';
}
// Set bucket-specific endpoint
if (_bAwsPath) {
conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str());
} else {
conn->rest.put_Host(_fullAWSURL.c_str());
}
std::string objectPath = _bAwsPath ? ("/" + normalizedPath) : ("/" + bucketName + "/" + normalizedPath);
CkStringBuilder sbResponse;
bool success = conn->rest.FullRequestNoBodySb("DELETE", objectPath.c_str(), sbResponse);
if (!success) {
_logger.LogError("ANSAWSS3::DeleteFolder",
"Failed to send DELETE request: " + std::string(conn->rest.lastErrorText()),
__FILE__, __LINE__);
}
else {
int statusCode = conn->rest.get_ResponseStatusCode();
// S3 returns 204 (No Content) for successful deletion
if (statusCode == 204 || statusCode == 200) {
_logger.LogDebug("ANSAWSS3::DeleteFolder",
"Successfully deleted folder marker: " + normalizedPath + " from bucket: " + bucketName,
__FILE__, __LINE__);
deleteSuccess = true;
}
else if (statusCode == 404) {
_logger.LogWarn("ANSAWSS3::DeleteFolder",
"Folder marker not found (already deleted?): " + normalizedPath,
__FILE__, __LINE__);
}
else {
std::string errorMsg = "HTTP " + std::to_string(statusCode);
std::string response = sbResponse.getAsString();
if (!response.empty()) {
errorMsg += " - " + response;
}
_logger.LogError("ANSAWSS3::DeleteFolder", errorMsg, __FILE__, __LINE__);
}
}
ReleaseConnection(std::move(conn));
}
catch (const std::exception& e) {
_logger.LogFatal("ANSAWSS3::DeleteFolder",
std::string("Exception: ") + e.what(),
__FILE__, __LINE__);
}
return deleteSuccess;
}
std::string ANSAWSS3::GetBucketRegion(const std::string& bucketName) {
// Early validation checks
if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) {
_logger.LogError("ANSAWSS3::GetBucketRegion",
!_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)",
__FILE__, __LINE__);
return "";
}
if (bucketName.empty()) {
_logger.LogError("ANSAWSS3::GetBucketRegion",
"Bucket name is empty",
__FILE__, __LINE__);
return "";
}
std::string region = "";
try {
auto conn = AcquireConnection();
if (!conn) {
_logger.LogError("GetBucketRegion", "Failed to acquire S3 connection", __FILE__, __LINE__);
return "";
}
// Set bucket-specific endpoint
if (_bAwsPath) {
conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str());
} else {
conn->rest.put_Host(_fullAWSURL.c_str());
}
// Send the GET request to query the bucket location
std::string locationPath = _bAwsPath ? "/?location" : ("/" + bucketName + "?location");
const char* strResult = conn->rest.fullRequestNoBody("GET", locationPath.c_str());
if (conn->rest.get_LastMethodSuccess() != true) {
_logger.LogError("ANSAWSS3::GetBucketRegion - Request failed",
conn->rest.lastErrorText(), __FILE__, __LINE__);
DiscardConnection(std::move(conn)); // transport failed — do not pool a suspect socket
return "";
}
int responseStatusCode = conn->rest.get_ResponseStatusCode();
if (responseStatusCode != 200) {
std::string errorMsg = "HTTP " + std::to_string(responseStatusCode);
if (strResult != nullptr && strResult[0] != '\0') {
errorMsg += " - " + std::string(strResult);
}
_logger.LogError("ANSAWSS3::GetBucketRegion", errorMsg, __FILE__, __LINE__);
ReleaseConnection(std::move(conn));
return "";
}
// Parse XML response
CkXml xml;
bool success = xml.LoadXml(strResult);
if (!success) {
_logger.LogError("ANSAWSS3::GetBucketRegion - Failed to parse XML",
xml.lastErrorText(), __FILE__, __LINE__);
ReleaseConnection(std::move(conn));
return "";
}
// Get the region from XML content
const char* regionContent = xml.content();
if (regionContent != nullptr && regionContent[0] != '\0') {
region = std::string(regionContent);
// AWS returns empty string for us-east-1 (classic region)
// Some implementations may return "null" or empty
if (region.empty() || region == "null") {
region = "us-east-1";
_logger.LogDebug("ANSAWSS3::GetBucketRegion",
"Bucket '" + bucketName + "' is in default region: us-east-1",
__FILE__, __LINE__);
}
else {
_logger.LogDebug("ANSAWSS3::GetBucketRegion",
"Bucket '" + bucketName + "' is in region: " + region,
__FILE__, __LINE__);
}
}
else {
// Empty response typically means us-east-1
region = "us-east-1";
_logger.LogDebug("ANSAWSS3::GetBucketRegion",
"Empty response - assuming default region: us-east-1",
__FILE__, __LINE__);
}
ReleaseConnection(std::move(conn));
}
catch (const std::exception& e) {
_logger.LogFatal("ANSAWSS3::GetBucketRegion",
std::string("Exception: ") + e.what(),
__FILE__, __LINE__);
}
return region;
}
// Uploads text data from a file to the specified S3 bucket
bool ANSAWSS3::UploadTextData(const std::string& bucketName, const std::string& textFilePath, std::string& uploadedFilePath) {
const std::string kOp = "ANSAWSS3::UploadTextData";
// _logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__);
// Early validation (permanent — do NOT retry)
if (!CheckUploadPreconditions(kOp)) return false;
if (bucketName.empty() || textFilePath.empty()) {
_logger.LogError(kOp, "Bucket name or file path is empty", __FILE__, __LINE__);
SetCallStatus(kCallPermanentError); // caller bug — retrying cannot help
return false;
}
// Filename and content type are attempt-independent.
const std::string fileName = ExtractFileName(textFilePath);
if (fileName.empty()) {
_logger.LogError(kOp,
"Failed to extract filename from path: " + textFilePath,
__FILE__, __LINE__);
SetCallStatus(kCallPermanentError); // bad input path — retrying cannot help
return false;
}
const std::string contentType = GetContentType(textFilePath);
const std::string objectPath =
_bAwsPath ? ("/" + fileName) : ("/" + bucketName + "/" + fileName);
return UploadWithRetry(kOp,
[&](std::string& lastError) -> AttemptResult {
auto conn = AcquireConnection();
if (!conn) {
lastError = "Failed to acquire S3 connection";
return AttemptResult::Transient;
}
if (_bAwsPath) {
conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str());
} else {
conn->rest.put_Host(_fullAWSURL.c_str());
}
// Read the file on each attempt (cheap; keeps the body fresh).
CkFileAccess fac;
const char* fileContents = fac.readEntireTextFile(textFilePath.c_str(), "utf-8");
if (!fac.get_LastMethodSuccess()) {
// File read failure is permanent (disk/IO, not network).
lastError = std::string("Failed to read file: ") + fac.lastErrorText();
ReleaseConnection(std::move(conn));
return AttemptResult::Permanent;
}
if (fileContents == nullptr || fileContents[0] == '\0') {
lastError = "File is empty: " + textFilePath;
ReleaseConnection(std::move(conn));
return AttemptResult::Permanent;
}
conn->rest.AddHeader("Content-Type", contentType.c_str());
conn->rest.AddHeader("Content-Encoding", "gzip");
conn->rest.AddHeader("Expect", "100-continue");
const char* responseBodyStr =
conn->rest.fullRequestString("PUT", objectPath.c_str(), fileContents);
if (!conn->rest.get_LastMethodSuccess()) {
return ClassifyFailedRequest(std::move(conn), lastError);
}
int statusCode = conn->rest.get_ResponseStatusCode();
if (statusCode == 200) {
std::string scheme = _bTls ? "https://" : "http://";
uploadedFilePath = scheme + (_bAwsPath ? _fullAWSURL : _baseDomain) + "/" + bucketName + "/" + fileName;
_logger.LogDebug(kOp,
"Successfully uploaded: " + fileName + " | URL: " + uploadedFilePath,
__FILE__, __LINE__);
ReleaseConnection(std::move(conn));
return AttemptResult::Success;
}
lastError = "HTTP " + std::to_string(statusCode);
if (responseBodyStr != nullptr && responseBodyStr[0] != '\0') {
lastError += " - " + std::string(responseBodyStr);
}
ReleaseConnection(std::move(conn));
bool retryable = (statusCode >= 500 && statusCode <= 599)
|| statusCode == 408 || statusCode == 429;
return retryable ? AttemptResult::Transient : AttemptResult::Permanent;
});
}
bool ANSAWSS3::UploadBinaryData(const std::string& bucketName, const std::string& dataFilePath, std::string& uploadedFilePath) {
const std::string kOp = "ANSAWSS3::UploadBinaryData";
// _logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__);
// Early validation (permanent — do NOT retry)
if (!CheckUploadPreconditions(kOp)) return false;
if (bucketName.empty() || dataFilePath.empty()) {
_logger.LogError(kOp, "Bucket name or file path is empty", __FILE__, __LINE__);
SetCallStatus(kCallPermanentError); // caller bug — retrying cannot help
return false;
}
const std::string fileName = ExtractFileName(dataFilePath);
if (fileName.empty()) {
_logger.LogError(kOp,
"Failed to extract filename from path: " + dataFilePath,
__FILE__, __LINE__);
SetCallStatus(kCallPermanentError); // bad input path — retrying cannot help
return false;
}
const std::string contentType = GetContentType(dataFilePath);
const std::string objectPath =
_bAwsPath ? ("/" + fileName) : ("/" + bucketName + "/" + fileName);
return UploadWithRetry(kOp,
[&](std::string& lastError) -> AttemptResult {
auto conn = AcquireConnection();
if (!conn) {
lastError = "Failed to acquire S3 connection";
return AttemptResult::Transient;
}
if (_bAwsPath) {
conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str());
} else {
conn->rest.put_Host(_fullAWSURL.c_str());
}
CkBinData binData;
if (!binData.LoadFile(dataFilePath.c_str())) {
lastError = std::string("Failed to load file: ") + binData.lastErrorText();
ReleaseConnection(std::move(conn));
return AttemptResult::Permanent; // disk/IO, not network
}
conn->rest.AddHeader("Content-Type", contentType.c_str());
conn->rest.AddHeader("Expect", "100-continue");
CkStringBuilder sbResponse;
if (!conn->rest.FullRequestBd("PUT", objectPath.c_str(), binData, sbResponse)) {
return ClassifyFailedRequest(std::move(conn), lastError);
}
int statusCode = conn->rest.get_ResponseStatusCode();
if (statusCode == 200) {
std::string scheme = _bTls ? "https://" : "http://";
uploadedFilePath = scheme + (_bAwsPath ? _fullAWSURL : _baseDomain) + "/" + bucketName + "/" + fileName;
_logger.LogDebug(kOp,
"Successfully uploaded: " + fileName + " | URL: " + uploadedFilePath,
__FILE__, __LINE__);
ReleaseConnection(std::move(conn));
return AttemptResult::Success;
}
lastError = "HTTP " + std::to_string(statusCode);
std::string body = sbResponse.getAsString();
if (!body.empty()) lastError += " - " + body;
ReleaseConnection(std::move(conn));
bool retryable = (statusCode >= 500 && statusCode <= 599)
|| statusCode == 408 || statusCode == 429;
return retryable ? AttemptResult::Transient : AttemptResult::Permanent;
});
}
bool ANSAWSS3::UploadPrefixBinaryData(const std::string& bucketName, const std::string& prefix, const std::string& dataFilePath, const std::string& objectName, std::string& uploadedFilePath) {
const std::string kOp = "ANSAWSS3::UploadPrefixBinaryData";
//_logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__);
// Early validation (permanent — do NOT retry)
if (!CheckUploadPreconditions(kOp)) return false;
if (bucketName.empty() || dataFilePath.empty()) {
_logger.LogError(kOp, "Bucket name or file path is empty", __FILE__, __LINE__);
SetCallStatus(kCallPermanentError); // caller bug — retrying cannot help
return false;
}
// Resolve filename (from objectName, fall back to path).
std::string fileName = objectName;
if (fileName.empty()) {
fileName = ExtractFileName(dataFilePath);
if (fileName.empty()) {
_logger.LogError(kOp,
"Failed to extract filename from path: " + dataFilePath,
__FILE__, __LINE__);
SetCallStatus(kCallPermanentError); // bad input path — retrying cannot help
return false;
}
}
// Build once: the object key and the HTTP request path. Keeping
// them separate avoids the path-style bucket-duplication bug in
// the returned URL.
std::string objectKey; // "<prefix>/<fileName>" — no leading '/'
if (!prefix.empty()) {
std::string normalizedPrefix = prefix;
if (!normalizedPrefix.empty() && normalizedPrefix.front() == '/') {
normalizedPrefix = normalizedPrefix.substr(1);
}
if (!normalizedPrefix.empty() && normalizedPrefix.back() != '/') {
normalizedPrefix += '/';
}
objectKey = normalizedPrefix;
}
objectKey += fileName;
const std::string objectPath = _bAwsPath
? ("/" + objectKey)
: ("/" + bucketName + "/" + objectKey);
const std::string contentType = GetContentType(dataFilePath);
return UploadWithRetry(kOp,
[&](std::string& lastError) -> AttemptResult {
auto conn = AcquireConnection();
if (!conn) {
lastError = "Failed to acquire S3 connection";
return AttemptResult::Transient;
}
if (_bAwsPath) {
conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str());
} else {
conn->rest.put_Host(_fullAWSURL.c_str());
}
CkBinData binData;
if (!binData.LoadFile(dataFilePath.c_str())) {
lastError = std::string("Failed to load file: ") + binData.lastErrorText();
ReleaseConnection(std::move(conn));
return AttemptResult::Permanent;
}
conn->rest.AddHeader("Content-Type", contentType.c_str());
conn->rest.AddHeader("Expect", "100-continue");
CkStringBuilder sbResponse;
if (!conn->rest.FullRequestBd("PUT", objectPath.c_str(), binData, sbResponse)) {
return ClassifyFailedRequest(std::move(conn), lastError);
}
int statusCode = conn->rest.get_ResponseStatusCode();
if (statusCode == 200) {
// Build canonical path-style URL from parts — independent
// of the addressing style we actually used.
std::string scheme = _bTls ? "https://" : "http://";
uploadedFilePath = scheme + (_bAwsPath ? _fullAWSURL : _baseDomain) + "/" + bucketName + "/" + objectKey;
_logger.LogDebug(kOp,
"Successfully uploaded to: " + objectKey + " | URL: " + uploadedFilePath,
__FILE__, __LINE__);
ReleaseConnection(std::move(conn));
return AttemptResult::Success;
}
lastError = "HTTP " + std::to_string(statusCode);
std::string body = sbResponse.getAsString();
if (!body.empty()) lastError += " - " + body;
ReleaseConnection(std::move(conn));
bool retryable = (statusCode >= 500 && statusCode <= 599)
|| statusCode == 408 || statusCode == 429;
return retryable ? AttemptResult::Transient : AttemptResult::Permanent;
});
}
bool ANSAWSS3::UploadFileStream(const std::string& bucketName, const std::string& dataFilePath, std::string& uploadedFilePath) {
const std::string kOp = "ANSAWSS3::UploadFileStream";
//_logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__);
// Early validation (permanent — do NOT retry)
if (!CheckUploadPreconditions(kOp)) return false;
if (bucketName.empty() || dataFilePath.empty()) {
_logger.LogError(kOp, "Bucket name or file path is empty", __FILE__, __LINE__);
SetCallStatus(kCallPermanentError); // caller bug — retrying cannot help
return false;
}
const std::string fileName = ExtractFileName(dataFilePath);
const std::string objectPath =
_bAwsPath ? ("/" + fileName) : ("/" + bucketName + "/" + fileName);
return UploadWithRetry(kOp,
[&](std::string& lastError) -> AttemptResult {
auto conn = AcquireConnection();
if (!conn) {
lastError = "Failed to acquire S3 connection";
return AttemptResult::Transient;
}
if (_bAwsPath) {
conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str());
} else {
conn->rest.put_Host(_fullAWSURL.c_str());
}
// Fresh stream per attempt (it holds OS file handle state).
CkStream fileStream;
fileStream.put_SourceFile(dataFilePath.c_str());
const char* responseStr =
conn->rest.fullRequestStream("PUT", objectPath.c_str(), fileStream);
if (conn->rest.get_LastMethodSuccess() != true) {
return ClassifyFailedRequest(std::move(conn), lastError);
}
int statusCode = conn->rest.get_ResponseStatusCode();
if (statusCode == 200) {
std::string scheme = _bTls ? "https://" : "http://";
uploadedFilePath = scheme + (_bAwsPath ? _fullAWSURL : _baseDomain) + "/" + bucketName + "/" + fileName;
_logger.LogDebug(kOp,
"Successfully uploaded file: " + fileName + " | URL: " + uploadedFilePath,
__FILE__, __LINE__);
ReleaseConnection(std::move(conn));
return AttemptResult::Success;
}
lastError = "HTTP " + std::to_string(statusCode);
if (responseStr != nullptr && responseStr[0] != '\0') {
lastError += " - " + std::string(responseStr);
}
ReleaseConnection(std::move(conn));
bool retryable = (statusCode >= 500 && statusCode <= 599)
|| statusCode == 408 || statusCode == 429;
return retryable ? AttemptResult::Transient : AttemptResult::Permanent;
});
}
bool ANSAWSS3::UploadMultipartData(const std::string& bucketName,const std::string& dataFilePath, std::string& uploadedFilePath, int partSize) {
const std::string kOp = "ANSAWSS3::UploadMultipartData";
//_logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__);
// Early validation (permanent — do NOT retry)
if (!CheckUploadPreconditions(kOp)) return false;
if (bucketName.empty() || dataFilePath.empty()) {
_logger.LogError(kOp, "Bucket name or file path is empty", __FILE__, __LINE__);
SetCallStatus(kCallPermanentError); // caller bug — retrying cannot help
return false;
}
if (partSize < 5242880) {
_logger.LogError(kOp,
"Part size must be at least 5MB (5242880 bytes)",
__FILE__, __LINE__);
SetCallStatus(kCallPermanentError); // caller bug — retrying cannot help
return false;
}
{
CkFileAccess fac;
if (!fac.FileExists(dataFilePath.c_str())) {
_logger.LogError(kOp, "File not found: " + dataFilePath, __FILE__, __LINE__);
SetCallStatus(kCallPermanentError); // bad input path — retrying cannot help
return false;
}
}
const std::string fileName = ExtractFileName(dataFilePath);
if (fileName.empty()) {
_logger.LogError(kOp,
"Failed to extract filename from path: " + dataFilePath,
__FILE__, __LINE__);
SetCallStatus(kCallPermanentError); // bad input path — retrying cannot help
return false;
}
const std::string objectPath =
_bAwsPath ? ("/" + fileName) : ("/" + bucketName + "/" + fileName);
const std::string fileFolderPath =
dataFilePath.substr(0, dataFilePath.find_last_of("/\\"));
// Fix 2: include a per-call unique token in the scratch-file name so
// two parallel multipart uploads of the same local file don't share
// the same partsList XML (which would race on create/delete).
const std::string partsListFilePath =
fileFolderPath + "/partsList_" + fileName + "_" +
MakeUniqueMultipartToken(this) + ".xml";
// Note on retry semantics for multipart:
// Each attempt performs a fresh Initiate → Parts → Complete cycle. We
// reset the on-disk partsList XML at the start of each attempt because
// ETags returned by a prior attempt are bound to that attempt's
// UploadId and are not valid for a new Initiate. Parts uploaded during
// a failed attempt therefore become orphans on S3 until the service's
// multipart-lifecycle rule cleans them up (default 7 days, configurable).
return UploadWithRetry(kOp,
[&](std::string& lastError) -> AttemptResult {
CkFileAccess fac;
// Ensure no stale parts list from a prior attempt.
if (fac.FileExists(partsListFilePath.c_str())) {
fac.FileDelete(partsListFilePath.c_str());
}
auto conn = AcquireConnection();
if (!conn) {
lastError = "Failed to acquire S3 connection";
return AttemptResult::Transient;
}
if (_bAwsPath) {
conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str());
} else {
conn->rest.put_Host(_fullAWSURL.c_str());
}
// ---- STEP 1: INITIATE ----
conn->rest.ClearAllQueryParams();
conn->rest.AddQueryParam("uploads", "");
const char* responseXml = conn->rest.fullRequestNoBody("POST", objectPath.c_str());
if (!conn->rest.get_LastMethodSuccess()) {
{ AttemptResult r = ClassifyFailedRequest(std::move(conn), lastError);
lastError = "Initiate: " + lastError; return r; }
}
int initStatus = conn->rest.get_ResponseStatusCode();
if (initStatus != 200) {
lastError = "Initiate HTTP " + std::to_string(initStatus);
if (responseXml != nullptr && responseXml[0] != '\0') lastError += " - " + std::string(responseXml);
ReleaseConnection(std::move(conn));
bool retryable = (initStatus >= 500 && initStatus <= 599) || initStatus == 408 || initStatus == 429;
return retryable ? AttemptResult::Transient : AttemptResult::Permanent;
}
CkXml xmlInit;
if (!xmlInit.LoadXml(responseXml)) {
lastError = std::string("Initiate XML parse error: ") + xmlInit.lastErrorText();
ReleaseConnection(std::move(conn));
return AttemptResult::Transient;
}
const char* uploadId = xmlInit.getChildContent("UploadId");
if (uploadId == nullptr || uploadId[0] == '\0') {
lastError = "UploadId not found in Initiate response";
ReleaseConnection(std::move(conn));
return AttemptResult::Transient;
}
_logger.LogDebug(kOp,
"Multipart initiated. UploadId: " + std::string(uploadId),
__FILE__, __LINE__);
// ---- STEP 2: UPLOAD PARTS ----
fac.OpenForRead(dataFilePath.c_str());
int numParts = fac.GetNumBlocks(partSize);
fac.FileClose();
CkXml partsListXml;
partsListXml.put_Tag("CompleteMultipartUpload");
CkStringBuilder sbPartNumber;
for (int partNumber = 1; partNumber <= numParts; ++partNumber) {
sbPartNumber.Clear();
sbPartNumber.AppendInt(partNumber);
CkStream fileStream;
fileStream.put_SourceFile(dataFilePath.c_str());
fileStream.put_SourceFilePartSize(partSize);
fileStream.put_SourceFilePart(partNumber - 1);
conn->rest.ClearAllQueryParams();
conn->rest.AddQueryParam("partNumber", sbPartNumber.getAsString());
conn->rest.AddQueryParam("uploadId", uploadId);
const char* responseStr = conn->rest.fullRequestStream("PUT", objectPath.c_str(), fileStream);
if (!conn->rest.get_LastMethodSuccess()) {
{ AttemptResult r = ClassifyFailedRequest(std::move(conn), lastError);
lastError = "Part " + std::to_string(partNumber) + ": " + lastError; return r; }
}
int partStatus = conn->rest.get_ResponseStatusCode();
if (partStatus != 200) {
lastError = "Part " + std::to_string(partNumber) + " HTTP " + std::to_string(partStatus);
if (responseStr != nullptr && responseStr[0] != '\0') lastError += " - " + std::string(responseStr);
ReleaseConnection(std::move(conn));
bool retryable = (partStatus >= 500 && partStatus <= 599) || partStatus == 408 || partStatus == 429;
return retryable ? AttemptResult::Transient : AttemptResult::Permanent;
}
const char* etag = conn->rest.responseHdrByName("ETag");
if (!conn->rest.get_LastMethodSuccess() || etag == nullptr || etag[0] == '\0') {
lastError = "Part " + std::to_string(partNumber) + ": ETag missing from response";
DiscardConnection(std::move(conn)); // transport failed — do not pool a suspect socket
return AttemptResult::Transient;
}
CkXml* xPart = partsListXml.NewChild("Part", "");
xPart->NewChildInt2("PartNumber", partNumber);
xPart->NewChild2("ETag", etag);
delete xPart;
// Persist partsList for diagnostics; not used for resume in this attempt.
partsListXml.SaveXml(partsListFilePath.c_str());
}
// ---- STEP 3: COMPLETE ----
conn->rest.ClearAllQueryParams();
conn->rest.AddQueryParam("uploadId", uploadId);
responseXml = conn->rest.fullRequestString("POST", objectPath.c_str(), partsListXml.getXml());
if (!conn->rest.get_LastMethodSuccess()) {
{ AttemptResult r = ClassifyFailedRequest(std::move(conn), lastError);
lastError = "Complete: " + lastError; return r; }
}
int completeStatus = conn->rest.get_ResponseStatusCode();
if (completeStatus != 200) {
lastError = "Complete HTTP " + std::to_string(completeStatus);
if (responseXml != nullptr && responseXml[0] != '\0') lastError += " - " + std::string(responseXml);
ReleaseConnection(std::move(conn));
bool retryable = (completeStatus >= 500 && completeStatus <= 599) || completeStatus == 408 || completeStatus == 429;
return retryable ? AttemptResult::Transient : AttemptResult::Permanent;
}
// Cleanup parts list on success.
if (fac.FileExists(partsListFilePath.c_str())) {
fac.FileDelete(partsListFilePath.c_str());
}
std::string scheme = _bTls ? "https://" : "http://";
uploadedFilePath = scheme + (_bAwsPath ? _fullAWSURL : _baseDomain) + "/" + bucketName + "/" + fileName;
_logger.LogDebug(kOp,
"Multipart upload completed for: " + fileName + " | URL: " + uploadedFilePath,
__FILE__, __LINE__);
ReleaseConnection(std::move(conn));
return AttemptResult::Success;
});
}
bool ANSAWSS3::UploadPrefixMultipartData(const std::string& bucketName, const std::string& prefix,const std::string& dataFilePath, const std::string& objectName, std::string& uploadedFilePath, int partSize) {
const std::string kOp = "ANSAWSS3::UploadPrefixMultipartData";
//_logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__);
// Early validation (permanent — do NOT retry)
if (!CheckUploadPreconditions(kOp)) return false;
if (bucketName.empty() || dataFilePath.empty()) {
_logger.LogError(kOp, "Bucket name or file path is empty", __FILE__, __LINE__);
SetCallStatus(kCallPermanentError); // caller bug — retrying cannot help
return false;
}
if (partSize < 5242880) {
_logger.LogError(kOp,
"Part size must be at least 5MB (5242880 bytes)",
__FILE__, __LINE__);
SetCallStatus(kCallPermanentError); // caller bug — retrying cannot help
return false;
}
{
CkFileAccess fac;
if (!fac.FileExists(dataFilePath.c_str())) {
_logger.LogError(kOp, "File not found: " + dataFilePath, __FILE__, __LINE__);
SetCallStatus(kCallPermanentError); // bad input path — retrying cannot help
return false;
}
}
std::string fileName = objectName;
if (fileName.empty()) {
fileName = ExtractFileName(dataFilePath);
if (fileName.empty()) {
_logger.LogError(kOp,
"Failed to extract filename from path: " + dataFilePath,
__FILE__, __LINE__);
SetCallStatus(kCallPermanentError); // bad input path — retrying cannot help
return false;
}
}
// Build once: the object key and the HTTP request path. Keeping
// them separate avoids the path-style bucket-duplication bug in
// the returned URL.
std::string objectKey; // "<prefix>/<fileName>" — no leading '/'
if (!prefix.empty()) {
std::string normalizedPrefix = prefix;
if (!normalizedPrefix.empty() && normalizedPrefix.front() == '/') {
normalizedPrefix = normalizedPrefix.substr(1);
}
if (!normalizedPrefix.empty() && normalizedPrefix.back() != '/') {
normalizedPrefix += '/';
}
objectKey = normalizedPrefix;
}
objectKey += fileName;
const std::string objectPath = _bAwsPath
? ("/" + objectKey)
: ("/" + bucketName + "/" + objectKey);
// Parts-list file uses a sanitized filename for safe local storage.
// Fix 2: also append a per-call unique token so parallel multipart
// uploads of the same local file don't share the same scratch file.
const std::string fileFolderPath =
dataFilePath.substr(0, dataFilePath.find_last_of("/\\"));
std::string safeFileName = fileName;
std::replace(safeFileName.begin(), safeFileName.end(), '/', '_');
std::replace(safeFileName.begin(), safeFileName.end(), '\\', '_');
const std::string partsListFilePath =
fileFolderPath + "/partsList_" + safeFileName + "_" +
MakeUniqueMultipartToken(this) + ".xml";
// See note in UploadMultipartData regarding multipart retry semantics.
return UploadWithRetry(kOp,
[&](std::string& lastError) -> AttemptResult {
CkFileAccess fac;
if (fac.FileExists(partsListFilePath.c_str())) {
fac.FileDelete(partsListFilePath.c_str());
}
auto conn = AcquireConnection();
if (!conn) {
lastError = "Failed to acquire S3 connection";
return AttemptResult::Transient;
}
if (_bAwsPath) {
conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str());
} else {
conn->rest.put_Host(_fullAWSURL.c_str());
}
// ---- STEP 1: INITIATE ----
conn->rest.ClearAllQueryParams();
conn->rest.AddQueryParam("uploads", "");
const char* responseXml = conn->rest.fullRequestNoBody("POST", objectPath.c_str());
if (!conn->rest.get_LastMethodSuccess()) {
{ AttemptResult r = ClassifyFailedRequest(std::move(conn), lastError);
lastError = "Initiate: " + lastError; return r; }
}
int initStatus = conn->rest.get_ResponseStatusCode();
if (initStatus != 200) {
lastError = "Initiate HTTP " + std::to_string(initStatus);
if (responseXml != nullptr && responseXml[0] != '\0') lastError += " - " + std::string(responseXml);
ReleaseConnection(std::move(conn));
bool retryable = (initStatus >= 500 && initStatus <= 599) || initStatus == 408 || initStatus == 429;
return retryable ? AttemptResult::Transient : AttemptResult::Permanent;
}
CkXml xmlInit;
if (!xmlInit.LoadXml(responseXml)) {
lastError = std::string("Initiate XML parse error: ") + xmlInit.lastErrorText();
ReleaseConnection(std::move(conn));
return AttemptResult::Transient;
}
const char* uploadId = xmlInit.getChildContent("UploadId");
if (uploadId == nullptr || uploadId[0] == '\0') {
lastError = "UploadId not found in Initiate response";
ReleaseConnection(std::move(conn));
return AttemptResult::Transient;
}
_logger.LogDebug(kOp,
"Multipart initiated. UploadId: " + std::string(uploadId),
__FILE__, __LINE__);
// ---- STEP 2: UPLOAD PARTS ----
fac.OpenForRead(dataFilePath.c_str());
int numParts = fac.GetNumBlocks(partSize);
fac.FileClose();
CkXml partsListXml;
partsListXml.put_Tag("CompleteMultipartUpload");
CkStringBuilder sbPartNumber;
for (int partNumber = 1; partNumber <= numParts; ++partNumber) {
sbPartNumber.Clear();
sbPartNumber.AppendInt(partNumber);
CkStream fileStream;
fileStream.put_SourceFile(dataFilePath.c_str());
fileStream.put_SourceFilePartSize(partSize);
fileStream.put_SourceFilePart(partNumber - 1);
conn->rest.ClearAllQueryParams();
conn->rest.AddQueryParam("partNumber", sbPartNumber.getAsString());
conn->rest.AddQueryParam("uploadId", uploadId);
const char* responseStr = conn->rest.fullRequestStream("PUT", objectPath.c_str(), fileStream);
if (!conn->rest.get_LastMethodSuccess()) {
{ AttemptResult r = ClassifyFailedRequest(std::move(conn), lastError);
lastError = "Part " + std::to_string(partNumber) + ": " + lastError; return r; }
}
int partStatus = conn->rest.get_ResponseStatusCode();
if (partStatus != 200) {
lastError = "Part " + std::to_string(partNumber) + " HTTP " + std::to_string(partStatus);
if (responseStr != nullptr && responseStr[0] != '\0') lastError += " - " + std::string(responseStr);
ReleaseConnection(std::move(conn));
bool retryable = (partStatus >= 500 && partStatus <= 599) || partStatus == 408 || partStatus == 429;
return retryable ? AttemptResult::Transient : AttemptResult::Permanent;
}
const char* etag = conn->rest.responseHdrByName("ETag");
if (!conn->rest.get_LastMethodSuccess() || etag == nullptr || etag[0] == '\0') {
lastError = "Part " + std::to_string(partNumber) + ": ETag missing from response";
DiscardConnection(std::move(conn)); // transport failed — do not pool a suspect socket
return AttemptResult::Transient;
}
CkXml* xPart = partsListXml.NewChild("Part", "");
xPart->NewChildInt2("PartNumber", partNumber);
xPart->NewChild2("ETag", etag);
delete xPart;
partsListXml.SaveXml(partsListFilePath.c_str());
}
// ---- STEP 3: COMPLETE ----
conn->rest.ClearAllQueryParams();
conn->rest.AddQueryParam("uploadId", uploadId);
responseXml = conn->rest.fullRequestString("POST", objectPath.c_str(), partsListXml.getXml());
if (!conn->rest.get_LastMethodSuccess()) {
{ AttemptResult r = ClassifyFailedRequest(std::move(conn), lastError);
lastError = "Complete: " + lastError; return r; }
}
int completeStatus = conn->rest.get_ResponseStatusCode();
if (completeStatus != 200) {
lastError = "Complete HTTP " + std::to_string(completeStatus);
if (responseXml != nullptr && responseXml[0] != '\0') lastError += " - " + std::string(responseXml);
ReleaseConnection(std::move(conn));
bool retryable = (completeStatus >= 500 && completeStatus <= 599) || completeStatus == 408 || completeStatus == 429;
return retryable ? AttemptResult::Transient : AttemptResult::Permanent;
}
if (fac.FileExists(partsListFilePath.c_str())) {
fac.FileDelete(partsListFilePath.c_str());
}
// Build canonical path-style URL from parts — independent
// of the addressing style we actually used.
std::string scheme = _bTls ? "https://" : "http://";
uploadedFilePath = scheme + (_bAwsPath ? _fullAWSURL : _baseDomain) + "/" + bucketName + "/" + objectKey;
_logger.LogDebug(kOp,
"Multipart upload completed. S3 key: " + objectKey + " | URL: " + uploadedFilePath,
__FILE__, __LINE__);
ReleaseConnection(std::move(conn));
return AttemptResult::Success;
});
}
// Upload jpeg data
bool ANSAWSS3::UploadJpegImage(const std::string& bucketName, unsigned char* jpeg_string, int32 bufferLength, const std::string& fileName, std::string& uploadedFilePath) {
const std::string kOp = "ANSAWSS3::UploadJpegImage";
// _logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__);
// Early validation checks (permanent — do NOT retry)
if (!CheckUploadPreconditions(kOp)) return false;
if (bucketName.empty() || bufferLength <= 0) {
_logger.LogError(kOp,
"Bucket name or buffer length is invalid",
__FILE__, __LINE__);
SetCallStatus(kCallPermanentError); // caller bug — retrying cannot help
return false;
}
// Object path is attempt-independent.
const std::string objectPath =
_bAwsPath ? ("/" + fileName) : ("/" + bucketName + "/" + fileName);
return UploadWithRetry(kOp,
[&](std::string& lastError) -> AttemptResult {
auto conn = AcquireConnection();
if (!conn) {
lastError = "Failed to acquire S3 connection";
return AttemptResult::Transient;
}
if (_bAwsPath) {
conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str());
} else {
conn->rest.put_Host(_fullAWSURL.c_str());
}
CkBinData binData;
CkByteData jpegBytes;
jpegBytes.append2(jpeg_string, static_cast<unsigned long>(bufferLength));
binData.AppendBinary(jpegBytes);
conn->rest.AddHeader("Content-Type", "image/jpeg");
conn->rest.AddHeader("Expect", "100-continue");
CkStringBuilder sbResponse;
if (!conn->rest.FullRequestBd("PUT", objectPath.c_str(), binData, sbResponse)) {
return ClassifyFailedRequest(std::move(conn), lastError);
}
int statusCode = conn->rest.get_ResponseStatusCode();
if (statusCode == 200) {
std::string scheme = _bTls ? "https://" : "http://";
uploadedFilePath = scheme + (_bAwsPath ? _fullAWSURL : _baseDomain) + "/" + bucketName + "/" + fileName;
_logger.LogDebug(kOp,
"Successfully uploaded: " + fileName + " | URL: " + uploadedFilePath,
__FILE__, __LINE__);
ReleaseConnection(std::move(conn));
return AttemptResult::Success;
}
lastError = "HTTP " + std::to_string(statusCode);
std::string body = sbResponse.getAsString();
if (!body.empty()) lastError += " - " + body;
ReleaseConnection(std::move(conn));
bool retryable = (statusCode >= 500 && statusCode <= 599)
|| statusCode == 408 || statusCode == 429;
return retryable ? AttemptResult::Transient : AttemptResult::Permanent;
});
}
bool ANSAWSS3::UploadPrefixJpegImage(const std::string& bucketName, const std::string& prefix,unsigned char* jpeg_string,int32 bufferLength,const std::string& fileName, std::string& uploadedFilePath) {
const std::string kOp = "ANSAWSS3::UploadPrefixJpegImage";
//_logger.LogError(kOp,"Start Uploading...",__FILE__, __LINE__);
// Early validation checks (permanent failures — do NOT retry)
if (!CheckUploadPreconditions(kOp)) return false;
if (bucketName.empty() || bufferLength <= 0 || fileName.empty()) {
_logger.LogError(kOp,
"Bucket name, buffer length, or filename is invalid",
__FILE__, __LINE__);
SetCallStatus(kCallPermanentError); // caller bug — retrying cannot help
return false;
}
// Build once: the object key (path within the bucket) and the HTTP
// request path (which depends on addressing style). Keeping these
// separate avoids the path-style bucket-duplication bug in the
// returned URL.
std::string objectKey; // "<prefix>/<fileName>" — no leading '/'
if (!prefix.empty()) {
std::string normalizedPrefix = prefix;
if (!normalizedPrefix.empty() && normalizedPrefix.front() == '/') {
normalizedPrefix = normalizedPrefix.substr(1);
}
if (!normalizedPrefix.empty() && normalizedPrefix.back() != '/') {
normalizedPrefix += '/';
}
objectKey = normalizedPrefix;
}
objectKey += fileName;
const std::string objectPath = _bAwsPath
? ("/" + objectKey)
: ("/" + bucketName + "/" + objectKey);
return UploadWithRetry(kOp,
[&](std::string& lastError) -> AttemptResult {
auto conn = AcquireConnection();
if (!conn) {
lastError = "Failed to acquire S3 connection";
return AttemptResult::Transient;
}
if (_bAwsPath) {
conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str());
} else {
conn->rest.put_Host(_fullAWSURL.c_str());
}
CkBinData binData;
CkByteData jpegBytes;
jpegBytes.append2(jpeg_string, static_cast<unsigned long>(bufferLength));
binData.AppendBinary(jpegBytes);
conn->rest.AddHeader("Content-Type", "image/jpeg");
conn->rest.AddHeader("Expect", "100-continue");
CkStringBuilder sbResponse;
if (!conn->rest.FullRequestBd("PUT", objectPath.c_str(), binData, sbResponse)) {
return ClassifyFailedRequest(std::move(conn), lastError);
}
int statusCode = conn->rest.get_ResponseStatusCode();
if (statusCode == 200) {
// Build the returned URL from parts (independent of
// addressing style) — always the canonical path-style URL.
std::string scheme = _bTls ? "https://" : "http://";
uploadedFilePath = scheme + (_bAwsPath ? _fullAWSURL : _baseDomain) + "/" + bucketName + "/" + objectKey;
_logger.LogDebug(kOp,
"Successfully uploaded: " + objectKey + " (" + std::to_string(bufferLength) + " bytes) | URL: " + uploadedFilePath,
__FILE__, __LINE__);
ReleaseConnection(std::move(conn));
return AttemptResult::Success;
}
lastError = "HTTP " + std::to_string(statusCode);
std::string body = sbResponse.getAsString();
if (!body.empty()) lastError += " - " + body;
ReleaseConnection(std::move(conn));
bool retryable = (statusCode >= 500 && statusCode <= 599)
|| statusCode == 408 || statusCode == 429;
return retryable ? AttemptResult::Transient : AttemptResult::Permanent;
});
}
// Generates a temporary pre-signed URL (AWS Signature V4) for an object
// already stored in S3. Public users can open the URL — without any AWS
// credentials — until it expires. SigV4 pre-signing is a purely local
// operation: it needs valid credentials and a known endpoint, but performs
// no network I/O, so it succeeds even when not connected.
bool ANSAWSS3::GeneratePresignedURL(const std::string& bucketName,
const std::string& prefix,
const std::string& objectName,
int validDays,
std::string& presignedURL) {
const std::string kOp = "ANSAWSS3::GeneratePresignedURL";
// Permanent failures — do NOT proceed.
if (!_isLicenseValid || !_isUnlockCodeValid) {
_logger.LogError(kOp,
!_isLicenseValid ? "Invalid license" : "Invalid unlock code",
__FILE__, __LINE__);
ANS_DBG("AWSS3", "GeneratePresignedURL: invalid license/unlock code");
return false;
}
if (bucketName.empty() || objectName.empty()) {
_logger.LogError(kOp, "Bucket name or object name is empty", __FILE__, __LINE__);
ANS_DBG("AWSS3", "GeneratePresignedURL: empty bucket/object name");
return false;
}
// AWS SigV4 caps pre-signed URL validity at 7 days.
if (validDays < 1 || validDays > 7) {
_logger.LogError(kOp,
"validDays must be between 1 and 7 (got " + std::to_string(validDays) + ")",
__FILE__, __LINE__);
ANS_DBG("AWSS3", "GeneratePresignedURL: validDays out of range (%d)", validDays);
return false;
}
// Snapshot the config/auth attributes under the config mutex.
std::string accessKey, secretKey, region, endpoint, service;
bool useHttps;
{
std::lock_guard<std::mutex> lock(_configMutex);
accessKey = _accessKey;
secretKey = _secretKey;
region = _bucketRegion;
endpoint = _fullAWSURL;
service = _serviceName;
useHttps = _bTls;
}
if (accessKey.empty() || secretKey.empty()) {
_logger.LogError(kOp,
"AWS credentials not set (call SetAuthentication first)",
__FILE__, __LINE__);
ANS_DBG("AWSS3", "GeneratePresignedURL: missing credentials");
return false;
}
if (endpoint.empty() || region.empty() || service.empty()) {
_logger.LogError(kOp,
"S3 endpoint/region/service not configured (call Connect first)",
__FILE__, __LINE__);
ANS_DBG("AWSS3", "GeneratePresignedURL: endpoint/region/service not configured");
return false;
}
// Build the object key the same way UploadPrefixJpegImage does, so the
// URL is generated for exactly the key that was uploaded.
std::string objectKey;
if (!prefix.empty()) {
std::string normalizedPrefix = prefix;
if (normalizedPrefix.front() == '/') {
normalizedPrefix = normalizedPrefix.substr(1);
}
if (!normalizedPrefix.empty() && normalizedPrefix.back() != '/') {
normalizedPrefix += '/';
}
objectKey = normalizedPrefix;
}
objectKey += objectName;
// Configure a CkHttp with the same credentials/endpoint the upload
// path uses so the signed host matches the stored object's host.
CkHttp http;
http.put_AwsAccessKey(accessKey.c_str());
http.put_AwsSecretKey(secretKey.c_str());
http.put_AwsRegion(region.c_str());
http.put_AwsEndpoint(endpoint.c_str());
const int numSecondsValid = validDays * 86400;
const char* url = http.s3_GenPresignedUrl("GET", useHttps,
bucketName.c_str(), objectKey.c_str(), numSecondsValid, service.c_str());
if (!http.get_LastMethodSuccess()) {
_logger.LogError(kOp,
std::string("Pre-signed URL generation failed: ") + http.lastErrorText(),
__FILE__, __LINE__);
ANS_DBG("AWSS3", "GeneratePresignedURL FAILED: key=%s err=%s",
objectKey.c_str(), http.lastErrorText());
return false;
}
presignedURL = (url != nullptr) ? url : "";
_logger.LogDebug(kOp,
"Generated pre-signed URL for '" + objectKey + "' valid " +
std::to_string(validDays) + " day(s): " + presignedURL,
__FILE__, __LINE__);
ANS_DBG("AWSS3", "GeneratePresignedURL OK: key=%s validDays=%d url=%s",
objectKey.c_str(), validDays, presignedURL.c_str());
return true;
}
// Downloads
bool ANSAWSS3::DownloadFile(const std::string& bucketName,
const std::string& objectName,
const std::string& saveFilePath) {
// Early validation checks
if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) {
_logger.LogError("ANSAWSS3::DownloadFile",
!_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)",
__FILE__, __LINE__);
return false;
}
if (bucketName.empty() || objectName.empty() || saveFilePath.empty()) {
_logger.LogError("ANSAWSS3::DownloadFile",
"Bucket name, object name, or save path is empty",
__FILE__, __LINE__);
return false;
}
bool downloadSuccess = false;
try {
auto conn = AcquireConnection();
if (!conn) {
_logger.LogError("DownloadFile", "Failed to acquire S3 connection", __FILE__, __LINE__);
return false;
}
// Set bucket-specific endpoint
if (_bAwsPath) {
conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str());
} else {
conn->rest.put_Host(_fullAWSURL.c_str());
}
// Normalize object name (remove leading slash if present)
std::string normalizedObjectName = objectName;
if (!normalizedObjectName.empty() && normalizedObjectName[0] == '/') {
normalizedObjectName = normalizedObjectName.substr(1);
}
std::string objectPath = _bAwsPath ? ("/" + normalizedObjectName) : ("/" + bucketName + "/" + normalizedObjectName);
_logger.LogDebug("ANSAWSS3::DownloadFile",
"Downloading: " + normalizedObjectName + " from bucket: " + bucketName,
__FILE__, __LINE__);
// Send GET request
if (!conn->rest.SendReqNoBody("GET", objectPath.c_str())) {
_logger.LogError("ANSAWSS3::DownloadFile - Request failed",
conn->rest.lastErrorText(), __FILE__, __LINE__);
DiscardConnection(std::move(conn)); // transport failed — do not pool a suspect socket
return false;
}
// Read the response header
int responseStatusCode = conn->rest.ReadResponseHeader();
if (responseStatusCode < 0) {
_logger.LogError("ANSAWSS3::DownloadFile - Failed to read response header",
conn->rest.lastErrorText(), __FILE__, __LINE__);
DiscardConnection(std::move(conn)); // transport failed — do not pool a suspect socket
return false;
}
// We expect a 200 response status if the data is coming
if (responseStatusCode == 200) {
// Determine the final file path
std::string finalFilePath;
// Check if saveFilePath is a directory or a file
CkFileAccess fac;
bool isDirectory = false;
// If path exists and is a directory, or ends with path separator
if (fac.DirExists(saveFilePath.c_str())) {
isDirectory = true;
}
else if (!saveFilePath.empty() &&
(saveFilePath.back() == '\\' || saveFilePath.back() == '/')) {
isDirectory = true;
}
if (isDirectory) {
// Save path is a directory - append object name
finalFilePath = saveFilePath;
// Ensure path ends with separator
if (!finalFilePath.empty() &&
finalFilePath.back() != '\\' &&
finalFilePath.back() != '/') {
finalFilePath += "\\";
}
// Extract just the filename from object name (in case it has path separators)
std::string filename = normalizedObjectName;
size_t lastSlash = filename.find_last_of("/\\");
if (lastSlash != std::string::npos) {
filename = filename.substr(lastSlash + 1);
}
finalFilePath += filename;
}
else {
// Save path is the complete file path
finalFilePath = saveFilePath;
}
// Ensure the directory exists
size_t lastSeparator = finalFilePath.find_last_of("/\\");
if (lastSeparator != std::string::npos) {
std::string directory = finalFilePath.substr(0, lastSeparator);
// Create directory if it doesn't exist
if (!fac.DirExists(directory.c_str())) {
if (!fac.DirCreate(directory.c_str())) {
_logger.LogError("ANSAWSS3::DownloadFile - Failed to create directory",
"Directory: " + directory + " - " + fac.lastErrorText(),
__FILE__, __LINE__);
ReleaseConnection(std::move(conn));
return false;
}
_logger.LogDebug("ANSAWSS3::DownloadFile",
"Created directory: " + directory,
__FILE__, __LINE__);
}
}
// Setup stream to write to file
CkStream bodyStream;
bodyStream.put_SinkFile(finalFilePath.c_str());
// Read the response body to the stream
if (!conn->rest.ReadRespBodyStream(bodyStream, true)) {
_logger.LogError("ANSAWSS3::DownloadFile - Failed to read response body",
conn->rest.lastErrorText(), __FILE__, __LINE__);
}
else {
_logger.LogDebug("ANSAWSS3::DownloadFile",
"Successfully downloaded: " + normalizedObjectName + " to: " + finalFilePath,
__FILE__, __LINE__);
downloadSuccess = true;
}
}
else {
// Handle non-200 response
const char* errResponse = conn->rest.readRespBodyString();
std::string errorMsg = "HTTP " + std::to_string(responseStatusCode);
if (!conn->rest.get_LastMethodSuccess()) {
errorMsg += " - Failed to read error response: " + std::string(conn->rest.lastErrorText());
}
else {
if (errResponse != nullptr && errResponse[0] != '\0') {
errorMsg += " - " + std::string(errResponse);
}
}
_logger.LogError("ANSAWSS3::DownloadFile", errorMsg, __FILE__, __LINE__);
}
ReleaseConnection(std::move(conn));
}
catch (const std::exception& e) {
_logger.LogFatal("ANSAWSS3::DownloadFile",
std::string("Exception: ") + e.what(),
__FILE__, __LINE__);
}
return downloadSuccess;
}
// Delete
bool ANSAWSS3::DeleteBucketObject(const std::string& bucketName, const std::string& objectName) {
// Early validation checks
if (!_isLicenseValid || !_isUnlockCodeValid || !_bConnected) {
_logger.LogError("ANSAWSS3::DeleteBucketObject",
!_isLicenseValid ? "Invalid license" : !_isUnlockCodeValid ? "Invalid unlock code" : _retryInProgress.load() ? "Not connected (waiting for internet, retrying in background)" : "Not connected (no internet or connection not established)",
__FILE__, __LINE__);
return false;
}
if (bucketName.empty() || objectName.empty()) {
_logger.LogError("ANSAWSS3::DeleteBucketObject",
"Bucket name or object name is empty",
__FILE__, __LINE__);
return false;
}
bool deleteSuccess = false;
try {
auto conn = AcquireConnection();
if (!conn) {
_logger.LogError("DeleteBucketObject", "Failed to acquire S3 connection", __FILE__, __LINE__);
return false;
}
// Set bucket-specific endpoint
if (_bAwsPath) {
conn->rest.put_Host((bucketName + "." + _fullAWSURL).c_str());
} else {
conn->rest.put_Host(_fullAWSURL.c_str());
}
std::string objectPath = _bAwsPath ? ("/" + objectName) : ("/" + bucketName + "/" + objectName);
CkStringBuilder sbResponse;
bool success = conn->rest.FullRequestNoBodySb("DELETE", objectPath.c_str(), sbResponse);
if (!success) {
_logger.LogError("ANSAWSS3::DeleteBucketObject",
"Failed to send DELETE request: " + std::string(conn->rest.lastErrorText()),
__FILE__, __LINE__);
}
else {
int statusCode = conn->rest.get_ResponseStatusCode();
// S3 returns 204 (No Content) for successful deletion
// Also accept 200 as success
if (statusCode == 204 || statusCode == 200) {
_logger.LogDebug("ANSAWSS3::DeleteBucketObject",
"Successfully deleted object: " + objectName + " from bucket: " + bucketName,
__FILE__, __LINE__);
deleteSuccess = true;
}
else if (statusCode == 404) {
// Object doesn't exist - you may want to treat this as success or failure
// depending on your use case
// Uncomment the next line if you want to treat "not found" as success:
deleteSuccess = true;
}
else {
std::string errorMsg = "HTTP " + std::to_string(statusCode);
std::string response = sbResponse.getAsString();
if (!response.empty()) {
errorMsg += " - " + response;
}
_logger.LogError("ANSAWSS3::DeleteBucketObject", errorMsg, __FILE__, __LINE__);
}
}
ReleaseConnection(std::move(conn));
}
catch (const std::exception& e) {
_logger.LogFatal("ANSAWSS3::DeleteBucketObject",
std::string("Exception: ") + e.what(),
__FILE__, __LINE__);
}
return deleteSuccess;
}
}