ANSCENTER/ANSCORE
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add more unicode APIs

Browse Source
This commit is contained in:
Tuan Nghia Nguyen
2026-04-07 07:59:46 +10:00
parent 6d792ee4c0
commit cb3e856a6e
5 changed files with 195 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
modules/ANSUtilities/ANSUtilities.cpp
View File

@@ -1031,6 +1031,54 @@ namespace ANSCENTER
return result;
}
std::string ANSUtilities::ConvertUTF8ToUnicodeEscapes(const std::string& utf8Str) {
if (utf8Str.empty()) return "";
std::string result;
result.reserve(utf8Str.size() * 2);
size_t i = 0;
while (i < utf8Str.size()) {
unsigned char c = static_cast<unsigned char>(utf8Str[i]);
if (c <= 0x7F) {
// ASCII byte -- pass through as-is (including \r, \n, \t, space, etc.)
result += utf8Str[i];
i++;
} else {
// Multi-byte UTF-8 sequence -- decode to Unicode codepoint
uint32_t codepoint = 0;
int seqLen = 0;
if ((c & 0xE0) == 0xC0) { codepoint = c & 0x1F; seqLen = 2; }
else if ((c & 0xF0) == 0xE0) { codepoint = c & 0x0F; seqLen = 3; }
else if ((c & 0xF8) == 0xF0) { codepoint = c & 0x07; seqLen = 4; }
else { result += utf8Str[i]; i++; continue; } // invalid lead byte
bool valid = true;
for (int j = 1; j < seqLen && i + j < utf8Str.size(); j++) {
unsigned char b = static_cast<unsigned char>(utf8Str[i + j]);
if ((b & 0xC0) != 0x80) { valid = false; break; }
codepoint = (codepoint << 6) | (b & 0x3F);
}
if (!valid || i + seqLen > utf8Str.size()) {
result += utf8Str[i]; i++; continue; // skip invalid
}
if (codepoint <= 0xFFFF) {
char buf[7];
snprintf(buf, sizeof(buf), "\\u%04x", codepoint);
result += buf;
} else {
// Surrogate pair for codepoints above U+FFFF
codepoint -= 0x10000;
uint16_t high = 0xD800 + (uint16_t)(codepoint >> 10);
uint16_t low = 0xDC00 + (uint16_t)(codepoint & 0x3FF);
char buf[14];
snprintf(buf, sizeof(buf), "\\u%04x\\u%04x", high, low);
result += buf;
}
i += seqLen;
}
}
return result;
}
std::string ANSUtilities::ConvertUnicodeEscapesToUTF8(const std::string& escapedStr) {
if (escapedStr.empty()) return "";
// First decode \uXXXX to UTF-16LE, then convert to UTF-8
@@ -1052,5 +1100,39 @@ namespace ANSCENTER
}
return ConvertUTF16LEToUTF8(utf16le.data(), (int)utf16le.size());
}
std::string ANSUtilities::DoubleEscapeUnicode(const std::string& str) {
if (str.empty()) return "";
std::string result;
result.reserve(str.size() + str.size() / 4);
for (size_t i = 0; i < str.size(); i++) {
if (str[i] == '\\' && i + 1 < str.size() && str[i + 1] == 'u') {
result += "\\\\u";
i++; // skip 'u', loop advances past '\'
} else {
result += str[i];
}
}
return result;
}
std::string ANSUtilities::ConvertUTF8ToDoubleEscapedUnicode(const std::string& utf8Str) {
return DoubleEscapeUnicode(ConvertUTF8ToUnicodeEscapes(utf8Str));
}
std::string ANSUtilities::UnescapeDoubleEscapedUnicode(const std::string& str) {
if (str.empty()) return "";
std::string result;
result.reserve(str.size());
for (size_t i = 0; i < str.size(); i++) {
if (str[i] == '\\' && i + 2 < str.size() && str[i + 1] == '\\' && str[i + 2] == 'u') {
result += "\\u";
i += 2; // skip past '\\' and 'u', loop will advance past first '\'
} else {
result += str[i];
}
}
return result;
}
}

21
modules/ANSUtilities/ANSUtilities.h
View File

@@ -112,6 +112,23 @@ namespace ANSCENTER {
// Input: ASCII string with \uXXXX escapes (e.g., "\\u6c5f\\u6771 599")
// Output: UTF-8 encoded Unicode string.
static std::string ConvertUnicodeEscapesToUTF8(const std::string& escapedStr);
// Convert a UTF-8 string to Unicode escape sequences (\uXXXX).
// ASCII chars (0x20-0x7E) are preserved, all others become \uXXXX.
// Useful for encoding Unicode text into JSON-safe ASCII.
static std::string ConvertUTF8ToUnicodeEscapes(const std::string& utf8Str);
// Double-escape \uXXXX sequences: \u1ee7 becomes \\u1ee7.
// Useful when the string will be embedded in JSON and the literal \uXXXX must survive parsing.
static std::string DoubleEscapeUnicode(const std::string& str);
// Convert UTF-8 to double-escaped Unicode: "Thủy" becomes "Th\\u1ee7y".
// Combines ConvertUTF8ToUnicodeEscapes + DoubleEscapeUnicode in one call.
static std::string ConvertUTF8ToDoubleEscapedUnicode(const std::string& utf8Str);
// Unescape double-escaped Unicode: "Th\\u1ee7y" becomes "Th\u1ee7y".
// Reverses DoubleEscapeUnicode so other languages can parse \uXXXX correctly.
static std::string UnescapeDoubleEscapedUnicode(const std::string& str);
};
// Connection bundle for pool
struct S3Connection {
@@ -259,6 +276,10 @@ extern "C" ANSULT_API int ANSDecodeJsonUnicodeToUTF16LE(const char* escapedStr,
extern "C" ANSULT_API int ANSConvertUTF16LEToUTF8(const unsigned char* utf16leBytes, int byteLen, LStrHandle result);
extern "C" ANSULT_API int ANSConvertUTF16LEToUnicodeEscapes(const unsigned char* utf16leBytes, int byteLen, LStrHandle result);
extern "C" ANSULT_API int ANSConvertUnicodeEscapesToUTF8(const char* escapedStr, LStrHandle result);
extern "C" ANSULT_API int ANSConvertUTF8ToUnicodeEscapes(const char* utf8Str, LStrHandle result);
extern "C" ANSULT_API int ANSDoubleEscapeUnicode(const char* str, LStrHandle result);
extern "C" ANSULT_API int ANSConvertUTF8ToDoubleEscapedUnicode(const char* utf8Str, LStrHandle result);
extern "C" ANSULT_API int ANSUnescapeDoubleEscapedUnicode(const char* str, LStrHandle result);
// AWS S3 class

68
modules/ANSUtilities/dllmain.cpp
View File

@@ -926,6 +926,74 @@ extern "C" ANSULT_API int ANSConvertUnicodeEscapesToUTF8(const char* escapedStr,
catch (...) { return -1; }
}
extern "C" ANSULT_API int ANSConvertUTF8ToUnicodeEscapes(const char* utf8Str, LStrHandle result) {
try {
if (!utf8Str || !result) return -1;
int len = (int)strlen(utf8Str);
if (len == 0) return 0;
std::string escaped = ANSCENTER::ANSUtilities::ConvertUTF8ToUnicodeEscapes(utf8Str);
if (escaped.empty()) return 0;
int size = static_cast<int>(escaped.size());
MgErr error = DSSetHandleSize(result, sizeof(int32) + size * sizeof(uChar));
if (error != noErr) return -2;
(*result)->cnt = size;
memcpy((*result)->str, escaped.data(), size);
return 1;
}
catch (...) { return -1; }
}
extern "C" ANSULT_API int ANSDoubleEscapeUnicode(const char* str, LStrHandle result) {
try {
if (!str || !result) return -1;
int len = (int)strlen(str);
if (len == 0) return 0;
std::string escaped = ANSCENTER::ANSUtilities::DoubleEscapeUnicode(str);
if (escaped.empty()) return 0;
int size = static_cast<int>(escaped.size());
MgErr error = DSSetHandleSize(result, sizeof(int32) + size * sizeof(uChar));
if (error != noErr) return -2;
(*result)->cnt = size;
memcpy((*result)->str, escaped.data(), size);
return 1;
}
catch (...) { return -1; }
}
extern "C" ANSULT_API int ANSConvertUTF8ToDoubleEscapedUnicode(const char* utf8Str, LStrHandle result) {
try {
if (!utf8Str || !result) return -1;
int len = (int)strlen(utf8Str);
if (len == 0) return 0;
std::string escaped = ANSCENTER::ANSUtilities::ConvertUTF8ToDoubleEscapedUnicode(utf8Str);
if (escaped.empty()) return 0;
int size = static_cast<int>(escaped.size());
MgErr error = DSSetHandleSize(result, sizeof(int32) + size * sizeof(uChar));
if (error != noErr) return -2;
(*result)->cnt = size;
memcpy((*result)->str, escaped.data(), size);
return 1;
}
catch (...) { return -1; }
}
extern "C" ANSULT_API int ANSUnescapeDoubleEscapedUnicode(const char* str, LStrHandle result) {
try {
if (!str || !result) return -1;
int len = (int)strlen(str);
if (len == 0) return 0;
std::string unescaped = ANSCENTER::ANSUtilities::UnescapeDoubleEscapedUnicode(str);
if (unescaped.empty()) return 0;
int size = static_cast<int>(unescaped.size());
MgErr error = DSSetHandleSize(result, sizeof(int32) + size * sizeof(uChar));
if (error != noErr) return -2;
(*result)->cnt = size;
memcpy((*result)->str, unescaped.data(), size);
return 1;
}
catch (...) { return -1; }
}
// AWSS3
extern "C" ANSULT_API int CreateANSAWSHandle(ANSCENTER::ANSAWSS3** Handle, const char* licenseKey) {
if (Handle == nullptr || licenseKey == nullptr) return 0;