ANSCENTER/ANSCORE
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add more unicode APIs

Browse Source
This commit is contained in:
Tuan Nghia Nguyen
2026-04-07 07:59:46 +10:00
parent 6d792ee4c0
commit cb3e856a6e
5 changed files with 195 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.claude/settings.local.json
View File

@@ -80,7 +80,8 @@
"Bash(grep -rn \"ApplyTracking\\\\|_trackerEnabled\" /c/Projects/CLionProjects/ANSCORE/modules/ANSODEngine/*.cpp)",
"Bash(cmake --build build --target ANSUtilities)",
"Bash(ls -d /c/Projects/CLionProjects/ANSCORE/cmake-build-* /c/Projects/CLionProjects/ANSCORE/out/*)",
"Bash(cmake --build /c/Projects/CLionProjects/ANSCORE/cmake-build-release --target ANSUtilities)"
"Bash(cmake --build /c/Projects/CLionProjects/ANSCORE/cmake-build-release --target ANSUtilities)",
"Bash(find /c/Projects/CLionProjects/ANSCORE -name *json* -o -name *Json*)"
]
}
}

22
core/ANSLicensingSystem/Utility.cpp
View File

@@ -1,5 +1,21 @@
#include "Utility.h"
#include <ctime>
#include <mutex>
#include <map>
#include <memory>
// Per-path mutex to serialize concurrent zip operations on the same target.
// Without this, two LabVIEW threads can race: one extracting a zip while
// another truncates/writes the same file, corrupting data and crashing LabVIEW.
static std::mutex g_zipPathMapMutex;
static std::map<std::string, std::shared_ptr<std::mutex>> g_zipPathLocks;
static std::shared_ptr<std::mutex> GetZipPathLock(const std::string& path) {
std::lock_guard<std::mutex> guard(g_zipPathMapMutex);
auto& ptr = g_zipPathLocks[path];
if (!ptr) ptr = std::make_shared<std::mutex>();
return ptr;
}
template <typename T>
T get_data(const boost::property_tree::ptree& pt, const std::string& key)
@@ -436,6 +452,9 @@ bool AddFolderContentsToZip(zip* archive, const char* folderPath, const char* zi
}
bool ZipFolderWithPassword(const char* folderPath, const char* zipFilePath, const char* password) {
auto pathLock = GetZipPathLock(std::string(zipFilePath));
std::lock_guard<std::mutex> zipGuard(*pathLock);
zip* zipArchive;
zip_flags_t flags = ZIP_CREATE | ZIP_TRUNCATE;
//std::cout << "Open Zip File: " << zipFilePath << std::endl;
@@ -819,6 +838,9 @@ std::string GetDateTimeString(const std::string& format) {
//}
bool ExtractProtectedZipFile(const std::string& zipFileName, const std::string& password, const std::string& modelName, const std::string outputFolder)
{
auto pathLock = GetZipPathLock(outputFolder);
std::lock_guard<std::mutex> zipGuard(*pathLock);
int error;
if (!FileExist(zipFileName))return false;
zip_t* archive = zip_open(zipFileName.c_str(), ZIP_RDONLY, &error);

82
modules/ANSUtilities/ANSUtilities.cpp
View File

@@ -1031,6 +1031,54 @@ namespace ANSCENTER
return result;
}
std::string ANSUtilities::ConvertUTF8ToUnicodeEscapes(const std::string& utf8Str) {
if (utf8Str.empty()) return "";
std::string result;
result.reserve(utf8Str.size() * 2);
size_t i = 0;
while (i < utf8Str.size()) {
unsigned char c = static_cast<unsigned char>(utf8Str[i]);
if (c <= 0x7F) {
// ASCII byte -- pass through as-is (including \r, \n, \t, space, etc.)
result += utf8Str[i];
i++;
} else {
// Multi-byte UTF-8 sequence -- decode to Unicode codepoint
uint32_t codepoint = 0;
int seqLen = 0;
if ((c & 0xE0) == 0xC0) { codepoint = c & 0x1F; seqLen = 2; }
else if ((c & 0xF0) == 0xE0) { codepoint = c & 0x0F; seqLen = 3; }
else if ((c & 0xF8) == 0xF0) { codepoint = c & 0x07; seqLen = 4; }
else { result += utf8Str[i]; i++; continue; } // invalid lead byte
bool valid = true;
for (int j = 1; j < seqLen && i + j < utf8Str.size(); j++) {
unsigned char b = static_cast<unsigned char>(utf8Str[i + j]);
if ((b & 0xC0) != 0x80) { valid = false; break; }
codepoint = (codepoint << 6) | (b & 0x3F);
}
if (!valid || i + seqLen > utf8Str.size()) {
result += utf8Str[i]; i++; continue; // skip invalid
}
if (codepoint <= 0xFFFF) {
char buf[7];
snprintf(buf, sizeof(buf), "\\u%04x", codepoint);
result += buf;
} else {
// Surrogate pair for codepoints above U+FFFF
codepoint -= 0x10000;
uint16_t high = 0xD800 + (uint16_t)(codepoint >> 10);
uint16_t low = 0xDC00 + (uint16_t)(codepoint & 0x3FF);
char buf[14];
snprintf(buf, sizeof(buf), "\\u%04x\\u%04x", high, low);
result += buf;
}
i += seqLen;
}
}
return result;
}
std::string ANSUtilities::ConvertUnicodeEscapesToUTF8(const std::string& escapedStr) {
if (escapedStr.empty()) return "";
// First decode \uXXXX to UTF-16LE, then convert to UTF-8
@@ -1052,5 +1100,39 @@ namespace ANSCENTER
}
return ConvertUTF16LEToUTF8(utf16le.data(), (int)utf16le.size());
}
std::string ANSUtilities::DoubleEscapeUnicode(const std::string& str) {
if (str.empty()) return "";
std::string result;
result.reserve(str.size() + str.size() / 4);
for (size_t i = 0; i < str.size(); i++) {
if (str[i] == '\\' && i + 1 < str.size() && str[i + 1] == 'u') {
result += "\\\\u";
i++; // skip 'u', loop advances past '\'
} else {
result += str[i];
}
}
return result;
}
std::string ANSUtilities::ConvertUTF8ToDoubleEscapedUnicode(const std::string& utf8Str) {
return DoubleEscapeUnicode(ConvertUTF8ToUnicodeEscapes(utf8Str));
}
std::string ANSUtilities::UnescapeDoubleEscapedUnicode(const std::string& str) {
if (str.empty()) return "";
std::string result;
result.reserve(str.size());
for (size_t i = 0; i < str.size(); i++) {
if (str[i] == '\\' && i + 2 < str.size() && str[i + 1] == '\\' && str[i + 2] == 'u') {
result += "\\u";
i += 2; // skip past '\\' and 'u', loop will advance past first '\'
} else {
result += str[i];
}
}
return result;
}
}

21
modules/ANSUtilities/ANSUtilities.h
View File

@@ -112,6 +112,23 @@ namespace ANSCENTER {
// Input: ASCII string with \uXXXX escapes (e.g., "\\u6c5f\\u6771 599")
// Output: UTF-8 encoded Unicode string.
static std::string ConvertUnicodeEscapesToUTF8(const std::string& escapedStr);
// Convert a UTF-8 string to Unicode escape sequences (\uXXXX).
// ASCII chars (0x20-0x7E) are preserved, all others become \uXXXX.
// Useful for encoding Unicode text into JSON-safe ASCII.
static std::string ConvertUTF8ToUnicodeEscapes(const std::string& utf8Str);
// Double-escape \uXXXX sequences: \u1ee7 becomes \\u1ee7.
// Useful when the string will be embedded in JSON and the literal \uXXXX must survive parsing.
static std::string DoubleEscapeUnicode(const std::string& str);
// Convert UTF-8 to double-escaped Unicode: "Thủy" becomes "Th\\u1ee7y".
// Combines ConvertUTF8ToUnicodeEscapes + DoubleEscapeUnicode in one call.
static std::string ConvertUTF8ToDoubleEscapedUnicode(const std::string& utf8Str);
// Unescape double-escaped Unicode: "Th\\u1ee7y" becomes "Th\u1ee7y".
// Reverses DoubleEscapeUnicode so other languages can parse \uXXXX correctly.
static std::string UnescapeDoubleEscapedUnicode(const std::string& str);
};
// Connection bundle for pool
struct S3Connection {
@@ -259,6 +276,10 @@ extern "C" ANSULT_API int ANSDecodeJsonUnicodeToUTF16LE(const char* escapedStr,
extern "C" ANSULT_API int ANSConvertUTF16LEToUTF8(const unsigned char* utf16leBytes, int byteLen, LStrHandle result);
extern "C" ANSULT_API int ANSConvertUTF16LEToUnicodeEscapes(const unsigned char* utf16leBytes, int byteLen, LStrHandle result);
extern "C" ANSULT_API int ANSConvertUnicodeEscapesToUTF8(const char* escapedStr, LStrHandle result);
extern "C" ANSULT_API int ANSConvertUTF8ToUnicodeEscapes(const char* utf8Str, LStrHandle result);
extern "C" ANSULT_API int ANSDoubleEscapeUnicode(const char* str, LStrHandle result);
extern "C" ANSULT_API int ANSConvertUTF8ToDoubleEscapedUnicode(const char* utf8Str, LStrHandle result);
extern "C" ANSULT_API int ANSUnescapeDoubleEscapedUnicode(const char* str, LStrHandle result);
// AWS S3 class

68
modules/ANSUtilities/dllmain.cpp
View File

@@ -926,6 +926,74 @@ extern "C" ANSULT_API int ANSConvertUnicodeEscapesToUTF8(const char* escapedStr,
catch (...) { return -1; }
}
extern "C" ANSULT_API int ANSConvertUTF8ToUnicodeEscapes(const char* utf8Str, LStrHandle result) {
try {
if (!utf8Str || !result) return -1;
int len = (int)strlen(utf8Str);
if (len == 0) return 0;
std::string escaped = ANSCENTER::ANSUtilities::ConvertUTF8ToUnicodeEscapes(utf8Str);
if (escaped.empty()) return 0;
int size = static_cast<int>(escaped.size());
MgErr error = DSSetHandleSize(result, sizeof(int32) + size * sizeof(uChar));
if (error != noErr) return -2;
(*result)->cnt = size;
memcpy((*result)->str, escaped.data(), size);
return 1;
}
catch (...) { return -1; }
}
extern "C" ANSULT_API int ANSDoubleEscapeUnicode(const char* str, LStrHandle result) {
try {
if (!str || !result) return -1;
int len = (int)strlen(str);
if (len == 0) return 0;
std::string escaped = ANSCENTER::ANSUtilities::DoubleEscapeUnicode(str);
if (escaped.empty()) return 0;
int size = static_cast<int>(escaped.size());
MgErr error = DSSetHandleSize(result, sizeof(int32) + size * sizeof(uChar));
if (error != noErr) return -2;
(*result)->cnt = size;
memcpy((*result)->str, escaped.data(), size);
return 1;
}
catch (...) { return -1; }
}
extern "C" ANSULT_API int ANSConvertUTF8ToDoubleEscapedUnicode(const char* utf8Str, LStrHandle result) {
try {
if (!utf8Str || !result) return -1;
int len = (int)strlen(utf8Str);
if (len == 0) return 0;
std::string escaped = ANSCENTER::ANSUtilities::ConvertUTF8ToDoubleEscapedUnicode(utf8Str);
if (escaped.empty()) return 0;
int size = static_cast<int>(escaped.size());
MgErr error = DSSetHandleSize(result, sizeof(int32) + size * sizeof(uChar));
if (error != noErr) return -2;
(*result)->cnt = size;
memcpy((*result)->str, escaped.data(), size);
return 1;
}
catch (...) { return -1; }
}
extern "C" ANSULT_API int ANSUnescapeDoubleEscapedUnicode(const char* str, LStrHandle result) {
try {
if (!str || !result) return -1;
int len = (int)strlen(str);
if (len == 0) return 0;
std::string unescaped = ANSCENTER::ANSUtilities::UnescapeDoubleEscapedUnicode(str);
if (unescaped.empty()) return 0;
int size = static_cast<int>(unescaped.size());
MgErr error = DSSetHandleSize(result, sizeof(int32) + size * sizeof(uChar));
if (error != noErr) return -2;
(*result)->cnt = size;
memcpy((*result)->str, unescaped.data(), size);
return 1;
}
catch (...) { return -1; }
}
// AWSS3
extern "C" ANSULT_API int CreateANSAWSHandle(ANSCENTER::ANSAWSS3** Handle, const char* licenseKey) {
if (Handle == nullptr || licenseKey == nullptr) return 0;