tomasz.sowa
/
pikotools
1
0
Fork 0
Code Issues 1 Pull Requests Releases Activity
Browse Source

changed utf8 functions: PascalCase to snake_case

pull/4/head
Tomasz Sowa 1 month ago
parent
b574289054
commit
59d4c9a9c8
13 changed files with 161 additions and 163 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 6
      src/csv/csvparser.cpp
  2. 6
      src/log/filelog.cpp
  3. 6
      src/log/log.cpp
  4. 4
      src/mainoptions/mainoptionsparser.cpp
  5. 2
      src/space/space.cpp
  6. 2
      src/space/space.h
  7. 8
      src/space/spaceparser.cpp
  8. 98
      src/utf8/utf8.cpp
  9. 64
      src/utf8/utf8.h
  10. 36
      src/utf8/utf8_private.cpp
  11. 46
      src/utf8/utf8_private.h
  12. 44
      src/utf8/utf8_templates.h
  13. 2
      tests/mainoptionsparser.cpp

6
src/csv/csvparser.cpp
View File

@ -81,7 +81,7 @@ CSVParser::Status CSVParser::parse_file(const wchar_t * file_name, Space & out_s
{
std::string file_name_utf8;
WideToUTF8(file_name, file_name_utf8);
wide_to_utf8(file_name, file_name_utf8);
return parse_file(file_name_utf8.c_str(), out_space);
}
@ -295,7 +295,7 @@ bool correct;
do
{
UTF8ToInt(file, c, correct);
utf8_to_int(file, c, correct);
if( !file )
return lastc;
@ -348,7 +348,7 @@ bool correct;
do
{
size_t len = UTF8ToInt(pchar_ascii, c, correct);
size_t len = utf8_to_int(pchar_ascii, c, correct);
pchar_ascii += len;
}
while( *pchar_ascii && !correct );

6
src/log/filelog.cpp
View File

@ -74,7 +74,7 @@ void FileLog::init(const std::wstring & log_file, bool log_stdout, int log_level
this->log_stdout = log_stdout;
this->log_level = log_level;
this->save_each_line = save_each_line;
WideToUTF8(log_file, this->log_file);
wide_to_utf8(log_file, this->log_file);
}
@ -111,7 +111,7 @@ void FileLog::save_log(WTextStream * buffer)
{
if( log_stdout )
{
WideStreamToUTF8(*buffer, std::cout);
wide_stream_to_utf8(*buffer, std::cout);
}
if( !log_file.empty() )
@ -126,7 +126,7 @@ void FileLog::save_log(WTextStream * buffer)
if( file )
{
WideStreamToUTF8(*buffer, file);
wide_stream_to_utf8(*buffer, file);
file.flush();
}
}

6
src/log/log.cpp
View File

@ -136,7 +136,7 @@ Log & Log::operator<<(const char * s)
{
if( buffer && file_log && s && current_level <= file_log->get_log_level() )
{
UTF8ToWide(s, *buffer, false);
utf8_to_wide(s, *buffer, false);
}
return *this;
@ -148,7 +148,7 @@ Log & Log::operator<<(const std::string & s)
{
if( buffer && file_log && current_level <= file_log->get_log_level() )
{
UTF8ToWide(s, *buffer, false);
utf8_to_wide(s, *buffer, false);
}
return *this;
@ -160,7 +160,7 @@ Log & Log::operator<<(const std::string * s)
{
if( buffer && file_log && current_level <= file_log->get_log_level() )
{
UTF8ToWide(*s, *buffer, false);
utf8_to_wide(*s, *buffer, false);
}
return *this;

4
src/mainoptions/mainoptionsparser.cpp
View File

@ -165,7 +165,7 @@ void MainOptionsParser::convert_str(const char * src, std::wstring & dst)
{
if( should_use_utf8 )
{
UTF8ToWide(src, dst);
utf8_to_wide(src, dst);
}
else
{
@ -181,7 +181,7 @@ void MainOptionsParser::convert_str(const char * src, size_t len, std::wstring &
{
if( should_use_utf8 )
{
UTF8ToWide(src, len, dst);
utf8_to_wide(src, len, dst);
}
else
{

2
src/space/space.cpp
View File

@ -942,7 +942,7 @@ std::wstring Space::to_wstr() const
if( type == type_string )
{
UTF8ToWide(value.value_string, str);
utf8_to_wide(value.value_string, str);
return str;
}

2
src/space/space.h
View File

@ -881,7 +881,7 @@ protected:
StreamType temp_stream;
// input is wide but output is utf8
WideToUTF8(input_str, temp_stream, false);
wide_to_utf8(input_str, temp_stream, false);
copy_input_stream_to_output(temp_stream, out_str, escape);
}
}

8
src/space/spaceparser.cpp
View File

@ -145,7 +145,7 @@ SpaceParser::Status SpaceParser::ParseJSONFile(const wchar_t * file_name)
{
std::string file_name_utf8;
WideToUTF8(file_name, file_name_utf8);
wide_to_utf8(file_name, file_name_utf8);
return ParseJSONFile(file_name_utf8.c_str());
}
@ -195,7 +195,7 @@ SpaceParser::Status SpaceParser::ParseSpaceFile(const wchar_t * file_name)
{
std::string file_name_utf8;
WideToUTF8(file_name, file_name_utf8);
wide_to_utf8(file_name, file_name_utf8);
return ParseSpaceFile(file_name_utf8.c_str());
}
@ -972,7 +972,7 @@ bool correct;
do
{
UTF8ToInt(file, c, correct);
utf8_to_int(file, c, correct);
if( !file )
return lastc;
@ -1025,7 +1025,7 @@ bool correct;
do
{
size_t len = UTF8ToInt(pchar_ascii, c, correct);
size_t len = utf8_to_int(pchar_ascii, c, correct);
pchar_ascii += len;
}
while( *pchar_ascii && !correct );

98
src/utf8/utf8.cpp
View File

@ -50,7 +50,7 @@ namespace pt
/*!
returns true if 'c' is a correct unicode character
*/
bool UTF8_CheckRange(int c)
bool utf8_check_range(int c)
{
return c>=0 && c<=0x10FFFF && !(c>=0xD800 && c<=0xDFFF);
}
@ -62,7 +62,7 @@ bool UTF8_CheckRange(int c)
this method is used when reading from an utf8 string
how_many_bytes - means how many bytes from the utf8 string were read
*/
bool UTF8_CheckRange(int c, int how_many_bytes)
bool utf8_check_range(int c, int how_many_bytes)
{
if( c >= 0x0000 && c <= 0x007f && how_many_bytes == 1 )
{
@ -111,7 +111,7 @@ return false;
(returns zero only if utf8_len is zero)
even if there are errors the functions returns a different from zero value
*/
size_t UTF8ToInt(const char * utf8, size_t utf8_len, int & res, bool & correct)
size_t utf8_to_int(const char * utf8, size_t utf8_len, int & res, bool & correct)
{
size_t i, len;
@ -121,7 +121,7 @@ size_t i, len;
if( utf8_len == 0 )
return 0;
if( !private_namespace::UTF8ToInt_FirstOctet(utf8[0], len, res) )
if( !private_namespace::utf8_to_int_first_octet(utf8[0], len, res) )
return 1;
if( utf8_len < len )
@ -129,11 +129,11 @@ size_t i, len;
for(i=1 ; i<len ; ++i)
{
if( !private_namespace::UTF8ToInt_AddNextOctet(utf8[i], res) )
if( !private_namespace::utf8_to_int_add_next_octet(utf8[i], res) )
return i;
}
if( UTF8_CheckRange(res, len) )
if( utf8_check_range(res, len) )
correct = true;
return len;
@ -155,7 +155,7 @@ return len;
(returns zero only if the string has '\0' at the first character)
even if there are errors the functions returns a different from zero value
*/
size_t UTF8ToInt(const char * utf8, int & res, bool & correct)
size_t utf8_to_int(const char * utf8, int & res, bool & correct)
{
size_t i, len;
@ -165,7 +165,7 @@ size_t i, len;
if( *utf8 == 0 )
return 0;
if( !private_namespace::UTF8ToInt_FirstOctet(utf8[0], len, res) )
if( !private_namespace::utf8_to_int_first_octet(utf8[0], len, res) )
return 1;
for(i=1 ; i<len ; ++i)
@ -173,11 +173,11 @@ size_t i, len;
if( utf8[i] == 0 )
return i;
if( !private_namespace::UTF8ToInt_AddNextOctet(utf8[i], res) )
if( !private_namespace::utf8_to_int_add_next_octet(utf8[i], res) )
return i;
}
if( UTF8_CheckRange(res, len) )
if( utf8_check_range(res, len) )
correct = true;
return len;
@ -200,9 +200,9 @@ return len;
(returns zero only if utf8 is empty)
even if there are errors the functions returns a different from zero value
*/
size_t UTF8ToInt(const std::string & utf8, int & res, bool & correct)
size_t utf8_to_int(const std::string & utf8, int & res, bool & correct)
{
return UTF8ToInt(utf8.c_str(), utf8.size(), res, correct);
return utf8_to_int(utf8.c_str(), utf8.size(), res, correct);
}
@ -219,7 +219,7 @@ size_t UTF8ToInt(const std::string & utf8, int & res, bool & correct)
the function returns how many characters have been used from the input stream
*/
size_t UTF8ToInt(std::istream & utf8, int & res, bool & correct)
size_t utf8_to_int(std::istream & utf8, int & res, bool & correct)
{
size_t i, len;
unsigned char uz;
@ -232,7 +232,7 @@ unsigned char uz;
if( !utf8 )
return 0;
if( !private_namespace::UTF8ToInt_FirstOctet(uz, len, res) )
if( !private_namespace::utf8_to_int_first_octet(uz, len, res) )
return 1;
for(i=1 ; i<len ; ++i)
@ -242,11 +242,11 @@ unsigned char uz;
if( !utf8 )
return i;
if( !private_namespace::UTF8ToInt_AddNextOctet(uz, res) )
if( !private_namespace::utf8_to_int_add_next_octet(uz, res) )
return i;
}
if( UTF8_CheckRange(res, len) )
if( utf8_check_range(res, len) )
correct = true;
return len;
@ -258,7 +258,7 @@ return len;
/*
*/
static void IntToWide(int c, std::wstring & res)
static void int_to_wide(int c, std::wstring & res)
{
if( sizeof(wchar_t)==2 && c>0xffff )
{
@ -291,13 +291,13 @@ static void IntToWide(int c, std::wstring & res)
the function returns false if there were some errors when converting
*/
bool UTF8ToWide(const char * utf8, size_t utf8_len, std::wstring & res, bool clear, int mode)
bool utf8_to_wide(const char * utf8, size_t utf8_len, std::wstring & res, bool clear, int mode)
{
if( clear )
res.clear();
bool status = private_namespace::UTF8ToWideGeneric(utf8, utf8_len, mode, [&res](int c) {
IntToWide(c, res);
bool status = private_namespace::utf8_to_wide_generic(utf8, utf8_len, mode, [&res](int c) {
int_to_wide(c, res);
});
return status;
@ -321,14 +321,14 @@ bool UTF8ToWide(const char * utf8, size_t utf8_len, std::wstring & res, bool cle
the function returns false if there were some errors when converting
*/
bool UTF8ToWide(const char * utf8, std::wstring & res, bool clear, int mode)
bool utf8_to_wide(const char * utf8, std::wstring & res, bool clear, int mode)
{
size_t utf8_len = 0;
while( utf8[utf8_len] != 0 )
utf8_len += 1;
return UTF8ToWide(utf8, utf8_len, res, clear, mode);
return utf8_to_wide(utf8, utf8_len, res, clear, mode);
}
@ -347,9 +347,9 @@ return UTF8ToWide(utf8, utf8_len, res, clear, mode);
the function returns false if there were some errors when converting
*/
bool UTF8ToWide(const std::string & utf8, std::wstring & res, bool clear, int mode)
bool utf8_to_wide(const std::string & utf8, std::wstring & res, bool clear, int mode)
{
return UTF8ToWide(utf8.c_str(), utf8.size(), res, clear, mode);
return utf8_to_wide(utf8.c_str(), utf8.size(), res, clear, mode);
}
@ -368,7 +368,7 @@ bool UTF8ToWide(const std::string & utf8, std::wstring & res, bool clear, int mo
the function returns false if there were some errors when converting
*/
bool UTF8ToWide(std::istream & utf8, std::wstring & res, bool clear, int mode)
bool utf8_to_wide(std::istream & utf8, std::wstring & res, bool clear, int mode)
{
int z;
bool correct, was_error = false;
@ -376,7 +376,7 @@ bool correct, was_error = false;
if( clear )
res.clear();
while( UTF8ToInt(utf8, z, correct) > 0 )
while( utf8_to_int(utf8, z, correct) > 0 )
{
if( !correct )
{
@ -387,7 +387,7 @@ bool correct, was_error = false;
}
else
{
IntToWide(z, res);
int_to_wide(z, res);
}
}
@ -410,13 +410,13 @@ return !was_error;
the function returns how many characters have been written to the utf8,
zero means the utf8 buffer is too small or 'z' is an incorrect unicode character
*/
size_t IntToUTF8(int z, char * utf8, size_t utf8_max_len)
size_t int_to_utf8(int z, char * utf8, size_t utf8_max_len)
{
char buf[10];
int i = 0;
int mask = 0x3f; // 6 first bits set
if( utf8_max_len==0 || !UTF8_CheckRange(z) )
if( utf8_max_len==0 || !utf8_check_range(z) )
return 0;
if( z <= 0x7f )
@ -464,14 +464,14 @@ return a;
the function returns how many characters have been written to the utf8 string,
zero means that 'z' is an incorrect unicode character
*/
size_t IntToUTF8(int z, std::string & utf8, bool clear)
size_t int_to_utf8(int z, std::string & utf8, bool clear)
{
char buf[10];
if( clear )
utf8.clear();
size_t len = IntToUTF8(z, buf, sizeof(buf)/sizeof(char));
size_t len = int_to_utf8(z, buf, sizeof(buf)/sizeof(char));
size_t i;
for(i=0 ; i<len ; ++i)
@ -497,7 +497,7 @@ return len;
this function returns false if there were some errors when converting
*/
bool WideToUTF8(const wchar_t * wide_string, size_t string_len, std::string & utf8, bool clear, int mode)
bool wide_to_utf8(const wchar_t * wide_string, size_t string_len, std::string & utf8, bool clear, int mode)
{
bool was_error = false;
size_t chars;
@ -507,7 +507,7 @@ size_t chars;
while( string_len > 0 )
{
chars = private_namespace::WideOneToUTF8(wide_string, string_len, utf8, was_error, mode);
chars = private_namespace::wide_one_to_utf8(wide_string, string_len, utf8, was_error, mode);
wide_string += chars;
string_len -= chars;
}
@ -531,7 +531,7 @@ return !was_error;
this function returns false if there were some errors when converting
*/
bool WideToUTF8(const wchar_t * wide_string, std::string & utf8, bool clear, int mode)
bool wide_to_utf8(const wchar_t * wide_string, std::string & utf8, bool clear, int mode)
{
bool was_error = false;
@ -539,7 +539,7 @@ bool was_error = false;
utf8.clear();
while( *wide_string )
wide_string += private_namespace::WideOneToUTF8(wide_string, utf8, was_error, mode);
wide_string += private_namespace::wide_one_to_utf8(wide_string, utf8, was_error, mode);
return !was_error;
}
@ -560,9 +560,9 @@ return !was_error;
this function returns false if there were some errors when converting
*/
bool WideToUTF8(const std::wstring & wide_string, std::string & utf8, bool clear, int mode)
bool wide_to_utf8(const std::wstring & wide_string, std::string & utf8, bool clear, int mode)
{
return WideToUTF8(wide_string.c_str(), wide_string.size(), utf8, clear, mode);
return wide_to_utf8(wide_string.c_str(), wide_string.size(), utf8, clear, mode);
}
@ -590,7 +590,7 @@ bool WideToUTF8(const std::wstring & wide_string, std::string & utf8, bool clear
if there is an error when converting (there is an incorrect character in the wide string) the function
will continue converting but if the buffer is too small the function breaks immediately
*/
bool WideToUTF8(const wchar_t * wide_string, size_t string_len, char * utf8, size_t utf8_len, size_t & utf8_written, int mode)
bool wide_to_utf8(const wchar_t * wide_string, size_t string_len, char * utf8, size_t utf8_len, size_t & utf8_written, int mode)
{
bool was_error = false;
bool was_buffer_to_small;
@ -600,7 +600,7 @@ size_t chars, utf8_saved;
while( string_len > 0 )
{
chars = private_namespace::WideOneToUTF8(wide_string, string_len, utf8, utf8_len, utf8_saved, was_buffer_to_small, was_error, mode);
chars = private_namespace::wide_one_to_utf8(wide_string, string_len, utf8, utf8_len, utf8_saved, was_buffer_to_small, was_error, mode);
if( was_buffer_to_small )
{
@ -644,9 +644,9 @@ return !was_error;
if there is an error when converting (there is an incorrect character in the wide string) the function
will continue converting but if the buffer is too small the function breaks immediately
*/
bool WideToUTF8(const std::wstring & wide_string, char * utf8, size_t utf8_len, size_t & utf8_written, int mode)
bool wide_to_utf8(const std::wstring & wide_string, char * utf8, size_t utf8_len, size_t & utf8_written, int mode)
{
return WideToUTF8(wide_string.c_str(), wide_string.size(), utf8, utf8_len, utf8_written, mode);
return wide_to_utf8(wide_string.c_str(), wide_string.size(), utf8, utf8_len, utf8_written, mode);
}
@ -672,7 +672,7 @@ bool WideToUTF8(const std::wstring & wide_string, char * utf8, size_t utf8_len,
will continue converting but if the buffer is too small the function breaks immediately
(in both cases the utf8 buffer is null terminated)
*/
bool WideToUTF8(const wchar_t * wide_string, size_t string_len, char * utf8, size_t utf8_len, int mode)
bool wide_to_utf8(const wchar_t * wide_string, size_t string_len, char * utf8, size_t utf8_len, int mode)
{
size_t utf8_saved;
bool res;
@ -680,7 +680,7 @@ bool res;
if( utf8_len == 0 )
return false;
res = WideToUTF8(wide_string, string_len, utf8, utf8_len - 1, utf8_saved, mode);
res = wide_to_utf8(wide_string, string_len, utf8, utf8_len - 1, utf8_saved, mode);
utf8[utf8_saved] = 0;
return res;
@ -708,9 +708,9 @@ return res;
will continue converting but if the buffer is too small the function breaks immediately
(in both cases the utf8 buffer is null terminated)
*/
bool WideToUTF8(const std::wstring & wide_string, char * utf8, size_t utf8_len, int mode)
bool wide_to_utf8(const std::wstring & wide_string, char * utf8, size_t utf8_len, int mode)
{
return WideToUTF8(wide_string.c_str(), wide_string.size(), utf8, utf8_len, mode);
return wide_to_utf8(wide_string.c_str(), wide_string.size(), utf8, utf8_len, mode);
}
@ -735,7 +735,7 @@ bool WideToUTF8(const std::wstring & wide_string, char * utf8, size_t utf8_len,
if there is an error when converting (there is an incorrect character in the wide string) the function
will continue converting but if the buffer is too small the function breaks immediately
*/
bool WideToUTF8(const wchar_t * wide_string, char * utf8, size_t utf8_len, size_t & utf8_written, int mode)
bool wide_to_utf8(const wchar_t * wide_string, char * utf8, size_t utf8_len, size_t & utf8_written, int mode)
{
bool was_error = false;
bool was_buffer_to_small;
@ -747,7 +747,7 @@ size_t len;
while( *wide_string )
{
len = (*(wide_string+1) == 0) ? 1 : 2;
chars = private_namespace::WideOneToUTF8(wide_string, len, utf8, utf8_len, utf8_saved, was_buffer_to_small, was_error, mode);
chars = private_namespace::wide_one_to_utf8(wide_string, len, utf8, utf8_len, utf8_saved, was_buffer_to_small, was_error, mode);
if( was_buffer_to_small )
{
@ -790,7 +790,7 @@ return !was_error;
will continue converting but if the buffer is too small the function breaks immediately
(in both cases the utf8 buffer is null terminated)
*/
bool WideToUTF8(const wchar_t * wide_string, char * utf8, size_t utf8_len, int mode)
bool wide_to_utf8(const wchar_t * wide_string, char * utf8, size_t utf8_len, int mode)
{
size_t utf8_saved;
bool res;
@ -798,7 +798,7 @@ bool res;
if( utf8_len == 0 )
return false;
res = WideToUTF8(wide_string, utf8, utf8_len - 1, utf8_saved, mode);
res = wide_to_utf8(wide_string, utf8, utf8_len - 1, utf8_saved, mode);
utf8[utf8_saved] = 0;
return res;

64
src/utf8/utf8.h
View File

@ -61,7 +61,7 @@ namespace pt
/*!
returns true if 'c' is a correct unicode character
*/
bool UTF8_CheckRange(int c);
bool utf8_check_range(int c);
/*!
@ -70,7 +70,7 @@ bool UTF8_CheckRange(int c);
this method is used when reading from an utf8 string
how_many_chars - means how many characters from utf8 string were read
*/
bool UTF8_CheckRange(int c, int how_many_bytes);
bool utf8_check_range(int c, int how_many_bytes);
/*
@ -86,31 +86,31 @@ bool UTF8_CheckRange(int c, int how_many_bytes);
/*!
converting one character from UTF-8 to an int
*/
size_t UTF8ToInt(const char * utf8, size_t utf8_len, int & res, bool & correct);
size_t UTF8ToInt(const char * utf8, int & res, bool & correct);
size_t UTF8ToInt(const std::string & utf8, int & res, bool & correct);
size_t UTF8ToInt(std::istream & utf8, int & res, bool & correct);
size_t utf8_to_int(const char * utf8, size_t utf8_len, int & res, bool & correct);
size_t utf8_to_int(const char * utf8, int & res, bool & correct);
size_t utf8_to_int(const std::string & utf8, int & res, bool & correct);
size_t utf8_to_int(std::istream & utf8, int & res, bool & correct);
/*!
converting UTF-8 string to a wide string
*/
bool UTF8ToWide(const char * utf8, size_t utf8_len, std::wstring & res, bool clear = true, int mode = 1);
bool UTF8ToWide(const char * utf8, std::wstring & res, bool clear = true, int mode = 1);
bool UTF8ToWide(const std::string & utf8, std::wstring & res, bool clear = true, int mode = 1);
bool UTF8ToWide(std::istream & utf8, std::wstring & res, bool clear = true, int mode = 1);
bool utf8_to_wide(const char * utf8, size_t utf8_len, std::wstring & res, bool clear = true, int mode = 1);
bool utf8_to_wide(const char * utf8, std::wstring & res, bool clear = true, int mode = 1);
bool utf8_to_wide(const std::string & utf8, std::wstring & res, bool clear = true, int mode = 1);
bool utf8_to_wide(std::istream & utf8, std::wstring & res, bool clear = true, int mode = 1);
template<typename StreamType>
bool UTF8ToWide(const char * utf8, size_t utf8_len, StreamType & res, bool clear = true, int mode = 1); // need to be tested
bool utf8_to_wide(const char * utf8, size_t utf8_len, StreamType & res, bool clear = true, int mode = 1); // need to be tested
template<typename StreamType>
bool UTF8ToWide(const char * utf8, StreamType & res, bool clear = true, int mode = 1); // need to be tested
bool utf8_to_wide(const char * utf8, StreamType & res, bool clear = true, int mode = 1); // need to be tested
template<typename StreamType>
bool UTF8ToWide(const std::string & utf8, StreamType & res, bool clear = true, int mode = 1); // need to be tested
bool utf8_to_wide(const std::string & utf8, StreamType & res, bool clear = true, int mode = 1); // need to be tested
template<typename StreamType>
bool UTF8ToWide(std::istream & utf8, StreamType & res, bool clear = true, int mode = 1); // need to be tested
bool utf8_to_wide(std::istream & utf8, StreamType & res, bool clear = true, int mode = 1); // need to be tested
@ -128,47 +128,45 @@ bool UTF8ToWide(std::istream & utf8, StreamType & res, bool clear = true, int mo
/*!
converting one int character to UTF-8
*/
size_t IntToUTF8(int z, char * utf8, size_t utf8_max_len);
size_t IntToUTF8(int z, std::string & utf8, bool clear = true);
size_t int_to_utf8(int z, char * utf8, size_t utf8_max_len);
size_t int_to_utf8(int z, std::string & utf8, bool clear = true);
template<typename StreamType>
size_t IntToUTF8(int z, StreamType & utf8);
size_t int_to_utf8(int z, StreamType & utf8);
/*!
converting a wide string to UTF-8 string
*/
bool WideToUTF8(const wchar_t * wide_string, size_t string_len, std::string & utf8, bool clear = true, int mode = 1);
bool WideToUTF8(const wchar_t * wide_string, std::string & utf8, bool clear = true, int mode = 1);
bool WideToUTF8(const std::wstring & wide_string, std::string & utf8, bool clear = true, int mode = 1);
bool wide_to_utf8(const wchar_t * wide_string, size_t string_len, std::string & utf8, bool clear = true, int mode = 1);
bool wide_to_utf8(const wchar_t * wide_string, std::string & utf8, bool clear = true, int mode = 1);
bool wide_to_utf8(const std::wstring & wide_string, std::string & utf8, bool clear = true, int mode = 1);
template<typename StreamType>
bool WideToUTF8(const wchar_t * wide_string, size_t string_len, StreamType & utf8, int mode = 1);
bool wide_to_utf8(const wchar_t * wide_string, size_t string_len, StreamType & utf8, int mode = 1);
template<typename StreamType>
bool WideToUTF8(const wchar_t * wide_string, StreamType & utf8, int mode = 1);
bool wide_to_utf8(const wchar_t * wide_string, StreamType & utf8, int mode = 1);
template<typename StreamType>
bool WideToUTF8(const std::wstring & wide_string, StreamType & utf8, int mode = 1);
bool wide_to_utf8(const std::wstring & wide_string, StreamType & utf8, int mode = 1);
bool WideToUTF8(const wchar_t * wide_string, size_t string_len, char * utf8, size_t utf8_len, size_t & utf8_written, int mode = 1);
bool WideToUTF8(const wchar_t * wide_string, char * utf8, size_t utf8_len, size_t & utf8_written, int mode = 1);
bool WideToUTF8(const std::wstring & wide_string, char * utf8, size_t utf8_len, size_t & utf8_written, int mode = 1);
// implement template<typename StreamType>
bool wide_to_utf8(const wchar_t * wide_string, size_t string_len, char * utf8, size_t utf8_len, size_t & utf8_written, int mode = 1);
bool wide_to_utf8(const wchar_t * wide_string, char * utf8, size_t utf8_len, size_t & utf8_written, int mode = 1);
bool wide_to_utf8(const std::wstring & wide_string, char * utf8, size_t utf8_len, size_t & utf8_written, int mode = 1);
bool WideToUTF8(const wchar_t * wide_string, size_t string_len, char * utf8, size_t utf8_len, int mode = 1);
bool WideToUTF8(const wchar_t * wide_string, char * utf8, size_t utf8_len, int mode = 1);
bool WideToUTF8(const std::wstring & wide_string, char * utf8, size_t utf8_len, int mode = 1);
// implement template<typename StreamType>
bool wide_to_utf8(const wchar_t * wide_string, size_t string_len, char * utf8, size_t utf8_len, int mode = 1);
bool wide_to_utf8(const wchar_t * wide_string, char * utf8, size_t utf8_len, int mode = 1);
bool wide_to_utf8(const std::wstring & wide_string, char * utf8, size_t utf8_len, int mode = 1);
template<typename StreamType>
void WideStreamToUTF8(StreamType & buffer, std::string & utf8, bool clear = true, int mode = 1); // not tested
void wide_stream_to_utf8(StreamType & buffer, std::string & utf8, bool clear = true, int mode = 1); // not tested
template<typename StreamTypeIn, typename StreamTypeOut>
void WideStreamToUTF8(StreamTypeIn & buffer, StreamTypeOut & utf8, int mode = 1); // not tested
void wide_stream_to_utf8(StreamTypeIn & buffer, StreamTypeOut & utf8, int mode = 1); // not tested

36
src/utf8/utf8_private.cpp
View File

@ -47,7 +47,7 @@ namespace private_namespace
/*!
an auxiliary function for converting from UTF-8 string
*/
bool UTF8ToInt_FirstOctet(unsigned char uz, size_t & len, int & res)
bool utf8_to_int_first_octet(unsigned char uz, size_t & len, int & res)
{
for(len=0 ; (uz & 0x80) != 0 ; ++len)
uz <<= 1;
@ -71,7 +71,7 @@ return true;
/*!
an auxiliary function for converting from UTF-8 string
*/
bool UTF8ToInt_AddNextOctet(unsigned char uz, int & res)
bool utf8_to_int_add_next_octet(unsigned char uz, int & res)
{
if( (uz & 0xc0) != 0x80 )
return false;
@ -93,7 +93,7 @@ return true;
returns how many wide characters were used
if string_len is greater than 0 then the return value is always greater than zero too
*/
size_t WideToInt(const wchar_t * wide_string, size_t string_len, int & z, bool & correct)
size_t wide_to_int(const wchar_t * wide_string, size_t string_len, int & z, bool & correct)
{
if( string_len == 0 )
{
@ -130,7 +130,7 @@ size_t WideToInt(const wchar_t * wide_string, size_t string_len, int & z, bool &
}
else
{
correct = UTF8_CheckRange(z);
correct = utf8_check_range(z);
return 1;
}
}
@ -144,7 +144,7 @@ size_t WideToInt(const wchar_t * wide_string, size_t string_len, int & z, bool &
returns how many wide characters were used
if wide_string has at least one character then the return value is always greater than zero too
*/
size_t WideToInt(const wchar_t * wide_string, int & z, bool & correct)
size_t wide_to_int(const wchar_t * wide_string, int & z, bool & correct)
{
size_t min_str_len = 1;
@ -158,7 +158,7 @@ size_t min_str_len = 1;
if( *(wide_string+1) != 0 )
min_str_len = 2;
return WideToInt(wide_string, min_str_len, z, correct);
return wide_to_int(wide_string, min_str_len, z, correct);
}
@ -177,7 +177,7 @@ return WideToInt(wide_string, min_str_len, z, correct);
was_error - will be true if there is an error when converting (there was an incorrect wide character)
(was_error will not be true if the utf8 buffer is too small)
*/
size_t WideOneToUTF8(const wchar_t * wide_string, size_t string_len, char * utf8, size_t utf8_len,
size_t wide_one_to_utf8(const wchar_t * wide_string, size_t string_len, char * utf8, size_t utf8_len,
size_t & utf8_written, bool & was_utf8_buf_too_small, bool & was_error, int mode)
{
int z;
@ -186,11 +186,11 @@ size_t chars;
utf8_written = 0;
was_utf8_buf_too_small = false;
chars = WideToInt(wide_string, string_len, z, correct);
chars = wide_to_int(wide_string, string_len, z, correct);
if( correct )
{
utf8_written = IntToUTF8(z, utf8, utf8_len);
utf8_written = int_to_utf8(z, utf8, utf8_len);
if( utf8_written == 0 )
was_utf8_buf_too_small = true;
@ -199,7 +199,7 @@ size_t chars;
{
if( mode == 1 )
{
utf8_written = IntToUTF8(0xFFFD, utf8, utf8_len); // U+FFFD "replacement character"
utf8_written = int_to_utf8(0xFFFD, utf8, utf8_len); // U+FFFD "replacement character"
if( utf8_written == 0 )
was_utf8_buf_too_small = true;
@ -219,21 +219,21 @@ return chars;
returns how many wide characters were used
if string_len is greater than 0 then the return value is always greater than zero too
*/
size_t WideOneToUTF8(const wchar_t * wide_string, size_t string_len, std::string & utf8, bool & was_error, int mode)
size_t wide_one_to_utf8(const wchar_t * wide_string, size_t string_len, std::string & utf8, bool & was_error, int mode)
{
int z;
bool correct;
size_t chars;
chars = WideToInt(wide_string, string_len, z, correct);
chars = wide_to_int(wide_string, string_len, z, correct);
if( correct )
correct = IntToUTF8(z, utf8, false) != 0;
correct = int_to_utf8(z, utf8, false) != 0;
if( !correct )
{
if( mode == 1 )
IntToUTF8(0xFFFD, utf8, false); // U+FFFD "replacement character"
int_to_utf8(0xFFFD, utf8, false); // U+FFFD "replacement character"
was_error = true;
}
@ -249,21 +249,21 @@ return chars;
returns how many wide characters were used
if wide_string has at least one character then the return value is always greater than zero too
*/
size_t WideOneToUTF8(const wchar_t * wide_string, std::string & utf8, bool & was_error, int mode)
size_t wide_one_to_utf8(const wchar_t * wide_string, std::string & utf8, bool & was_error, int mode)
{
int z;
bool correct;
size_t chars;
chars = WideToInt(wide_string, z, correct);
chars = wide_to_int(wide_string, z, correct);
if( correct )
correct = IntToUTF8(z, utf8, false) != 0;
correct = int_to_utf8(z, utf8, false) != 0;
if( !correct )
{
if( mode == 1 )
IntToUTF8(0xFFFD, utf8, false); // U+FFFD "replacement character"
int_to_utf8(0xFFFD, utf8, false); // U+FFFD "replacement character"
was_error = true;
}

46
src/utf8/utf8_private.h
View File

@ -44,26 +44,26 @@
namespace pt
{
bool UTF8_CheckRange(int c);
size_t IntToUTF8(int z, char * utf8, size_t utf8_max_len);
size_t IntToUTF8(int z, std::string & utf8, bool clear);
size_t UTF8ToInt(const char * utf8, size_t utf8_len, int & res, bool & correct);
bool utf8_check_range(int c);
size_t int_to_utf8(int z, char * utf8, size_t utf8_max_len);
size_t int_to_utf8(int z, std::string & utf8, bool clear);
size_t utf8_to_int(const char * utf8, size_t utf8_len, int & res, bool & correct);
namespace private_namespace
{
bool UTF8ToInt_FirstOctet(unsigned char uz, size_t & len, int & res);
bool UTF8ToInt_AddNextOctet(unsigned char uz, int & res);
bool utf8_to_int_first_octet(unsigned char uz, size_t & len, int & res);
bool utf8_to_int_add_next_octet(unsigned char uz, int & res);
size_t WideToInt(const wchar_t * wide_string, size_t string_len, int & z, bool & correct);
size_t WideToInt(const wchar_t * wide_string, int & z, bool & correct);
size_t wide_to_int(const wchar_t * wide_string, size_t string_len, int & z, bool & correct);
size_t wide_to_int(const wchar_t * wide_string, int & z, bool & correct);
size_t WideOneToUTF8(const wchar_t * wide_string, size_t string_len, char * utf8, size_t utf8_len,
size_t wide_one_to_utf8(const wchar_t * wide_string, size_t string_len, char * utf8, size_t utf8_len,
size_t & utf8_written, bool & was_utf8_buf_too_small, bool & was_error, int mode);
size_t WideOneToUTF8(const wchar_t * wide_string, size_t string_len, std::string & utf8, bool & was_error, int mode);
size_t wide_one_to_utf8(const wchar_t * wide_string, size_t string_len, std::string & utf8, bool & was_error, int mode);
size_t WideOneToUTF8(const wchar_t * wide_string, std::string & utf8, bool & was_error, int mode);
size_t wide_one_to_utf8(const wchar_t * wide_string, std::string & utf8, bool & was_error, int mode);
/*!
@ -73,21 +73,21 @@ size_t WideOneToUTF8(const wchar_t * wide_string, std::string & utf8, bool & was
if string_len is greater than 0 then the return value is always greater than zero too
*/
template<typename StreamType>
static size_t WideOneToUTF8(const wchar_t * wide_string, size_t string_len, StreamType & utf8, bool & was_error, int mode)
static size_t wide_one_to_utf8(const wchar_t * wide_string, size_t string_len, StreamType & utf8, bool & was_error, int mode)
{
int z;
bool correct;
size_t chars;
chars = WideToInt(wide_string, string_len, z, correct);
chars = wide_to_int(wide_string, string_len, z, correct);
if( correct )
correct = IntToUTF8(z, utf8) != 0;
correct = int_to_utf8(z, utf8) != 0;
if( !correct )
{
if( mode == 1 )
IntToUTF8(0xFFFD, utf8); // U+FFFD "replacement character"
int_to_utf8(0xFFFD, utf8); // U+FFFD "replacement character"
was_error = true;
}
@ -100,7 +100,7 @@ return chars;
an auxiliary function for converting from wide characters to UTF-8
*/
template<typename StreamType>
static size_t WideOneToUTF8(const wchar_t * wide_string, StreamType & utf8, bool & was_error, int mode)
static size_t wide_one_to_utf8(const wchar_t * wide_string, StreamType & utf8, bool & was_error, int mode)
{
size_t min_str_len = 1;
@ -110,18 +110,18 @@ static size_t WideOneToUTF8(const wchar_t * wide_string, StreamType & utf8, bool
if( *(wide_string+1) != 0 )
min_str_len = 2;
return WideOneToUTF8(wide_string, min_str_len, utf8, was_error, mode);
return wide_one_to_utf8(wide_string, min_str_len, utf8, was_error, mode);
}
// declared in utf8.h, defined in utf8.cpp
size_t UTF8ToInt(const char * utf8, size_t utf8_len, int & res, bool & correct);
size_t utf8_to_int(const char * utf8, size_t utf8_len, int & res, bool & correct);
template<typename function_type>
bool UTF8ToWideGeneric(const char * utf8, size_t utf8_len, int mode, function_type convert_function)
bool utf8_to_wide_generic(const char * utf8, size_t utf8_len, int mode, function_type convert_function)
{
int z;
size_t len;
@ -138,7 +138,7 @@ bool correct, was_error = false;
}
else
{
len = pt::UTF8ToInt(utf8, utf8_len, z, correct); // the len will be different from zero
len = pt::utf8_to_int(utf8, utf8_len, z, correct); // the len will be different from zero
}
if( !correct )
@ -163,7 +163,7 @@ return !was_error;
template<typename StreamType>
void IntToWide(int c, StreamType & res)
void int_to_wide(int c, StreamType & res)
{
if( sizeof(wchar_t)==2 && c>0xffff )
{
@ -183,7 +183,7 @@ void IntToWide(int c, StreamType & res)
// FIX ME it is not using surrogate pairs from input stream
// and mode parameter
template<typename char_type, size_t stack_size, size_t heap_block_size, typename function_type>
void WideToUTF8Generic(TextStreamBase<char_type, stack_size, heap_block_size> & buffer, int mode, function_type write_function)
void wide_to_utf8_generic(TextStreamBase<char_type, stack_size, heap_block_size> & buffer, int mode, function_type write_function)
{
char utf8_buffer[256];
std::size_t buffer_len = sizeof(utf8_buffer) / sizeof(char);
@ -200,7 +200,7 @@ void WideToUTF8Generic(TextStreamBase<char_type, stack_size, heap_block_size> &
index = 0;
}
index += IntToUTF8(*i, utf8_buffer + index, buffer_len - index);
index += int_to_utf8(*i, utf8_buffer + index, buffer_len - index);
++i;
}

44
src/utf8/utf8_templates.h
View File

@ -53,13 +53,13 @@ namespace pt
*/
// need to be tested
template<typename StreamType>
bool UTF8ToWide(const char * utf8, size_t utf8_len, StreamType & res, bool clear, int mode)
bool utf8_to_wide(const char * utf8, size_t utf8_len, StreamType & res, bool clear, int mode)
{
if( clear )
res.clear();
bool status = private_namespace::UTF8ToWideGeneric(utf8, utf8_len, mode, [&res](int c) {
private_namespace::IntToWide(c, res);
bool status = private_namespace::utf8_to_wide_generic(utf8, utf8_len, mode, [&res](int c) {
private_namespace::int_to_wide(c, res);
});
return status;
@ -69,29 +69,29 @@ bool UTF8ToWide(const char * utf8, size_t utf8_len, StreamType & res, bool clear
template<typename StreamType>
bool UTF8ToWide(const char * utf8, StreamType & res, bool clear, int mode)
bool utf8_to_wide(const char * utf8, StreamType & res, bool clear, int mode)
{
size_t utf8_len = 0;
while( utf8[utf8_len] != 0 )
utf8_len += 1;
return UTF8ToWide(utf8, utf8_len, res, clear, mode);
return utf8_to_wide(utf8, utf8_len, res, clear, mode);
}
template<typename StreamType>
bool UTF8ToWide(const std::string & utf8, StreamType & res, bool clear, int mode)
bool utf8_to_wide(const std::string & utf8, StreamType & res, bool clear, int mode)
{
return UTF8ToWide(utf8.c_str(), utf8.size(), res, clear, mode);
return utf8_to_wide(utf8.c_str(), utf8.size(), res, clear, mode);
}
// need to be tested
template<typename StreamType>
bool UTF8ToWide(std::istream & utf8, StreamType & res, bool clear, int mode)
bool utf8_to_wide(std::istream & utf8, StreamType & res, bool clear, int mode)
{
int z;
bool correct, was_error = false;
@ -99,7 +99,7 @@ bool correct, was_error = false;
if( clear )
res.clear();
while( UTF8ToInt(utf8, z, correct) > 0 )
while( utf8_to_int(utf8, z, correct) > 0 )
{
if( !correct )
{
@ -110,7 +110,7 @@ bool correct, was_error = false;
}
else
{
private_namespace::IntToWide(z, res);
private_namespace::int_to_wide(z, res);
}
}
@ -137,11 +137,11 @@ return !was_error;
zero means that 'z' is an incorrect unicode character
*/
template<typename StreamType>
size_t IntToUTF8(int z, StreamType & utf8)
size_t int_to_utf8(int z, StreamType & utf8)
{
char buf[10];
size_t len = IntToUTF8(z, buf, sizeof(buf)/sizeof(char));
size_t len = int_to_utf8(z, buf, sizeof(buf)/sizeof(char));
if( len > 0 )
utf8.write(buf, len);
@ -169,14 +169,14 @@ size_t IntToUTF8(int z, StreamType & utf8)
this function returns false if there were some errors when converting
*/
template<typename StreamType>
bool WideToUTF8(const wchar_t * wide_string, size_t string_len, StreamType & utf8, int mode)
bool wide_to_utf8(const wchar_t * wide_string, size_t string_len, StreamType & utf8, int mode)
{
bool was_error = false;
size_t chars;
while( string_len > 0 )
{
chars = private_namespace::WideOneToUTF8(wide_string, string_len, utf8, was_error, mode);
chars = private_namespace::wide_one_to_utf8(wide_string, string_len, utf8, was_error, mode);
wide_string += chars;
string_len -= chars;
}
@ -203,12 +203,12 @@ return !was_error;
this function returns false if there were some errors when converting
*/
template<typename StreamType>
bool WideToUTF8(const wchar_t * wide_string, StreamType & utf8, int mode)
bool wide_to_utf8(const wchar_t * wide_string, StreamType & utf8, int mode)
{
bool was_error = false;
while( *wide_string )
wide_string += private_namespace::WideOneToUTF8(wide_string, utf8, was_error, mode);
wide_string += private_namespace::wide_one_to_utf8(wide_string, utf8, was_error, mode);
return !was_error;
}
@ -230,21 +230,21 @@ return !was_error;
this function returns false if there were some errors when converting
*/
template<typename StreamType>
bool WideToUTF8(const std::wstring & wide_string, StreamType & utf8, int mode)
bool wide_to_utf8(const std::wstring & wide_string, StreamType & utf8, int mode)
{
return WideToUTF8(wide_string.c_str(), wide_string.size(), utf8, mode);
return wide_to_utf8(wide_string.c_str(), wide_string.size(), utf8, mode);
}
template<typename StreamType>
void WideStreamToUTF8(StreamType & buffer, std::string & utf8, bool clear, int mode)
void wide_stream_to_utf8(StreamType & buffer, std::string & utf8, bool clear, int mode)
{
if( clear )
utf8.clear();
private_namespace::WideToUTF8Generic(buffer, mode, [&utf8](const char * utf8_buffer, std::size_t buffer_len){
private_namespace::wide_to_utf8_generic(buffer, mode, [&utf8](const char * utf8_buffer, std::size_t buffer_len){
utf8.append(utf8_buffer, buffer_len);
});
}
@ -252,9 +252,9 @@ void WideStreamToUTF8(StreamType & buffer, std::string & utf8, bool clear, int m
// not tested
template<typename StreamTypeIn, typename StreamTypeOut>
void WideStreamToUTF8(StreamTypeIn & buffer, StreamTypeOut & utf8, int mode)
void wide_stream_to_utf8(StreamTypeIn & buffer, StreamTypeOut & utf8, int mode)
{
private_namespace::WideToUTF8Generic(buffer, mode, [&utf8](const char * utf8_buffer, std::size_t buffer_len){
private_namespace::wide_to_utf8_generic(buffer, mode, [&utf8](const char * utf8_buffer, std::size_t buffer_len){
utf8.write(utf8_buffer, buffer_len);
});
}

2
tests/mainoptionsparser.cpp
View File

@ -134,7 +134,7 @@ void test_mainoptionsparser(size_t len, const char ** argv, const Space & argume
std::wstring & err_wstr = parser.get_wrong_option();
std::string err_str;
WideToUTF8(err_wstr, err_str);
wide_to_utf8(err_wstr, err_str);
std::string json;
space.serialize_to_json_to(json);

Write Preview
Loading…
Cancel
Save