tomasz.sowa/pikotools
1
0
Fork 0
Code Issues 1 Pull Requests Releases Activity

add a new utf8_to_output_function() method

Browse Source 
add such a method:
template<typename OutputFunction>
bool utf8_to_output_function(const char * utf8, OutputFunction output_function, int mode = 1);
This commit is contained in:
Tomasz Sowa
2024-06-19 22:54:16 +02:00
parent f85f1dade5
commit 6165a2ece3
1 changed files with 55 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
src/utf8/utf8.h
View File

@@ -169,16 +169,6 @@ size_t int_to_wide(int c, wchar_t * res, size_t max_buf_len);
bool int_to_wide(int c, std::wstring & res);
/*
* call a convert_function for each character from an utf8 string
*
* this function takes one int parameter:
* output_function(int c)
*/
template<typename OutputFunction>
bool utf8_to_output_function(const char * utf8, size_t utf8_len, OutputFunction output_function, int mode = 1);
/*
* convert an UTF-8 string to a wide string
@@ -209,6 +199,18 @@ bool utf8_to_output_function_by_index(const Stream & stream, OutputFunction outp
template<typename StreamIteratorType, typename StreamOrStringType>
bool utf8_to_wide(StreamIteratorType & iterator_in, const StreamIteratorType & iterator_end, StreamOrStringType & out_stream, bool clear_stream = true, int mode = 1);
/*
* call a convert_function for each character from an utf8 string
*
* this function takes one int parameter:
* output_function(int c)
*/
template<typename OutputFunction>
bool utf8_to_output_function(const char * utf8, size_t utf8_len, OutputFunction output_function, int mode = 1);
template<typename OutputFunction>
bool utf8_to_output_function(const char * utf8, OutputFunction output_function, int mode = 1);
template<typename StreamIteratorType, typename OutputFunction>
bool utf8_to_output_function(StreamIteratorType & iterator_in, const StreamIteratorType & iterator_end, OutputFunction output_function, int mode = 1);
@@ -571,6 +573,46 @@ return !was_error;
template<typename OutputFunction>
bool utf8_to_output_function(const char * utf8, OutputFunction output_function, int mode)
{
int z;
size_t len;
bool correct, was_error = false;
while( *utf8 )
{
if( (unsigned char)*utf8 <= 0x7f )
{
// small optimization
len = 1;
correct = true;
z = static_cast<unsigned char>(*utf8);
}
else
{
len = utf8_to_int(utf8, z, correct); // the len will be different from zero
}
if( !correct )
{
if( mode == 1 )
output_function(0xFFFD); // U+FFFD "replacement character"
was_error = true;
}
else
{
output_function(z);
}
utf8 += len;
}
return !was_error;
}
template<typename OutputFunction>
bool wide_to_output_function(const wchar_t * str, size_t len, OutputFunction output_function, int mode)
{
@@ -976,6 +1018,9 @@ bool utf8_to_wide(StreamIteratorType & iterator_in, const StreamIteratorType & i
template<typename StreamIteratorType, typename OutputFunction>
bool utf8_to_output_function(StreamIteratorType & iterator_in, const StreamIteratorType & iterator_end, OutputFunction output_function, int mode)
{