tomasz.sowa/pikotools
1
0
Fork 0
Code Issues 1 Pull Requests Releases Activity

fixed: using size() when serializing strings - this allows to serialize a string which contain a null character

Browse Source 
fixed: printing null character in space format: \u0000 (before was \0 which is not correct in json)
fixed: in serialize_string_buffer(const char * input_str, ...) a temporary fixed was used when copying input string
added support for surrogate pairs when reading \uHHHH format
added support to parse \u{H...} format (only if parsing Space format)
This commit is contained in:
Tomasz Sowa
2021-06-14 13:48:32 +02:00
parent 49c2b478c0
commit 4d70ae9e87
9 changed files with 262 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/space/space.cpp
View File

@@ -953,7 +953,7 @@ std::string Space::to_str() const
if( type == type_wstring )
{
TextStream stream;
serialize_string_buffer(value.value_wstring.c_str(), stream, Escape::no_escape);
serialize_string_buffer(value.value_wstring.c_str(), value.value_wstring.size(), stream, Escape::no_escape);
stream.to_string(str);
return str;
}

89
src/space/space.h
View File

@@ -768,8 +768,18 @@ protected:
template<typename StreamType>
void escape_to_space_format(int c, StreamType & out) const
{
// IMPLEMENT ME
escape_to_json_format(c, out);
switch(c)
{
case 0: out << '\\'; out << 'u' << '{' << '0' << '}'; break;
case '\r': out << '\\'; out << 'r'; break; // 13
case '\n': out << '\\'; out << 'n'; break; // 10
case '\\': out << '\\'; out << '\\'; break;
case '"': out << '\\'; out << '\"'; break;
case '\b': out << '\\'; out << 'b'; break; // 8
case '\f': out << '\\'; out << 'f'; break; // 12
default:
out << static_cast<typename StreamType::char_type>(c);
}
}
@@ -778,14 +788,14 @@ protected:
{
switch(c)
{
case 0: out << '\\'; out << '0'; break;
case '\r': out << '\\'; out << 'r'; break;
case '\n': out << '\\'; out << 'n'; break;
case '\\': out << '\\'; out << '\\'; break;
case '"': out << '\\'; out << '\"'; break;
//case '(': out << '\\'; out << '('; break;
//case ')': out << '\\'; out << ')'; break;
//case '=': out << '\\'; out << '='; break;
case 0: out << '\\'; out << 'u' << '0' << '0' << '0' << '0'; break;
case '\r': out << '\\'; out << 'r'; break; // 13
case '\n': out << '\\'; out << 'n'; break; // 10
case '\\': out << '\\'; out << '\\'; break;
case '"': out << '\\'; out << '\"'; break;
case '\t': out << '\\'; out << 't'; break; // 9
case '\b': out << '\\'; out << 'b'; break; // 8
case '\f': out << '\\'; out << 'f'; break; // 12
default:
out << static_cast<typename StreamType::char_type>(c);
}
@@ -811,6 +821,22 @@ protected:
}
}
template<typename CharT, typename StreamType>
void copy_input_string_to_output(const CharT * input_str, size_t len, StreamType & out_str, Escape escape) const
{
for(size_t i=0 ; i < len ; ++i)
{
if( escape == Escape::no_escape )
out_str << static_cast<typename StreamType::char_type>(input_str[i]);
else
if( escape == Escape::escape_space )
escape_to_space_format(input_str[i], out_str);
else
if( escape == Escape::escape_json )
escape_to_json_format(input_str[i], out_str);
}
}
template<typename StreamType>
void copy_input_stream_to_output(const StreamType & input_str, StreamType & out_str, Escape escape) const
{
@@ -833,23 +859,37 @@ protected:
template<typename StreamType>
void serialize_string_buffer(const char * input_str, StreamType & out_str, Escape escape) const
void serialize_string_buffer(const char * input_str, size_t len, StreamType & out_str, Escape escape) const
{
if constexpr ( sizeof(char) == sizeof(typename StreamType::char_type) )
{
// input and output are char (we assume it is utf8)
copy_input_string_to_output(input_str, out_str, escape);
copy_input_string_to_output(input_str, len, out_str, escape);
}
else
{
// input is utf8 but output is wide
copy_input_string_to_output(input_str, out_str, escape); // temporarily
StreamType temp_stream;
utf8_to_wide(input_str, len, temp_stream, false);
copy_input_stream_to_output(temp_stream, out_str, escape);
}
}
// !!!!!!!!!!!!!!!!!!! FIXME
// StreamType temp_stream;
// UTF8ToWide(input_str, temp_stream, false);
//
// copy_input_stream_to_output(temp_stream, out_str, escape);
template<typename StreamType>
void serialize_string_buffer(const wchar_t * input_str, size_t len, StreamType & out_str, Escape escape) const
{
if constexpr ( sizeof(wchar_t) == sizeof(typename StreamType::char_type) )
{
// input and output are wide characters
copy_input_string_to_output(input_str, len, out_str, escape);
}
else
{
// input is wide but output is utf8
StreamType temp_stream;
wide_to_utf8(input_str, len, temp_stream, false);
copy_input_stream_to_output(temp_stream, out_str, escape);
}
}
@@ -864,17 +904,14 @@ protected:
}
else
{
StreamType temp_stream;
// input is wide but output is utf8
StreamType temp_stream;
wide_to_utf8(input_str, temp_stream, false);
copy_input_stream_to_output(temp_stream, out_str, escape);
}
}
template<typename StreamType>
void serialize_space_null(StreamType & str) const
{
@@ -937,7 +974,7 @@ protected:
void serialize_space_string(StreamType & str) const
{
str << '"';
serialize_string_buffer(value.value_string.c_str(), str, Escape::escape_space);
serialize_string_buffer(value.value_string.c_str(), value.value_string.size(), str, Escape::escape_space);
str << '"';
}
@@ -945,7 +982,7 @@ protected:
void serialize_space_wstring(StreamType & str) const
{
str << '"';
serialize_string_buffer(value.value_wstring.c_str(), str, Escape::escape_space);
serialize_string_buffer(value.value_wstring.c_str(), value.value_wstring.size(), str, Escape::escape_space);
str << '"';
}
@@ -1121,7 +1158,7 @@ protected:
void serialize_json_string(StreamType & str) const
{
str << '"';
serialize_string_buffer(value.value_string.c_str(), str, Escape::escape_json);
serialize_string_buffer(value.value_string.c_str(), value.value_string.size(), str, Escape::escape_json);
str << '"';
}
@@ -1129,7 +1166,7 @@ protected:
void serialize_json_wstring(StreamType & str) const
{
str << '"';
serialize_string_buffer(value.value_wstring.c_str(), str, Escape::escape_json);
serialize_string_buffer(value.value_wstring.c_str(), value.value_wstring.size(), str, Escape::escape_json);
str << '"';
}

121
src/space/spaceparser.cpp
View File

@@ -1037,25 +1037,130 @@ return 0;
}
void SpaceParser::read_unicode_code_point()
/*
* format: \uHHHH where H is a hex digit 0-F
*/
bool SpaceParser::read_unicode_four_digit_format(bool has_first_byte, int first_byte)
{
wchar_t c;
int c;
int value = 0;
for(int i=0 ; i<4 ; ++i)
{
c = read_char_no_escape();
if( i == 0 && has_first_byte )
{
c = first_byte;
}
else
{
c = read_char_no_escape();
}
if( !is_hex_digit(c) )
{
status = syntax_error;
return;
return false;
}
value = (value << 4) | hex_to_int(c);
}
lastc = (wchar_t)value;
lastc = static_cast<wchar_t>(value);
return true;
}
/*
* format: \uHHHH and optionally following by \uHHHH
*
*/
void SpaceParser::read_unicode_json_format(bool has_first_byte, int first_byte)
{
bool ok = read_unicode_four_digit_format(has_first_byte, first_byte);
if( ok && pt::is_first_surrogate_char(lastc) )
{
int c1 = lastc;
int c = read_char_no_escape();
ok = ok && (c == '\\');
if( ok )
{
c = read_char_no_escape();
ok = ok && (c == 'u');
ok = ok && read_unicode_four_digit_format(false, 0);
if( ok && pt::is_second_surrogate_char(lastc) )
{
int c2 = lastc;
ok = ok && pt::surrogate_pair_to_int(c1, c2, lastc);
}
}
}
if( !ok || !pt::utf8_check_range(lastc) )
{
lastc = 0xFFFD; // U+FFFD "replacement character";
}
}
/*
* format: \u{H...} where H is a hex digit 0-F, minimum digits: 1, maximum digits: 6
*/
void SpaceParser::read_unicode_floating_format()
{
int c;
int value = 0;
int i;
// max 6 hex digits + '}'
for(i=0 ; i<7 ; ++i)
{
c = read_char_no_escape();
if( !is_hex_digit(c) )
{
break;
}
value = (value << 4) | hex_to_int(c);
}
if( i > 0 && c == '}' && pt::utf8_check_range(value) )
{
lastc = static_cast<wchar_t>(value);
}
else
{
lastc = 0xFFFD; // U+FFFD "replacement character";
}
}
void SpaceParser::read_unicode_code_point()
{
if( parsing_space )
{
int c = read_char_no_escape();
if( c == '{' )
{
read_unicode_floating_format();
}
else
{
read_unicode_json_format(true, c);
}
}
else
{
read_unicode_json_format(false, 0);
}
}
@@ -1087,6 +1192,10 @@ return lastc;
} // namespace

4
src/space/spaceparser.h
View File

@@ -300,6 +300,10 @@ private:
void trim_last_white(std::wstring & s);
bool is_hex_digit(wchar_t c);
int hex_to_int(wchar_t c);
bool read_unicode_four_digit_format(bool has_first_byte, int first_byte);
void read_unicode_json_format(bool has_first_byte, int first_byte);
void read_unicode_floating_format();
void read_unicode_code_point();
};