improve escaping characters from multibyte strings

Read the whole character from a multibyte string and then escape it.

while here:
- add more virtual methods for escaping strings to BaseExpression
This commit is contained in:
Tomasz Sowa 2024-06-19 23:02:37 +02:00
parent e46c090b41
commit 8e757aff71
Signed by: tomasz.sowa
GPG Key ID: 662CC1438638588B
7 changed files with 158 additions and 64 deletions

View File

@ -3,8 +3,9 @@
./main.o: ../../pikotools/src/mainoptions/mainoptionsparser.h
./main.o: ../../pikotools/src/space/space.h
./main.o: ../../pikotools/src/convert/inttostr.h
./main.o: ../../pikotools/src/utf8/utf8.h
./main.o: ../../pikotools/src/textstream/stream.h sample01.h basesample.h
./main.o: ../../pikotools/src/convert/misc.h
./main.o: ../../pikotools/src/textstream/stream.h
./main.o: ../../pikotools/src/utf8/utf8.h sample01.h basesample.h
./main.o: ../src/morm.h ../src/version.h ../src/morm_types.h ../src/model.h
./main.o: ../../pikotools/src/textstream/textstream.h
./main.o: ../../pikotools/src/textstream/stream.h

View File

@ -8,6 +8,7 @@
./baseexpression.o: ../../pikotools/src/textstream/textstream.h
./baseexpression.o: ../../pikotools/src/textstream/stream.h
./baseexpression.o: ../../pikotools/src/space/space.h
./baseexpression.o: ../../pikotools/src/convert/misc.h
./baseexpression.o: ../../pikotools/src/utf8/utf8.h
./baseexpression.o: ../../pikotools/src/membuffer/membuffer.h
./baseexpression.o: ../../pikotools/src/textstream/types.h
@ -20,8 +21,9 @@
./clearer.o: clearer.h ../../pikotools/src/date/date.h
./clearer.o: ../../pikotools/src/convert/inttostr.h
./clearer.o: ../../pikotools/src/space/space.h
./clearer.o: ../../pikotools/src/utf8/utf8.h
./clearer.o: ../../pikotools/src/textstream/stream.h ft.h model.h
./clearer.o: ../../pikotools/src/convert/misc.h
./clearer.o: ../../pikotools/src/textstream/stream.h
./clearer.o: ../../pikotools/src/utf8/utf8.h ft.h model.h
./clearer.o: ../../pikotools/src/textstream/textstream.h
./clearer.o: ../../pikotools/src/textstream/stream.h
./clearer.o: ../../pikotools/src/membuffer/membuffer.h
@ -36,8 +38,9 @@
./dbconnector.o: ../../pikotools/src/space/spaceparser.h
./dbconnector.o: ../../pikotools/src/space/space.h
./dbconnector.o: ../../pikotools/src/convert/inttostr.h
./dbconnector.o: ../../pikotools/src/utf8/utf8.h
./dbconnector.o: ../../pikotools/src/convert/misc.h
./dbconnector.o: ../../pikotools/src/textstream/stream.h
./dbconnector.o: ../../pikotools/src/utf8/utf8.h
./dbconnector.o: ../../pikotools/src/convert/baseparser.h
./dbconnector.o: ../../pikotools/src/textstream/textstream.h
./dbconnector.o: ../../pikotools/src/textstream/stream.h
@ -69,6 +72,7 @@
./dbexpression.o: ../../pikotools/src/textstream/textstream.h
./dbexpression.o: ../../pikotools/src/textstream/stream.h
./dbexpression.o: ../../pikotools/src/space/space.h
./dbexpression.o: ../../pikotools/src/convert/misc.h
./dbexpression.o: ../../pikotools/src/utf8/utf8.h
./dbexpression.o: ../../pikotools/src/membuffer/membuffer.h
./dbexpression.o: ../../pikotools/src/textstream/types.h
@ -87,6 +91,7 @@
./flatconnector.o: ../../pikotools/src/textstream/textstream.h
./flatconnector.o: ../../pikotools/src/textstream/stream.h
./flatconnector.o: ../../pikotools/src/space/space.h
./flatconnector.o: ../../pikotools/src/convert/misc.h
./flatconnector.o: ../../pikotools/src/utf8/utf8.h
./flatconnector.o: ../../pikotools/src/membuffer/membuffer.h
./flatconnector.o: ../../pikotools/src/textstream/types.h
@ -105,6 +110,7 @@
./flatexpression.o: ../../pikotools/src/textstream/textstream.h
./flatexpression.o: ../../pikotools/src/textstream/stream.h
./flatexpression.o: ../../pikotools/src/space/space.h
./flatexpression.o: ../../pikotools/src/convert/misc.h
./flatexpression.o: ../../pikotools/src/utf8/utf8.h
./flatexpression.o: ../../pikotools/src/membuffer/membuffer.h
./flatexpression.o: ../../pikotools/src/textstream/types.h
@ -122,6 +128,7 @@
./jsonconnector.o: ../../pikotools/src/textstream/textstream.h
./jsonconnector.o: ../../pikotools/src/textstream/stream.h
./jsonconnector.o: ../../pikotools/src/space/space.h
./jsonconnector.o: ../../pikotools/src/convert/misc.h
./jsonconnector.o: ../../pikotools/src/utf8/utf8.h
./jsonconnector.o: ../../pikotools/src/membuffer/membuffer.h
./jsonconnector.o: ../../pikotools/src/textstream/types.h
@ -138,6 +145,7 @@
./jsonexpression.o: ../../pikotools/src/textstream/textstream.h
./jsonexpression.o: ../../pikotools/src/textstream/stream.h
./jsonexpression.o: ../../pikotools/src/space/space.h
./jsonexpression.o: ../../pikotools/src/convert/misc.h
./jsonexpression.o: ../../pikotools/src/utf8/utf8.h
./jsonexpression.o: ../../pikotools/src/membuffer/membuffer.h
./jsonexpression.o: ../../pikotools/src/textstream/types.h
@ -145,14 +153,13 @@
./jsonexpression.o: ../../pikotools/src/log/filelog.h finderhelper.h
./jsonexpression.o: fieldvaluehelper.h wrapper.h spacewrapper.h
./jsonexpression.o: baseobjectwrapper.h modelcontainerwrapper.h select.h ft.h
./jsonexpression.o: ../../pikotools/src/convert/misc.h
./model.o: model.h ../../pikotools/src/textstream/textstream.h
./model.o: ../../pikotools/src/textstream/stream.h
./model.o: ../../pikotools/src/space/space.h
./model.o: ../../pikotools/src/convert/inttostr.h
./model.o: ../../pikotools/src/utf8/utf8.h
./model.o: ../../pikotools/src/convert/misc.h
./model.o: ../../pikotools/src/textstream/stream.h
./model.o: ../../pikotools/src/date/date.h
./model.o: ../../pikotools/src/utf8/utf8.h ../../pikotools/src/date/date.h
./model.o: ../../pikotools/src/membuffer/membuffer.h
./model.o: ../../pikotools/src/textstream/types.h
./model.o: ../../pikotools/src/textstream/stream_private.h modelconnector.h
@ -167,9 +174,10 @@
./modelconnector.o: ../../pikotools/src/date/date.h
./modelconnector.o: ../../pikotools/src/convert/inttostr.h
./modelconnector.o: ../../pikotools/src/space/space.h
./modelconnector.o: ../../pikotools/src/utf8/utf8.h
./modelconnector.o: ../../pikotools/src/textstream/stream.h ft.h
./modelconnector.o: dbconnector.h ../../pikotools/src/log/log.h
./modelconnector.o: ../../pikotools/src/convert/misc.h
./modelconnector.o: ../../pikotools/src/textstream/stream.h
./modelconnector.o: ../../pikotools/src/utf8/utf8.h ft.h dbconnector.h
./modelconnector.o: ../../pikotools/src/log/log.h
./modelconnector.o: ../../pikotools/src/textstream/textstream.h
./modelconnector.o: ../../pikotools/src/textstream/stream.h
./modelconnector.o: ../../pikotools/src/membuffer/membuffer.h
@ -187,6 +195,7 @@
./postgresqlexpression.o: ../../pikotools/src/textstream/textstream.h
./postgresqlexpression.o: ../../pikotools/src/textstream/stream.h
./postgresqlexpression.o: ../../pikotools/src/space/space.h
./postgresqlexpression.o: ../../pikotools/src/convert/misc.h
./postgresqlexpression.o: ../../pikotools/src/utf8/utf8.h
./postgresqlexpression.o: ../../pikotools/src/membuffer/membuffer.h
./postgresqlexpression.o: ../../pikotools/src/textstream/types.h
@ -201,8 +210,9 @@
./postgresqlqueryresult.o: ../../pikotools/src/textstream/stream.h
./postgresqlqueryresult.o: ../../pikotools/src/space/space.h
./postgresqlqueryresult.o: ../../pikotools/src/convert/inttostr.h
./postgresqlqueryresult.o: ../../pikotools/src/utf8/utf8.h
./postgresqlqueryresult.o: ../../pikotools/src/convert/misc.h
./postgresqlqueryresult.o: ../../pikotools/src/textstream/stream.h
./postgresqlqueryresult.o: ../../pikotools/src/utf8/utf8.h
./postgresqlqueryresult.o: ../../pikotools/src/date/date.h
./postgresqlqueryresult.o: ../../pikotools/src/membuffer/membuffer.h
./postgresqlqueryresult.o: ../../pikotools/src/textstream/types.h
@ -213,8 +223,9 @@
./queryresult.o: ../../pikotools/src/textstream/stream.h
./queryresult.o: ../../pikotools/src/space/space.h
./queryresult.o: ../../pikotools/src/convert/inttostr.h
./queryresult.o: ../../pikotools/src/utf8/utf8.h
./queryresult.o: ../../pikotools/src/convert/misc.h
./queryresult.o: ../../pikotools/src/textstream/stream.h
./queryresult.o: ../../pikotools/src/utf8/utf8.h
./queryresult.o: ../../pikotools/src/date/date.h
./queryresult.o: ../../pikotools/src/membuffer/membuffer.h
./queryresult.o: ../../pikotools/src/textstream/types.h
@ -229,6 +240,7 @@
./csvexpression.o: ../../pikotools/src/textstream/textstream.h
./csvexpression.o: ../../pikotools/src/textstream/stream.h
./csvexpression.o: ../../pikotools/src/space/space.h
./csvexpression.o: ../../pikotools/src/convert/misc.h
./csvexpression.o: ../../pikotools/src/utf8/utf8.h
./csvexpression.o: ../../pikotools/src/membuffer/membuffer.h
./csvexpression.o: ../../pikotools/src/textstream/types.h
@ -236,7 +248,6 @@
./csvexpression.o: ../../pikotools/src/log/filelog.h finderhelper.h
./csvexpression.o: fieldvaluehelper.h wrapper.h spacewrapper.h
./csvexpression.o: baseobjectwrapper.h modelcontainerwrapper.h select.h ft.h
./csvexpression.o: ../../pikotools/src/convert/misc.h
./xmlexpression.o: xmlexpression.h flatexpression.h baseexpression.h
./xmlexpression.o: ../../pikotools/src/textstream/stream.h
./xmlexpression.o: ../../pikotools/src/date/date.h
@ -246,6 +257,7 @@
./xmlexpression.o: ../../pikotools/src/textstream/textstream.h
./xmlexpression.o: ../../pikotools/src/textstream/stream.h
./xmlexpression.o: ../../pikotools/src/space/space.h
./xmlexpression.o: ../../pikotools/src/convert/misc.h
./xmlexpression.o: ../../pikotools/src/utf8/utf8.h
./xmlexpression.o: ../../pikotools/src/membuffer/membuffer.h
./xmlexpression.o: ../../pikotools/src/textstream/types.h
@ -253,7 +265,6 @@
./xmlexpression.o: ../../pikotools/src/log/filelog.h finderhelper.h
./xmlexpression.o: fieldvaluehelper.h wrapper.h spacewrapper.h
./xmlexpression.o: baseobjectwrapper.h modelcontainerwrapper.h select.h ft.h
./xmlexpression.o: ../../pikotools/src/convert/misc.h
./postgresqlconnector.o: postgresqlconnector.h dbconnector.h
./postgresqlconnector.o: ../../pikotools/src/textstream/stream.h
./postgresqlconnector.o: ../../pikotools/src/log/log.h
@ -261,6 +272,7 @@
./postgresqlconnector.o: ../../pikotools/src/textstream/stream.h
./postgresqlconnector.o: ../../pikotools/src/space/space.h
./postgresqlconnector.o: ../../pikotools/src/convert/inttostr.h
./postgresqlconnector.o: ../../pikotools/src/convert/misc.h
./postgresqlconnector.o: ../../pikotools/src/utf8/utf8.h
./postgresqlconnector.o: ../../pikotools/src/date/date.h
./postgresqlconnector.o: ../../pikotools/src/membuffer/membuffer.h
@ -277,8 +289,9 @@
./transaction.o: ../../pikotools/src/textstream/stream.h
./transaction.o: ../../pikotools/src/space/space.h
./transaction.o: ../../pikotools/src/convert/inttostr.h
./transaction.o: ../../pikotools/src/utf8/utf8.h
./transaction.o: ../../pikotools/src/convert/misc.h
./transaction.o: ../../pikotools/src/textstream/stream.h
./transaction.o: ../../pikotools/src/utf8/utf8.h
./transaction.o: ../../pikotools/src/date/date.h
./transaction.o: ../../pikotools/src/membuffer/membuffer.h
./transaction.o: ../../pikotools/src/textstream/types.h
@ -295,6 +308,7 @@
./xmlconnector.o: ../../pikotools/src/textstream/textstream.h
./xmlconnector.o: ../../pikotools/src/textstream/stream.h
./xmlconnector.o: ../../pikotools/src/space/space.h
./xmlconnector.o: ../../pikotools/src/convert/misc.h
./xmlconnector.o: ../../pikotools/src/utf8/utf8.h
./xmlconnector.o: ../../pikotools/src/membuffer/membuffer.h
./xmlconnector.o: ../../pikotools/src/textstream/types.h
@ -312,6 +326,7 @@
./csvconnector.o: ../../pikotools/src/textstream/textstream.h
./csvconnector.o: ../../pikotools/src/textstream/stream.h
./csvconnector.o: ../../pikotools/src/space/space.h
./csvconnector.o: ../../pikotools/src/convert/misc.h
./csvconnector.o: ../../pikotools/src/utf8/utf8.h
./csvconnector.o: ../../pikotools/src/membuffer/membuffer.h
./csvconnector.o: ../../pikotools/src/textstream/types.h

View File

@ -568,53 +568,95 @@ void BaseExpression::esc(char32_t val, pt::Stream & stream, const FT & field_typ
}
void BaseExpression::esc(const wchar_t * val, bool has_known_length, size_t len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env)
void BaseExpression::esc_numeric_string(const char * val, bool has_known_length, size_t len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env)
{
if( field_type.is_numeric() )
{
esc_numeric_string(val, has_known_length, len, stream, field_type, model_env);
}
else
{
esc_normal_string(val, has_known_length, len, stream, field_type, model_env);
}
esc_numeric_string_generic(val, has_known_length, len, stream, field_type, model_env);
}
void BaseExpression::esc(const char * val, bool has_known_length, size_t len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env)
void BaseExpression::esc_numeric_string(const wchar_t * val, bool has_known_length, size_t len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env)
{
if( field_type.is_numeric() )
{
esc_numeric_string(val, has_known_length, len, stream, field_type, model_env);
}
else
{
esc_normal_string(val, has_known_length, len, stream, field_type, model_env);
}
esc_numeric_string_generic(val, has_known_length, len, stream, field_type, model_env);
}
void BaseExpression::esc_hex_string(const char * val, bool has_known_length, size_t len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env)
{
esc_hex_string_generic(val, has_known_length, len, stream, field_type, model_env);
}
void BaseExpression::esc_hex_string(const wchar_t * val, bool has_known_length, size_t len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env)
{
esc_hex_string_generic(val, has_known_length, len, stream, field_type, model_env);
}
void BaseExpression::esc_bin_string(const char * val, bool has_known_length, size_t len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env)
{
esc_hex_string(val, has_known_length, len, stream, field_type, model_env);
}
void BaseExpression::esc_bin_string(const wchar_t * val, bool has_known_length, size_t len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env)
{
esc_hex_string(val, has_known_length, len, stream, field_type, model_env);
}
void BaseExpression::esc_ordinary_string(const char * val, size_t val_len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env)
{
pt::utf8_to_output_function(val, val_len, [&](int c){
esc(static_cast<char32_t>(c), stream, field_type, model_env);
});
}
void BaseExpression::esc_ordinary_string(const char * val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env)
{
pt::utf8_to_output_function(val, [&](int c){
esc(static_cast<char32_t>(c), stream, field_type, model_env);
});
}
void BaseExpression::esc_ordinary_string(const wchar_t * val, size_t val_len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env)
{
pt::wide_to_output_function(val, val_len, [&](int c){
esc(static_cast<char32_t>(c), stream, field_type, model_env);
});
}
void BaseExpression::esc_ordinary_string(const wchar_t * val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env)
{
pt::wide_to_output_function(val, [&](int c){
esc(static_cast<char32_t>(c), stream, field_type, model_env);
});
}
void BaseExpression::esc(const std::wstring & val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env)
{
esc(val.c_str(), true, val.size(), stream, field_type, model_env);
esc_string(val.c_str(), true, val.size(), stream, field_type, model_env);
}
void BaseExpression::esc(const wchar_t * val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env)
{
esc(val, false, 0, stream, field_type, model_env);
esc_string(val, false, 0, stream, field_type, model_env);
}
void BaseExpression::esc(const std::string & val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env)
{
esc(val.c_str(), true, val.size(), stream, field_type, model_env);
esc_string(val.c_str(), true, val.size(), stream, field_type, model_env);
}
void BaseExpression::esc(const char * val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env)
{
esc(val, false, 0, stream, field_type, model_env);
esc_string(val, false, 0, stream, field_type, model_env);
}

View File

@ -768,29 +768,26 @@ protected:
virtual void before_field_value_string(const FT & field_type, ModelEnv * model_env);
virtual void after_field_value_string(const FT & field_type, ModelEnv * model_env);
char char_to_hex_part(char c);
void char_to_hex(char c, pt::Stream & stream);
void char_to_hex(char32_t c, pt::Stream & stream);
void esc(const wchar_t * val, bool has_known_length, size_t len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env);
void esc(const char * val, bool has_known_length, size_t len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env);
bool is_empty_field(const wchar_t * value);
virtual char char_to_hex_part(char c);
virtual void char_to_hex(char c, pt::Stream & stream);
virtual void char_to_hex(char32_t c, pt::Stream & stream);
virtual void esc_numeric_string(const char * val, bool has_known_length, size_t len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env);
virtual void esc_numeric_string(const wchar_t * val, bool has_known_length, size_t len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env);
virtual void esc_hex_string(const char * val, bool has_known_length, size_t len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env);
virtual void esc_hex_string(const wchar_t * val, bool has_known_length, size_t len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env);
virtual void esc_bin_string(const char * val, bool has_known_length, size_t len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env);
virtual void esc_bin_string(const wchar_t * val, bool has_known_length, size_t len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env);
virtual void esc_ordinary_string(const char * val, size_t val_len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env);
virtual void esc_ordinary_string(const char * val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env);
virtual void esc_ordinary_string(const wchar_t * val, size_t val_len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env);
virtual void esc_ordinary_string(const wchar_t * val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env);
// virtual void esc(const wchar_t * val, bool has_known_length, size_t len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env);
// virtual void esc(const char * val, bool has_known_length, size_t len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env);
template<typename CharType>
void esc_normal_string(CharType * val, bool has_known_length, size_t len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env)
{
for(size_t i = 0 ; has_known_length ? (i < len) : val[i] != 0 ; ++i)
{
esc(val[i], stream, field_type, model_env);
}
}
template<typename CharType>
void esc_numeric_string(CharType * val, bool has_known_length, size_t len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env)
void esc_numeric_string_generic(const CharType * val, bool has_known_length, size_t len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env)
{
bool was_comma = false;
bool was_something_printed = false;
@ -833,6 +830,49 @@ protected:
}
template<typename CharType>
void esc_hex_string_generic(const CharType * val, bool has_known_length, size_t len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env)
{
for(size_t i = 0 ; has_known_length ? (i < len) : val[i] != 0 ; ++i)
{
esc(val[i], stream, field_type, model_env);
}
}
template<typename CharType>
void esc_string(const CharType * val, bool has_known_length, size_t len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env)
{
if( field_type.is_numeric() )
{
esc_numeric_string(val, has_known_length, len, stream, field_type, model_env);
}
else
{
if( field_type.is_hexadecimal() )
{
esc_hex_string(val, has_known_length, len, stream, field_type, model_env);
}
else
if( field_type.is_binary() )
{
esc_bin_string(val, has_known_length, len, stream, field_type, model_env);
}
else
{
if( has_known_length )
{
esc_ordinary_string(val, len, stream, field_type, model_env);
}
else
{
esc_ordinary_string(val, stream, field_type, model_env);
}
}
}
}
template<typename FieldValue>
void put_field_value_or_null(const FieldValue & field_value, void (Model::*getter_method)(pt::Stream &), const FT & field_type, ModelEnv * model_env)
{
@ -939,6 +979,8 @@ protected:
}
bool is_empty_field(const wchar_t * value);
};
}

View File

@ -37,7 +37,6 @@
#include <vector>
#include "baseexpression.h"
#include "morm_types.h"
namespace morm

View File

@ -39,7 +39,6 @@ namespace morm
{
void PostgreSQLExpression::before_field_value_string(const FT & field_type, ModelEnv * model_env)
{
if( field_type.is_binary() )
@ -65,6 +64,7 @@ void PostgreSQLExpression::before_field_value_string(const FT & field_type, Mode
}
}
void PostgreSQLExpression::after_field_value_string(const FT & field_type, ModelEnv * model_env)
{
if( model_env && model_env->add_postfix_percent )

View File

@ -50,11 +50,6 @@ public:
protected:
private:
void before_field_value_string(const FT & field_type, ModelEnv * model_env) override;
void after_field_value_string(const FT & field_type, ModelEnv * model_env) override;