From 90b4d9af0b00f67ed025c7f68d42541e0d9fe777 Mon Sep 17 00:00:00 2001 From: Tomasz Sowa Date: Sat, 1 Jun 2024 00:17:01 +0200 Subject: [PATCH] use a char32_t character as a main character when converting strings Use a char32_t instead of a wchar_t type. This is needed on systems where sizeof(wchar_t) is equal to 2. --- src/Makefile.dep | 47 ++++++--------- src/baseexpression.cpp | 34 ++++++++--- src/baseexpression.h | 21 +++---- src/csvexpression.cpp | 2 +- src/csvexpression.h | 2 +- src/dbconnector.cpp | 20 ++++--- src/dbconnector.h | 33 ++++++----- src/jsonexpression.cpp | 2 +- src/jsonexpression.h | 2 +- src/postgresqlconnector.cpp | 109 +---------------------------------- src/postgresqlconnector.h | 2 +- src/postgresqlexpression.cpp | 2 +- src/postgresqlexpression.h | 2 +- src/xmlexpression.cpp | 2 +- src/xmlexpression.h | 2 +- 15 files changed, 92 insertions(+), 190 deletions(-) diff --git a/src/Makefile.dep b/src/Makefile.dep index 4205586..e1f0baa 100644 --- a/src/Makefile.dep +++ b/src/Makefile.dep @@ -59,7 +59,6 @@ ./dbconnector.o: ../../pikotools/src/convert/strtoint.h ./dbconnector.o: ../../pikotools/src/convert/text.h ./dbconnector.o: ../../pikotools/src/convert/misc.h -./dbconnector.o: ../../pikotools/src/textstream/types.h ./dbconnector.o: ../../pikotools/src/convert/double.h ./dbexpression.o: dbexpression.h baseexpression.h ./dbexpression.o: ../../pikotools/src/textstream/stream.h @@ -147,8 +146,6 @@ ./jsonexpression.o: fieldvaluehelper.h wrapper.h spacewrapper.h ./jsonexpression.o: baseobjectwrapper.h modelcontainerwrapper.h select.h ft.h ./jsonexpression.o: ../../pikotools/src/convert/misc.h -./jsonexpression.o: ../../pikotools/src/convert/text.h -./jsonexpression.o: ../../pikotools/src/textstream/types.h ./model.o: model.h ../../pikotools/src/textstream/textstream.h ./model.o: ../../pikotools/src/textstream/stream.h ./model.o: ../../pikotools/src/space/space.h @@ -240,8 +237,6 @@ ./csvexpression.o: fieldvaluehelper.h wrapper.h spacewrapper.h ./csvexpression.o: baseobjectwrapper.h modelcontainerwrapper.h select.h ft.h ./csvexpression.o: ../../pikotools/src/convert/misc.h -./csvexpression.o: ../../pikotools/src/convert/text.h -./csvexpression.o: ../../pikotools/src/textstream/types.h ./xmlexpression.o: xmlexpression.h flatexpression.h baseexpression.h ./xmlexpression.o: ../../pikotools/src/textstream/stream.h ./xmlexpression.o: ../../pikotools/src/date/date.h @@ -259,8 +254,24 @@ ./xmlexpression.o: fieldvaluehelper.h wrapper.h spacewrapper.h ./xmlexpression.o: baseobjectwrapper.h modelcontainerwrapper.h select.h ft.h ./xmlexpression.o: ../../pikotools/src/convert/misc.h -./xmlexpression.o: ../../pikotools/src/convert/text.h -./xmlexpression.o: ../../pikotools/src/textstream/types.h +./postgresqlconnector.o: postgresqlconnector.h dbconnector.h +./postgresqlconnector.o: ../../pikotools/src/textstream/stream.h +./postgresqlconnector.o: ../../pikotools/src/log/log.h +./postgresqlconnector.o: ../../pikotools/src/textstream/textstream.h +./postgresqlconnector.o: ../../pikotools/src/textstream/stream.h +./postgresqlconnector.o: ../../pikotools/src/space/space.h +./postgresqlconnector.o: ../../pikotools/src/convert/inttostr.h +./postgresqlconnector.o: ../../pikotools/src/utf8/utf8.h +./postgresqlconnector.o: ../../pikotools/src/date/date.h +./postgresqlconnector.o: ../../pikotools/src/membuffer/membuffer.h +./postgresqlconnector.o: ../../pikotools/src/textstream/types.h +./postgresqlconnector.o: ../../pikotools/src/textstream/stream_private.h +./postgresqlconnector.o: ../../pikotools/src/log/filelog.h queryresult.h ft.h +./postgresqlconnector.o: postgresqlqueryresult.h postgresqlexpression.h +./postgresqlconnector.o: dbexpression.h baseexpression.h morm_types.h +./postgresqlconnector.o: modelenv.h modeldata.h cursorhelper.h finderhelper.h +./postgresqlconnector.o: fieldvaluehelper.h wrapper.h spacewrapper.h +./postgresqlconnector.o: baseobjectwrapper.h modelcontainerwrapper.h select.h ./transaction.o: transaction.h ../../pikotools/src/log/log.h ./transaction.o: ../../pikotools/src/textstream/textstream.h ./transaction.o: ../../pikotools/src/textstream/stream.h @@ -308,25 +319,3 @@ ./csvconnector.o: ../../pikotools/src/log/filelog.h finderhelper.h ./csvconnector.o: fieldvaluehelper.h wrapper.h spacewrapper.h ./csvconnector.o: baseobjectwrapper.h modelcontainerwrapper.h select.h ft.h -./postgresqlconnector.o: postgresqlconnector.h dbconnector.h -./postgresqlconnector.o: ../../pikotools/src/textstream/stream.h -./postgresqlconnector.o: ../../pikotools/src/log/log.h -./postgresqlconnector.o: ../../pikotools/src/textstream/textstream.h -./postgresqlconnector.o: ../../pikotools/src/textstream/stream.h -./postgresqlconnector.o: ../../pikotools/src/space/space.h -./postgresqlconnector.o: ../../pikotools/src/convert/inttostr.h -./postgresqlconnector.o: ../../pikotools/src/utf8/utf8.h -./postgresqlconnector.o: ../../pikotools/src/date/date.h -./postgresqlconnector.o: ../../pikotools/src/membuffer/membuffer.h -./postgresqlconnector.o: ../../pikotools/src/textstream/types.h -./postgresqlconnector.o: ../../pikotools/src/textstream/stream_private.h -./postgresqlconnector.o: ../../pikotools/src/log/filelog.h queryresult.h ft.h -./postgresqlconnector.o: postgresqlqueryresult.h postgresqlexpression.h -./postgresqlconnector.o: dbexpression.h baseexpression.h morm_types.h -./postgresqlconnector.o: modelenv.h modeldata.h cursorhelper.h finderhelper.h -./postgresqlconnector.o: fieldvaluehelper.h wrapper.h spacewrapper.h -./postgresqlconnector.o: baseobjectwrapper.h modelcontainerwrapper.h select.h -./postgresqlconnector.o: ../../pikotools/src/convert/strtoint.h -./postgresqlconnector.o: ../../pikotools/src/convert/text.h -./postgresqlconnector.o: ../../pikotools/src/convert/misc.h -./postgresqlconnector.o: ../../pikotools/src/textstream/types.h diff --git a/src/baseexpression.cpp b/src/baseexpression.cpp index babf731..16e1284 100644 --- a/src/baseexpression.cpp +++ b/src/baseexpression.cpp @@ -467,7 +467,7 @@ void BaseExpression::char_to_hex(char c, pt::Stream & stream) } -void BaseExpression::char_to_hex(wchar_t c, pt::Stream & stream) +void BaseExpression::char_to_hex(char32_t c, pt::Stream & stream) { unsigned int z = static_cast(c); @@ -485,17 +485,28 @@ void BaseExpression::char_to_hex(wchar_t c, pt::Stream & stream) */ bool BaseExpression::esc_char(char val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env) { - return esc_char((wchar_t)(unsigned char)val, stream, field_type, model_env); + return esc_char((char32_t)(unsigned char)val, stream, field_type, model_env); } /* * return true if the val character was escaped and put (or ignored) to the stream * - * in most cases you have to provide your own esc_char(wchar_t val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env) method * */ bool BaseExpression::esc_char(wchar_t val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env) +{ + return esc_char((char32_t)val, stream, field_type, model_env); +} + + +/* + * return true if the val character was escaped and put (or ignored) to the stream + * + * in most cases you have to provide your own esc_char(char32_t val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env) method + * + */ +bool BaseExpression::esc_char(char32_t val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env) { return false; } @@ -524,6 +535,12 @@ void BaseExpression::esc(unsigned char val, pt::Stream & stream, const FT & fiel void BaseExpression::esc(wchar_t val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env) +{ + esc(static_cast(val), stream, field_type, model_env); +} + + +void BaseExpression::esc(char32_t val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env) { if( field_type.is_binary() || field_type.is_hexadecimal() ) { @@ -720,18 +737,17 @@ void BaseExpression::esc(const pt::Stream & val, pt::Stream & stream, const FT & if( val.is_char_stream() ) { // from utf8 to utf8 or from utf8 to wide - pt::utf8_to_output_function(val, [&](int z) { - esc(static_cast(z), stream, field_type, model_env); + pt::utf8_to_output_function_by_index(val, [&](int z) { + esc(static_cast(z), stream, field_type, model_env); }); } else if( val.is_wchar_stream() ) { // from wide to wide or from wide to utf8 - for(size_t i=0 ; i < val.size() ; ++i) - { - esc(val.get_wchar(i), stream, field_type, model_env); - } + pt::wide_to_output_function_by_index(val, [&](int z) { + esc(static_cast(z), stream, field_type, model_env); + }); } } diff --git a/src/baseexpression.h b/src/baseexpression.h index 4fc29ba..4656dd2 100644 --- a/src/baseexpression.h +++ b/src/baseexpression.h @@ -332,10 +332,12 @@ public: */ virtual bool esc_char(char val, pt::Stream & stream, const FT & field_type = FT::default_type, ModelEnv * model_env = nullptr); virtual bool esc_char(wchar_t val, pt::Stream & stream, const FT & field_type = FT::default_type, ModelEnv * model_env = nullptr); + virtual bool esc_char(char32_t val, pt::Stream & stream, const FT & field_type = FT::default_type, ModelEnv * model_env = nullptr); virtual void esc(char val, pt::Stream & stream, const FT & field_type = FT::default_type, ModelEnv * model_env = nullptr); virtual void esc(unsigned char val, pt::Stream & stream, const FT & field_type = FT::default_type, ModelEnv * model_env = nullptr); virtual void esc(wchar_t val, pt::Stream & stream, const FT & field_type = FT::default_type, ModelEnv * model_env = nullptr); + virtual void esc(char32_t val, pt::Stream & stream, const FT & field_type = FT::default_type, ModelEnv * model_env = nullptr); virtual void esc(const std::wstring & val, pt::Stream & stream, const FT & field_type = FT::default_type, ModelEnv * model_env = nullptr); virtual void esc(const wchar_t * val, pt::Stream & stream, const FT & field_type = FT::default_type, ModelEnv * model_env = nullptr); @@ -366,24 +368,17 @@ public: if( val.is_char_stream() ) { // from utf8 to utf8 or from utf8 to wide - typename pt::TextStreamBase::const_iterator start = val.begin(); - typename pt::TextStreamBase::const_iterator end = val.end(); - - pt::utf8_to_output_function(start, end, [&](int z) { - esc(static_cast(z), stream, field_type, model_env); + pt::utf8_to_output_function(val, [&](int z) { + esc(static_cast(z), stream, field_type, model_env); }); } else if( val.is_wchar_stream() ) { // from wide to wide or from wide to utf8 - typename pt::TextStreamBase::const_iterator i = val.begin(); - typename pt::TextStreamBase::const_iterator end = val.end(); - - for(; i != end ; ++i) - { - esc(*i, stream, field_type, model_env); - } + pt::wide_to_output_function(val, [&](int z) { + esc(static_cast(z), stream, field_type, model_env); + }); } } @@ -762,7 +757,7 @@ protected: char char_to_hex_part(char c); void char_to_hex(char c, pt::Stream & stream); - void char_to_hex(wchar_t c, pt::Stream & stream); + void char_to_hex(char32_t c, pt::Stream & stream); void esc(const wchar_t * val, bool has_known_length, size_t len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env); void esc(const char * val, bool has_known_length, size_t len, pt::Stream & stream, const FT & field_type, ModelEnv * model_env); diff --git a/src/csvexpression.cpp b/src/csvexpression.cpp index 0d61ded..0b44d47 100644 --- a/src/csvexpression.cpp +++ b/src/csvexpression.cpp @@ -88,7 +88,7 @@ void CSVExpression::after_field_value_string(const FT & field_type, ModelEnv * m } -bool CSVExpression::esc_char(wchar_t val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env) +bool CSVExpression::esc_char(char32_t val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env) { return pt::try_esc_to_csv(val, stream); } diff --git a/src/csvexpression.h b/src/csvexpression.h index 447dacc..5f6caba 100644 --- a/src/csvexpression.h +++ b/src/csvexpression.h @@ -58,7 +58,7 @@ protected: void after_field_value_string(const FT & field_type, ModelEnv * model_env) override; using BaseExpression::esc; - bool esc_char(wchar_t val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env) override; + bool esc_char(char32_t val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env) override; }; diff --git a/src/dbconnector.cpp b/src/dbconnector.cpp index 33d7ee7..5239fab 100644 --- a/src/dbconnector.cpp +++ b/src/dbconnector.cpp @@ -596,11 +596,11 @@ void DbConnector::unescape_hex_char(const char * str, char & c) } -void DbConnector::unescape_hex_char(const char * str, wchar_t & c) +void DbConnector::unescape_hex_char(const char * str, char32_t & c) { unsigned int res = 0; unescape_hex_char(str, sizeof(wchar_t) * 2, res); - c = (wchar_t)res; + c = (char32_t)res; } @@ -610,7 +610,7 @@ void DbConnector::unescape_bin_char(const char * str, char & c) } -void DbConnector::unescape_bin_char(const char * str, wchar_t & c) +void DbConnector::unescape_bin_char(const char * str, char32_t & c) { unescape_hex_char(str, c); } @@ -680,8 +680,15 @@ void DbConnector::get_value(const char * value_str, unsigned char & field_value, } - void DbConnector::get_value(const char * value_str, wchar_t & field_value, const FT & field_type) +{ + char32_t tmp; + get_value(value_str, tmp, field_type); + field_value = static_cast(tmp); +} + + +void DbConnector::get_value(const char * value_str, char32_t & field_value, const FT & field_type) { field_value = 0; @@ -705,7 +712,7 @@ void DbConnector::get_value(const char * value_str, wchar_t & field_value, const if( is_correct ) { - field_value = static_cast(value_int); + field_value = static_cast(value_int); } else { @@ -714,13 +721,12 @@ void DbConnector::get_value(const char * value_str, wchar_t & field_value, const } else { - field_value = static_cast((unsigned char)*value_str); + field_value = static_cast((unsigned char)*value_str); } } } - void DbConnector::get_value(const char * value_str, std::string & field_value, const FT & field_type) { if( field_type.is_hexadecimal() ) diff --git a/src/dbconnector.h b/src/dbconnector.h index 78f0b6e..156c7a4 100644 --- a/src/dbconnector.h +++ b/src/dbconnector.h @@ -136,25 +136,26 @@ public: virtual size_t get_transaction_group(); - virtual void get_value(const char * value_str, char & field_value, const FT & field_type = FT::default_type); + virtual void get_value(const char * value_str, char & field_value, const FT & field_type = FT::default_type); virtual void get_value(const char * value_str, unsigned char & field_value, const FT & field_type = FT::default_type); - virtual void get_value(const char * value_str, wchar_t & field_value, const FT & field_type = FT::default_type); - virtual void get_value(const char * value_str, std::wstring & field_value, const FT & field_type = FT::default_type); - virtual void get_value(const char * value_str, std::string & field_value, const FT & field_type = FT::default_type); - virtual void get_value(const char * value_str, std::string_view & field_value, const FT & field_type = FT::default_type); - virtual void get_value(const char * value_str, bool & field_value, const FT & field_type = FT::default_type); + virtual void get_value(const char * value_str, wchar_t & field_value, const FT & field_type = FT::default_type); + virtual void get_value(const char * value_str, char32_t & field_value, const FT & field_type = FT::default_type); + virtual void get_value(const char * value_str, std::wstring & field_value, const FT & field_type = FT::default_type); + virtual void get_value(const char * value_str, std::string & field_value, const FT & field_type = FT::default_type); + virtual void get_value(const char * value_str, std::string_view & field_value, const FT & field_type = FT::default_type); + virtual void get_value(const char * value_str, bool & field_value, const FT & field_type = FT::default_type); virtual void get_value(const char * value_str, short & field_value, const FT & field_type = FT::default_type); - virtual void get_value(const char * value_str, unsigned short & field_value, const FT & field_type = FT::default_type); - virtual void get_value(const char * value_str, int & field_value, const FT & field_type = FT::default_type); - virtual void get_value(const char * value_str, unsigned int & field_value, const FT & field_type = FT::default_type); - virtual void get_value(const char * value_str, long & field_value, const FT & field_type = FT::default_type); + virtual void get_value(const char * value_str, unsigned short & field_value, const FT & field_type = FT::default_type); + virtual void get_value(const char * value_str, int & field_value, const FT & field_type = FT::default_type); + virtual void get_value(const char * value_str, unsigned int & field_value, const FT & field_type = FT::default_type); + virtual void get_value(const char * value_str, long & field_value, const FT & field_type = FT::default_type); virtual void get_value(const char * value_str, unsigned long & field_value, const FT & field_type = FT::default_type); virtual void get_value(const char * value_str, long long & field_value, const FT & field_type = FT::default_type); - virtual void get_value(const char * value_str, unsigned long long & field_value, const FT & field_type = FT::default_type); + virtual void get_value(const char * value_str, unsigned long long & field_value, const FT & field_type = FT::default_type); virtual void get_value(const char * value_str, float & field_value, const FT & field_type = FT::default_type); - virtual void get_value(const char * value_str, double & field_value, const FT & field_type = FT::default_type); - virtual void get_value(const char * value_str, long double & field_value, const FT & field_type = FT::default_type); - virtual void get_value(const char * value_str, pt::Date & field_value, const FT & field_type = FT::default_type); + virtual void get_value(const char * value_str, double & field_value, const FT & field_type = FT::default_type); + virtual void get_value(const char * value_str, long double & field_value, const FT & field_type = FT::default_type); + virtual void get_value(const char * value_str, pt::Date & field_value, const FT & field_type = FT::default_type); virtual void get_value(const char * value_str, pt::Space & field_value, const FT & field_type = FT::default_type); // add get_value for pt::TextStream and pt::WTextStream @@ -197,10 +198,10 @@ protected: virtual const char * query_last_sequence(const wchar_t * sequence_table_name); virtual void unescape_hex_char(const char * str, char & c); - virtual void unescape_hex_char(const char * str, wchar_t & c); + virtual void unescape_hex_char(const char * str, char32_t & c); virtual void unescape_bin_char(const char * str, char & c); - virtual void unescape_bin_char(const char * str, wchar_t & c); + virtual void unescape_bin_char(const char * str, char32_t & c); virtual void unescape_hex_string(const char * str, std::string & out); virtual void unescape_hex_string(const char * str, std::wstring & out); diff --git a/src/jsonexpression.cpp b/src/jsonexpression.cpp index 220a46a..569ca4f 100644 --- a/src/jsonexpression.cpp +++ b/src/jsonexpression.cpp @@ -135,7 +135,7 @@ void JSONExpression::after_field_value_list() } -bool JSONExpression::esc_char(wchar_t val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env) +bool JSONExpression::esc_char(char32_t val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env) { return pt::try_esc_to_json(val, stream); } diff --git a/src/jsonexpression.h b/src/jsonexpression.h index 7c0f198..f1de9a1 100644 --- a/src/jsonexpression.h +++ b/src/jsonexpression.h @@ -62,7 +62,7 @@ protected: void before_field_value_list() override; void after_field_value_list() override; - bool esc_char(wchar_t val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env) override; + bool esc_char(char32_t val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env) override; void esc(const pt::Space & space, pt::Stream & stream, const FT & field_type, ModelEnv * model_env) override; diff --git a/src/postgresqlconnector.cpp b/src/postgresqlconnector.cpp index 9e696d8..f579aaf 100644 --- a/src/postgresqlconnector.cpp +++ b/src/postgresqlconnector.cpp @@ -34,16 +34,14 @@ // for sleep() #include - #include "postgresqlconnector.h" -#include "utf8/utf8.h" #include "postgresqlexpression.h" -#include "convert/strtoint.h" namespace morm { + PostgreSQLConnector::PostgreSQLConnector() { pg_conn = nullptr; @@ -66,7 +64,6 @@ void PostgreSQLConnector::close() } - void PostgreSQLConnector::allocate_default_expression() { deallocate_expression(); @@ -75,9 +72,6 @@ void PostgreSQLConnector::allocate_default_expression() } - - - QueryResult * PostgreSQLConnector::create_query_result() { return new PostgreSQLQueryResult(); @@ -279,97 +273,6 @@ bool PostgreSQLConnector::query_declare_cursor(const pt::Stream & stream, QueryR } - - - - - - - - - -//void PostgreSQLConnector::CreateIdList(const std::vector & id_tab, std::wstring & list, bool add_parentheses) -//{ -//wchar_t buffer[50]; -//size_t buffer_len = sizeof(buffer) / sizeof(wchar_t); -// -// list.clear(); -// -// if( add_parentheses ) -// list += '('; -// -// for(size_t i=0 ; iuse_escaping_for_like ) { diff --git a/src/postgresqlexpression.h b/src/postgresqlexpression.h index 3f9c675..19ab5f2 100644 --- a/src/postgresqlexpression.h +++ b/src/postgresqlexpression.h @@ -58,7 +58,7 @@ private: void before_field_value_string(const FT & field_type, ModelEnv * model_env) override; void after_field_value_string(const FT & field_type, ModelEnv * model_env) override; - bool esc_char(wchar_t val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env) override; + bool esc_char(char32_t val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env) override; }; diff --git a/src/xmlexpression.cpp b/src/xmlexpression.cpp index 07ae13f..9b01fa9 100644 --- a/src/xmlexpression.cpp +++ b/src/xmlexpression.cpp @@ -93,7 +93,7 @@ void XMLExpression::put_name_value_separator() } -bool XMLExpression::esc_char(wchar_t val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env) +bool XMLExpression::esc_char(char32_t val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env) { return pt::try_esc_to_xml(val, stream); } diff --git a/src/xmlexpression.h b/src/xmlexpression.h index 695778e..89bfb13 100644 --- a/src/xmlexpression.h +++ b/src/xmlexpression.h @@ -55,7 +55,7 @@ protected: void put_name_value_separator() override; using BaseExpression::esc; - bool esc_char(wchar_t val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env) override; + bool esc_char(char32_t val, pt::Stream & stream, const FT & field_type, ModelEnv * model_env) override; void esc(const pt::Space & space, pt::Stream & stream, const FT & field_type, ModelEnv * model_env) override;