From 82281b4363d57143285ac67bba304d17d057b77b Mon Sep 17 00:00:00 2001 From: Tomasz Sowa Date: Sat, 30 Jul 2022 04:00:35 +0200 Subject: [PATCH] add a debug mode - fill a Space structure with some info from CURLOPT_DEBUGFUNCTION while here: - allow to set headers from a string - add a seek function (CURLOPT_SEEKFUNCTION) --- winixd/utils/http.cpp | 449 ++++++++++++++++++++++++++++++++++++------ winixd/utils/http.h | 101 +++++++++- 2 files changed, 487 insertions(+), 63 deletions(-) diff --git a/winixd/utils/http.cpp b/winixd/utils/http.cpp index b57b793..5719f7d 100644 --- a/winixd/utils/http.cpp +++ b/winixd/utils/http.cpp @@ -39,13 +39,21 @@ #include "core/header.h" - namespace Winix { Http::Http() +{ + begin(); +} + + +/* + * prepare to start a new request + */ +Http & Http::begin() { curl = nullptr; synchro = nullptr; @@ -54,11 +62,23 @@ Http::Http() http_headers = nullptr; read_function_input = nullptr; read_function_index = 0; - additional_headers_to_send = nullptr; + additional_space_headers_to_send = nullptr; + additional_string_headers_to_send = nullptr; + additional_wstring_headers_to_send = nullptr; bearer_token = nullptr; output_headers_space = nullptr; change_header_names_to_lower = true; output_content_type = nullptr; + debug_mode = 0; + debug_info = nullptr; + + return *this; +} + + +void Http::clear() +{ + reset_headers(); } @@ -69,9 +89,112 @@ Http::~Http() -Http & Http::add_input_headers(pt::Space * headers) +Http & Http::add_header(const pt::TextStream & header) { - additional_headers_to_send = headers; + header.to_str(temp_header_ascii); + add_header(temp_header_ascii); + + return *this; +} + + +Http & Http::add_header(const pt::WTextStream & header) +{ + header.to_str(temp_header_ascii); + add_header(temp_header_ascii); + + return *this; +} + + +Http & Http::add_header(const std::string & header) +{ + if( !header.empty() ) + { + http_headers = curl_slist_append(http_headers, header.c_str()); + } + + return *this; +} + + +Http & Http::add_header(const std::wstring & header) +{ + if( !header.empty() ) + { + pt::wide_to_utf8(header, temp_header_ascii); + http_headers = curl_slist_append(http_headers, temp_header_ascii.c_str()); + } + + return *this; +} + + +Http & Http::add_header(const char * str, size_t len) +{ + if( len > 0 ) + { + temp_header_ascii.assign(str, len); + http_headers = curl_slist_append(http_headers, temp_header_ascii.c_str()); + } + + return *this; +} + + +Http & Http::add_header(const wchar_t * str, size_t len) +{ + if( len > 0 ) + { + pt::wide_to_utf8(str, len, temp_header_ascii); + http_headers = curl_slist_append(http_headers, temp_header_ascii.c_str()); + } + + return *this; +} + + + + + +Http & Http::add_headers(pt::Space * headers) +{ + additional_space_headers_to_send = headers; + return *this; +} + + +Http & Http::add_headers(pt::Space & headers) +{ + additional_space_headers_to_send = &headers; + return *this; +} + + +Http & Http::add_headers(const char * headers) +{ + additional_string_headers_to_send = headers; + return *this; +} + + +Http & Http::add_headers(const wchar_t * headers) +{ + additional_wstring_headers_to_send = headers; + return *this; +} + + +Http & Http::add_headers(const std::string & headers) +{ + additional_string_headers_to_send = headers.c_str(); + return *this; +} + + +Http & Http::add_headers(const std::wstring & headers) +{ + additional_wstring_headers_to_send = headers.c_str(); return *this; } @@ -112,13 +235,8 @@ bool Http::get(const wchar_t * url, std::wstring & out, bool clear_str) pt::wide_to_utf8(url, url_ascii); - reset_headers(); bool status = fetch_internal(url_ascii.c_str(), nullptr, out_stream); - - // IMPROVE pikotools, add: pt::UTF8ToInt(out_stream, out); - std::string temp; - out_stream.to_str(temp); - pt::utf8_to_wide(temp, out, clear_str); + pt::utf8_to_wide(out_stream, out, clear_str); return status; } @@ -144,24 +262,8 @@ bool Http::get(const wchar_t * url, pt::WTextStream & out, bool clear_stream) pt::TextStream out_stream; pt::wide_to_utf8(url, url_ascii); - - reset_headers(); bool status = fetch_internal(url_ascii.c_str(), nullptr, out_stream); - - // IMPROVE pikotools, add: pt::UTF8ToInt(out_stream, out); - std::string temp; - out_stream.to_str(temp); - - // similar, improve pikotools - std::wstring temp_wide; - pt::utf8_to_wide(temp, temp_wide); - - if( clear_stream ) - { - out.clear(); - } - - out << temp_wide; + out << out_stream; return status; } @@ -194,7 +296,6 @@ bool Http::put(const wchar_t * url, const std::string & in, pt::WTextStream & ou pt::wide_to_utf8(url, url_ascii); - reset_headers(); bool status = fetch_internal(url_ascii.c_str(), &in, out_stream); out << out_stream; @@ -231,6 +332,16 @@ long Http::get_status() +void Http::clear_tmp_objects() +{ + out_headers_stream.clear(); + temp_header.clear(); + temp_header_ascii.clear(); + temp_header_value.clear(); + temp_header_value_ascii.clear(); +} + + void Http::reset_headers() { if( http_headers ) @@ -241,27 +352,55 @@ void Http::reset_headers() } -void Http::add_additional_headers() +void Http::add_additional_space_headers() { - if( additional_headers_to_send && additional_headers_to_send->is_object() ) + if( additional_space_headers_to_send && additional_space_headers_to_send->is_object() ) { pt::WTextStream header; - pt::Space::ObjectType::iterator i = additional_headers_to_send->value.value_object.begin(); + pt::Space::ObjectType::iterator i = additional_space_headers_to_send->value.value_object.begin(); - while( i != additional_headers_to_send->value.value_object.end() ) + while( i != additional_space_headers_to_send->value.value_object.end() ) { header.clear(); - header << i->first << ": "; - header << i->second->to_wstr(); - add_header(header); + pt::Space * value = i->second; + if( value->is_wstr() ) + { + header << value->value.value_wstring; + } + else + if( value->is_str() ) + { + header << value->value.value_string; + } + else + { + header << i->second->to_wstr(); + } + + add_header(header); ++i; } } } +void Http::add_additional_string_headers() +{ + if( additional_string_headers_to_send ) + { + add_additional_string_headers(additional_string_headers_to_send); + } + + if( additional_wstring_headers_to_send ) + { + add_additional_string_headers(additional_wstring_headers_to_send); + } +} + + + void Http::add_bearer_token() { if( bearer_token && bearer_token[0] != 0 ) @@ -273,19 +412,6 @@ void Http::add_bearer_token() } -void Http::add_header(const pt::WTextStream & header) -{ - header.to_str(temp_header); - add_header(temp_header); - temp_header.clear(); -} - - -void Http::add_header(const std::wstring & header) -{ - pt::wide_to_utf8(header, temp_header_ascii); - http_headers = curl_slist_append(http_headers, temp_header_ascii.c_str()); -} void Http::initialize_curl_if_needed() @@ -310,8 +436,6 @@ void Http::initialize_curl_if_needed() void Http::uninitialize_curl() { - reset_headers(); - if( curl ) { curl_easy_cleanup(curl); @@ -320,9 +444,17 @@ void Http::uninitialize_curl() } +void Http::use_debug_mode(bool debug, pt::Space * debug_info) +{ + this->debug_mode = debug ? 1 : 0; + this->debug_info = debug_info; +} + + // in can be pointer to const char * bool Http::fetch_internal(const char * url, const std::string * in, pt::TextStream & out) { + bool status = false; initialize_curl_if_needed(); if( output_headers_space ) @@ -334,7 +466,7 @@ bool Http::fetch_internal(const char * url, const std::string * in, pt::TextStre if( !curl ) { log << log1 << "Http: I can't initialize curl easy session" << logend; - return false; + return status; } error_buf[0] = 0; @@ -346,11 +478,13 @@ bool Http::fetch_internal(const char * url, const std::string * in, pt::TextStre { curl_easy_setopt(curl, CURLOPT_READFUNCTION, fetch_read_function); curl_easy_setopt(curl, CURLOPT_READDATA, this); + curl_easy_setopt(curl, CURLOPT_SEEKFUNCTION, fetch_seek_function); + curl_easy_setopt(curl, CURLOPT_SEEKDATA, this); curl_easy_setopt(curl, CURLOPT_POST, 1); /* * do not set content-leght header here - * curl uses "Expect: 100-continue" and it collides if content-length is set + * curl uses "Expect: 100-continue" and it collides if Content-Length is set * https://daniel.haxx.se/blog/2020/02/27/expect-tweaks-in-curl/ * */ @@ -367,6 +501,13 @@ bool Http::fetch_internal(const char * url, const std::string * in, pt::TextStre curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 20); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); + if( debug_mode == 1 ) + { + curl_easy_setopt(curl, CURLOPT_VERBOSE, debug_mode); + curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, debug_function); + curl_easy_setopt(curl, CURLOPT_DEBUGDATA, this); + } + if( output_headers_space || output_content_type ) { out_headers_stream.clear(); @@ -374,7 +515,20 @@ bool Http::fetch_internal(const char * url, const std::string * in, pt::TextStre curl_easy_setopt(curl, CURLOPT_HEADERDATA, &out_headers_stream); } - add_additional_headers(); + // block the Expect: 100-continue header + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Expect + // https://httpwg.org/specs/rfc7231.html#header.expect + // https://curl.se/libcurl/c/post-callback.html + /* + Using POST with HTTP 1.1 implies the use of a "Expect: 100-continue" + header. You can disable this header with CURLOPT_HTTPHEADER as usual. + NOTE: if you want chunked transfer too, you need to combine these two + since you can only set one list of headers with CURLOPT_HTTPHEADER. + */ + //add_header(L"Expect:"); + + add_additional_space_headers(); + add_additional_string_headers(); add_bearer_token(); if( http_headers ) @@ -383,23 +537,21 @@ bool Http::fetch_internal(const char * url, const std::string * in, pt::TextStre } CURLcode res = curl_easy_perform(curl); - reset_headers(); if( res == CURLE_OK ) { + status = true; parse_headers(); - out_headers_stream.clear(); } else { log << log1 << "Http: fetching failed: " << error_buf << ", code: " << static_cast(res) << logend; - - return false; } - return true; + clear_tmp_objects(); + return status; } @@ -433,6 +585,110 @@ size_t Http::fetch_read_function(char * ptr, size_t size, size_t nmemb, void * u } +int Http::fetch_seek_set(Http * http, curl_off_t offset) +{ + int status = CURL_SEEKFUNC_CANTSEEK; + size_t offset_size = static_cast(offset); + + if( offset_size >=0 && offset_size < http->read_function_input->size() ) + { + http->read_function_index = offset_size; + status = CURL_SEEKFUNC_OK; + } + + return status; +} + + +int Http::fetch_seek_cur(Http * http, curl_off_t offset) +{ + int status = CURL_SEEKFUNC_CANTSEEK; + + if( offset >= 0 ) + { + size_t offset_size = static_cast(offset); + + if( http->read_function_index + offset_size < http->read_function_input->size() ) + { + http->read_function_index += offset_size; + status = CURL_SEEKFUNC_OK; + } + } + else + { + size_t offset_size = static_cast(-offset); + + if( http->read_function_index >= offset_size ) + { + http->read_function_index -= offset_size; + status = CURL_SEEKFUNC_OK; + } + } + + return status; +} + + +int Http::fetch_seek_end(Http * http, curl_off_t offset) +{ + int status = CURL_SEEKFUNC_CANTSEEK; + + if( offset < 0 ) + { + size_t offset_size = static_cast(-offset); + + if( offset_size <= http->read_function_input->size() ) + { + http->read_function_index = http->read_function_input->size() - offset_size; + status = CURL_SEEKFUNC_OK; + } + } + + return status; +} + + + + +/* + * + * https://curl.se/libcurl/c/CURLOPT_SEEKFUNCTION.html + * + * These are the return codes for the seek callbacks + * + * #define CURL_SEEKFUNC_OK 0 + * #define CURL_SEEKFUNC_FAIL 1 // fail the entire transfer + * #define CURL_SEEKFUNC_CANTSEEK 2 / /tell libcurl seeking cannot be done, so + * libcurl might try other means instead + * + */ +int Http::fetch_seek_function(void * userdata, curl_off_t offset, int origin) +{ + Http * http = reinterpret_cast(userdata); + int status = CURL_SEEKFUNC_FAIL; + + if( origin == SEEK_SET ) + { + status = fetch_seek_set(http, offset); + } + else + if( origin == SEEK_CUR ) + { + status = fetch_seek_cur(http, offset); + } + else + if( origin == SEEK_END ) + { + status = fetch_seek_end(http, offset); + } + + return status; +} + + + + + size_t Http::fetch_write_function(char * ptr, size_t size, size_t nmemb, void * userdata) { size_t len = size * nmemb; @@ -576,6 +832,83 @@ void Http::parse_headers() } +void Http::debug_function_append(pt::Space & debug, const char * prefix, const char * start, size_t part_size) +{ + if( part_size > 0 ) + { + debug.add(prefix); + std::string & last_table_str_item = debug.value.value_table.back()->value.value_string; + last_table_str_item.append(start, part_size); + } +} + + +void Http::debug_function_append(Http * http, const char * prefix, char * data, size_t size) +{ + pt::Space & debug = *http->debug_info; + const char * start = data; + size_t part_size = 0; + size_t i = 0; + + while( i < size ) + { + if( data[i]== '\r' && i+1 < size && data[i+1]=='\n' ) + { + debug_function_append(debug, prefix, start, part_size); + i += 2; + start = data + i; + part_size = 0; + } + else + if( data[i]== '\n' ) + { + debug_function_append(debug, prefix, start, part_size); + i += 1; + start = data + i; + part_size = 0; + } + else + { + part_size += 1; + i += 1; + } + } + + debug_function_append(debug, prefix, start, part_size); +} + + + +int Http::debug_function(CURL * handle, curl_infotype type, char * data, size_t size, void * userdata) +{ + Http * http = reinterpret_cast(userdata); + + if( http->debug_info ) + { + switch(type) + { + case CURLINFO_TEXT: + debug_function_append(http, "", data, size); + break; + + case CURLINFO_HEADER_IN: + debug_function_append(http, "Header received: ", data, size); + break; + + case CURLINFO_HEADER_OUT: + debug_function_append(http, "Header sent: ", data, size); + break; + + default: + break; + } + } + + return 0; +} + + + } diff --git a/winixd/utils/http.h b/winixd/utils/http.h index ecd57e5..0740939 100644 --- a/winixd/utils/http.h +++ b/winixd/utils/http.h @@ -59,12 +59,49 @@ public: Http(const Http &) = delete; Http(Http &&) = delete; + /* + * start a new request + * called automatically from cctor + * you have to call it manually to make another requests + * + */ + Http & begin(); + + + + /* + * add one header (should not end with \r\n or \n ) + * + */ + Http & add_header(const pt::TextStream & header); + Http & add_header(const pt::WTextStream & header); + Http & add_header(const std::string & header); + Http & add_header(const std::wstring & header); + Http & add_header(const char * str, size_t len); + Http & add_header(const wchar_t * str, size_t len); + + + /* + * additional headers (we do not copy it too) + * headers can be separated with \r\n or just with \n + * last \n (or \r\n) can be omitted + * empty lines will be ignored + * + * headers from wchar_t* will be replaced to char* by using pt::wide_to_utf8 + */ + Http & add_headers(const char * headers); + Http & add_headers(const wchar_t * headers); + Http & add_headers(const std::string & headers); + Http & add_headers(const std::wstring & headers); + /* * we do not copy the space structure but only get a pointer to it * so you have to preserve the structure until get()/put() is called */ - Http & add_input_headers(pt::Space * headers); + Http & add_headers(pt::Space * headers); + Http & add_headers(pt::Space & headers); + /* * output headers will be provided in a Space structure as key/value pairs (object) @@ -90,6 +127,15 @@ public: void initialize_curl_if_needed(); void uninitialize_curl(); + /* + * if true set additional information how the curl library works when connection is made + * debug_info will be a table in such a case + * + * if debug is false then debug_info can be nullptr (if not null then will be set to Space null) + * + */ + void use_debug_mode(bool debug, pt::Space * debug_info); + /* * we do not copy the string but only get a pointer to its c_str() * so you have to preserve the string until get()/put() is called @@ -124,12 +170,16 @@ private: size_t read_function_index; const std::string * read_function_input; curl_slist * http_headers; - pt::Space * additional_headers_to_send; + pt::Space * additional_space_headers_to_send; + const char * additional_string_headers_to_send; + const wchar_t * additional_wstring_headers_to_send; pt::Space * output_headers_space; const wchar_t * bearer_token = nullptr; pt::TextStream out_headers_stream; bool change_header_names_to_lower; std::wstring * output_content_type; + long debug_mode; + pt::Space * debug_info; std::wstring temp_header; std::string temp_header_ascii; @@ -139,18 +189,59 @@ private: bool fetch_internal(const char * url, const std::string * in, pt::TextStream & out); static size_t fetch_read_function(char * ptr, size_t size, size_t nmemb, void * userdata); + static int fetch_seek_set(Http * http, curl_off_t offset); + static int fetch_seek_cur(Http * http, curl_off_t offset); + static int fetch_seek_end(Http * http, curl_off_t offset); + static int fetch_seek_function(void * userdata, curl_off_t offset, int origin); static size_t fetch_write_function(char * ptr, size_t size, size_t nmemb, void * userdata); static size_t fetch_header_function(char * ptr, size_t size, size_t nmemb, void * userdata); + static void debug_function_append(pt::Space & debug, const char * prefix, const char * start, size_t part_size); + static void debug_function_append(Http * http, const char * prefix, char * data, size_t size); + static int debug_function(CURL * handle, curl_infotype type, char * data, size_t size, void * userdata); + void clear_tmp_objects(); void reset_headers(); - void add_additional_headers(); + void add_additional_space_headers(); + void add_additional_string_headers(); void add_bearer_token(); - void add_header(const pt::WTextStream & header); - void add_header(const std::wstring & header); void skip_white(pt::TextStream::iterator & i); void parse_header_name(pt::TextStream::iterator & i, std::string & name); void parse_header_value(pt::TextStream::iterator & i, std::string & value); void parse_headers(); + template + void add_additional_string_headers(StringType * str) + { + StringType * part_start = str; + size_t part_size = 0; + size_t i = 0; + + while( str[i] != 0 ) + { + if( str[i]== '\r' && str[i+1]=='\n' ) + { + add_header(part_start, part_size); + i += 2; + part_start = str + i; + part_size = 0; + } + else + if( str[i]== '\n' ) + { + add_header(part_start, part_size); + i += 1; + part_start = str + i; + part_size = 0; + } + else + { + part_size += 1; + i += 1; + } + } + + add_header(part_start, part_size); + } + };