add a debug mode - fill a Space structure with some info from CURLOPT_DEBUGFUNCTION

while here:
- allow to set headers from a string
- add a seek function (CURLOPT_SEEKFUNCTION)
This commit is contained in:
Tomasz Sowa 2022-07-30 04:00:35 +02:00
parent 9d8d35b64f
commit 82281b4363
2 changed files with 487 additions and 63 deletions

View File

@ -39,13 +39,21 @@
#include "core/header.h"
namespace Winix
{
Http::Http()
{
begin();
}
/*
* prepare to start a new request
*/
Http & Http::begin()
{
curl = nullptr;
synchro = nullptr;
@ -54,11 +62,23 @@ Http::Http()
http_headers = nullptr;
read_function_input = nullptr;
read_function_index = 0;
additional_headers_to_send = nullptr;
additional_space_headers_to_send = nullptr;
additional_string_headers_to_send = nullptr;
additional_wstring_headers_to_send = nullptr;
bearer_token = nullptr;
output_headers_space = nullptr;
change_header_names_to_lower = true;
output_content_type = nullptr;
debug_mode = 0;
debug_info = nullptr;
return *this;
}
void Http::clear()
{
reset_headers();
}
@ -69,9 +89,112 @@ Http::~Http()
Http & Http::add_input_headers(pt::Space * headers)
Http & Http::add_header(const pt::TextStream & header)
{
additional_headers_to_send = headers;
header.to_str(temp_header_ascii);
add_header(temp_header_ascii);
return *this;
}
Http & Http::add_header(const pt::WTextStream & header)
{
header.to_str(temp_header_ascii);
add_header(temp_header_ascii);
return *this;
}
Http & Http::add_header(const std::string & header)
{
if( !header.empty() )
{
http_headers = curl_slist_append(http_headers, header.c_str());
}
return *this;
}
Http & Http::add_header(const std::wstring & header)
{
if( !header.empty() )
{
pt::wide_to_utf8(header, temp_header_ascii);
http_headers = curl_slist_append(http_headers, temp_header_ascii.c_str());
}
return *this;
}
Http & Http::add_header(const char * str, size_t len)
{
if( len > 0 )
{
temp_header_ascii.assign(str, len);
http_headers = curl_slist_append(http_headers, temp_header_ascii.c_str());
}
return *this;
}
Http & Http::add_header(const wchar_t * str, size_t len)
{
if( len > 0 )
{
pt::wide_to_utf8(str, len, temp_header_ascii);
http_headers = curl_slist_append(http_headers, temp_header_ascii.c_str());
}
return *this;
}
Http & Http::add_headers(pt::Space * headers)
{
additional_space_headers_to_send = headers;
return *this;
}
Http & Http::add_headers(pt::Space & headers)
{
additional_space_headers_to_send = &headers;
return *this;
}
Http & Http::add_headers(const char * headers)
{
additional_string_headers_to_send = headers;
return *this;
}
Http & Http::add_headers(const wchar_t * headers)
{
additional_wstring_headers_to_send = headers;
return *this;
}
Http & Http::add_headers(const std::string & headers)
{
additional_string_headers_to_send = headers.c_str();
return *this;
}
Http & Http::add_headers(const std::wstring & headers)
{
additional_wstring_headers_to_send = headers.c_str();
return *this;
}
@ -112,13 +235,8 @@ bool Http::get(const wchar_t * url, std::wstring & out, bool clear_str)
pt::wide_to_utf8(url, url_ascii);
reset_headers();
bool status = fetch_internal(url_ascii.c_str(), nullptr, out_stream);
// IMPROVE pikotools, add: pt::UTF8ToInt(out_stream, out);
std::string temp;
out_stream.to_str(temp);
pt::utf8_to_wide(temp, out, clear_str);
pt::utf8_to_wide(out_stream, out, clear_str);
return status;
}
@ -144,24 +262,8 @@ bool Http::get(const wchar_t * url, pt::WTextStream & out, bool clear_stream)
pt::TextStream out_stream;
pt::wide_to_utf8(url, url_ascii);
reset_headers();
bool status = fetch_internal(url_ascii.c_str(), nullptr, out_stream);
// IMPROVE pikotools, add: pt::UTF8ToInt(out_stream, out);
std::string temp;
out_stream.to_str(temp);
// similar, improve pikotools
std::wstring temp_wide;
pt::utf8_to_wide(temp, temp_wide);
if( clear_stream )
{
out.clear();
}
out << temp_wide;
out << out_stream;
return status;
}
@ -194,7 +296,6 @@ bool Http::put(const wchar_t * url, const std::string & in, pt::WTextStream & ou
pt::wide_to_utf8(url, url_ascii);
reset_headers();
bool status = fetch_internal(url_ascii.c_str(), &in, out_stream);
out << out_stream;
@ -231,6 +332,16 @@ long Http::get_status()
void Http::clear_tmp_objects()
{
out_headers_stream.clear();
temp_header.clear();
temp_header_ascii.clear();
temp_header_value.clear();
temp_header_value_ascii.clear();
}
void Http::reset_headers()
{
if( http_headers )
@ -241,27 +352,55 @@ void Http::reset_headers()
}
void Http::add_additional_headers()
void Http::add_additional_space_headers()
{
if( additional_headers_to_send && additional_headers_to_send->is_object() )
if( additional_space_headers_to_send && additional_space_headers_to_send->is_object() )
{
pt::WTextStream header;
pt::Space::ObjectType::iterator i = additional_headers_to_send->value.value_object.begin();
pt::Space::ObjectType::iterator i = additional_space_headers_to_send->value.value_object.begin();
while( i != additional_headers_to_send->value.value_object.end() )
while( i != additional_space_headers_to_send->value.value_object.end() )
{
header.clear();
header << i->first << ": ";
header << i->second->to_wstr();
add_header(header);
pt::Space * value = i->second;
if( value->is_wstr() )
{
header << value->value.value_wstring;
}
else
if( value->is_str() )
{
header << value->value.value_string;
}
else
{
header << i->second->to_wstr();
}
add_header(header);
++i;
}
}
}
void Http::add_additional_string_headers()
{
if( additional_string_headers_to_send )
{
add_additional_string_headers(additional_string_headers_to_send);
}
if( additional_wstring_headers_to_send )
{
add_additional_string_headers(additional_wstring_headers_to_send);
}
}
void Http::add_bearer_token()
{
if( bearer_token && bearer_token[0] != 0 )
@ -273,19 +412,6 @@ void Http::add_bearer_token()
}
void Http::add_header(const pt::WTextStream & header)
{
header.to_str(temp_header);
add_header(temp_header);
temp_header.clear();
}
void Http::add_header(const std::wstring & header)
{
pt::wide_to_utf8(header, temp_header_ascii);
http_headers = curl_slist_append(http_headers, temp_header_ascii.c_str());
}
void Http::initialize_curl_if_needed()
@ -310,8 +436,6 @@ void Http::initialize_curl_if_needed()
void Http::uninitialize_curl()
{
reset_headers();
if( curl )
{
curl_easy_cleanup(curl);
@ -320,9 +444,17 @@ void Http::uninitialize_curl()
}
void Http::use_debug_mode(bool debug, pt::Space * debug_info)
{
this->debug_mode = debug ? 1 : 0;
this->debug_info = debug_info;
}
// in can be pointer to const char *
bool Http::fetch_internal(const char * url, const std::string * in, pt::TextStream & out)
{
bool status = false;
initialize_curl_if_needed();
if( output_headers_space )
@ -334,7 +466,7 @@ bool Http::fetch_internal(const char * url, const std::string * in, pt::TextStre
if( !curl )
{
log << log1 << "Http: I can't initialize curl easy session" << logend;
return false;
return status;
}
error_buf[0] = 0;
@ -346,11 +478,13 @@ bool Http::fetch_internal(const char * url, const std::string * in, pt::TextStre
{
curl_easy_setopt(curl, CURLOPT_READFUNCTION, fetch_read_function);
curl_easy_setopt(curl, CURLOPT_READDATA, this);
curl_easy_setopt(curl, CURLOPT_SEEKFUNCTION, fetch_seek_function);
curl_easy_setopt(curl, CURLOPT_SEEKDATA, this);
curl_easy_setopt(curl, CURLOPT_POST, 1);
/*
* do not set content-leght header here
* curl uses "Expect: 100-continue" and it collides if content-length is set
* curl uses "Expect: 100-continue" and it collides if Content-Length is set
* https://daniel.haxx.se/blog/2020/02/27/expect-tweaks-in-curl/
*
*/
@ -367,6 +501,13 @@ bool Http::fetch_internal(const char * url, const std::string * in, pt::TextStre
curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 20);
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
if( debug_mode == 1 )
{
curl_easy_setopt(curl, CURLOPT_VERBOSE, debug_mode);
curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, debug_function);
curl_easy_setopt(curl, CURLOPT_DEBUGDATA, this);
}
if( output_headers_space || output_content_type )
{
out_headers_stream.clear();
@ -374,7 +515,20 @@ bool Http::fetch_internal(const char * url, const std::string * in, pt::TextStre
curl_easy_setopt(curl, CURLOPT_HEADERDATA, &out_headers_stream);
}
add_additional_headers();
// block the Expect: 100-continue header
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Expect
// https://httpwg.org/specs/rfc7231.html#header.expect
// https://curl.se/libcurl/c/post-callback.html
/*
Using POST with HTTP 1.1 implies the use of a "Expect: 100-continue"
header. You can disable this header with CURLOPT_HTTPHEADER as usual.
NOTE: if you want chunked transfer too, you need to combine these two
since you can only set one list of headers with CURLOPT_HTTPHEADER.
*/
//add_header(L"Expect:");
add_additional_space_headers();
add_additional_string_headers();
add_bearer_token();
if( http_headers )
@ -383,23 +537,21 @@ bool Http::fetch_internal(const char * url, const std::string * in, pt::TextStre
}
CURLcode res = curl_easy_perform(curl);
reset_headers();
if( res == CURLE_OK )
{
status = true;
parse_headers();
out_headers_stream.clear();
}
else
{
log << log1 << "Http: fetching failed: " << error_buf << ", code: "
<< static_cast<int>(res) << logend;
return false;
}
return true;
clear_tmp_objects();
return status;
}
@ -433,6 +585,110 @@ size_t Http::fetch_read_function(char * ptr, size_t size, size_t nmemb, void * u
}
int Http::fetch_seek_set(Http * http, curl_off_t offset)
{
int status = CURL_SEEKFUNC_CANTSEEK;
size_t offset_size = static_cast<size_t>(offset);
if( offset_size >=0 && offset_size < http->read_function_input->size() )
{
http->read_function_index = offset_size;
status = CURL_SEEKFUNC_OK;
}
return status;
}
int Http::fetch_seek_cur(Http * http, curl_off_t offset)
{
int status = CURL_SEEKFUNC_CANTSEEK;
if( offset >= 0 )
{
size_t offset_size = static_cast<size_t>(offset);
if( http->read_function_index + offset_size < http->read_function_input->size() )
{
http->read_function_index += offset_size;
status = CURL_SEEKFUNC_OK;
}
}
else
{
size_t offset_size = static_cast<size_t>(-offset);
if( http->read_function_index >= offset_size )
{
http->read_function_index -= offset_size;
status = CURL_SEEKFUNC_OK;
}
}
return status;
}
int Http::fetch_seek_end(Http * http, curl_off_t offset)
{
int status = CURL_SEEKFUNC_CANTSEEK;
if( offset < 0 )
{
size_t offset_size = static_cast<size_t>(-offset);
if( offset_size <= http->read_function_input->size() )
{
http->read_function_index = http->read_function_input->size() - offset_size;
status = CURL_SEEKFUNC_OK;
}
}
return status;
}
/*
*
* https://curl.se/libcurl/c/CURLOPT_SEEKFUNCTION.html
*
* These are the return codes for the seek callbacks
*
* #define CURL_SEEKFUNC_OK 0
* #define CURL_SEEKFUNC_FAIL 1 // fail the entire transfer
* #define CURL_SEEKFUNC_CANTSEEK 2 / /tell libcurl seeking cannot be done, so
* libcurl might try other means instead
*
*/
int Http::fetch_seek_function(void * userdata, curl_off_t offset, int origin)
{
Http * http = reinterpret_cast<Http*>(userdata);
int status = CURL_SEEKFUNC_FAIL;
if( origin == SEEK_SET )
{
status = fetch_seek_set(http, offset);
}
else
if( origin == SEEK_CUR )
{
status = fetch_seek_cur(http, offset);
}
else
if( origin == SEEK_END )
{
status = fetch_seek_end(http, offset);
}
return status;
}
size_t Http::fetch_write_function(char * ptr, size_t size, size_t nmemb, void * userdata)
{
size_t len = size * nmemb;
@ -576,6 +832,83 @@ void Http::parse_headers()
}
void Http::debug_function_append(pt::Space & debug, const char * prefix, const char * start, size_t part_size)
{
if( part_size > 0 )
{
debug.add(prefix);
std::string & last_table_str_item = debug.value.value_table.back()->value.value_string;
last_table_str_item.append(start, part_size);
}
}
void Http::debug_function_append(Http * http, const char * prefix, char * data, size_t size)
{
pt::Space & debug = *http->debug_info;
const char * start = data;
size_t part_size = 0;
size_t i = 0;
while( i < size )
{
if( data[i]== '\r' && i+1 < size && data[i+1]=='\n' )
{
debug_function_append(debug, prefix, start, part_size);
i += 2;
start = data + i;
part_size = 0;
}
else
if( data[i]== '\n' )
{
debug_function_append(debug, prefix, start, part_size);
i += 1;
start = data + i;
part_size = 0;
}
else
{
part_size += 1;
i += 1;
}
}
debug_function_append(debug, prefix, start, part_size);
}
int Http::debug_function(CURL * handle, curl_infotype type, char * data, size_t size, void * userdata)
{
Http * http = reinterpret_cast<Http*>(userdata);
if( http->debug_info )
{
switch(type)
{
case CURLINFO_TEXT:
debug_function_append(http, "", data, size);
break;
case CURLINFO_HEADER_IN:
debug_function_append(http, "Header received: ", data, size);
break;
case CURLINFO_HEADER_OUT:
debug_function_append(http, "Header sent: ", data, size);
break;
default:
break;
}
}
return 0;
}
}

View File

@ -59,12 +59,49 @@ public:
Http(const Http &) = delete;
Http(Http &&) = delete;
/*
* start a new request
* called automatically from cctor
* you have to call it manually to make another requests
*
*/
Http & begin();
/*
* add one header (should not end with \r\n or \n )
*
*/
Http & add_header(const pt::TextStream & header);
Http & add_header(const pt::WTextStream & header);
Http & add_header(const std::string & header);
Http & add_header(const std::wstring & header);
Http & add_header(const char * str, size_t len);
Http & add_header(const wchar_t * str, size_t len);
/*
* additional headers (we do not copy it too)
* headers can be separated with \r\n or just with \n
* last \n (or \r\n) can be omitted
* empty lines will be ignored
*
* headers from wchar_t* will be replaced to char* by using pt::wide_to_utf8
*/
Http & add_headers(const char * headers);
Http & add_headers(const wchar_t * headers);
Http & add_headers(const std::string & headers);
Http & add_headers(const std::wstring & headers);
/*
* we do not copy the space structure but only get a pointer to it
* so you have to preserve the structure until get()/put() is called
*/
Http & add_input_headers(pt::Space * headers);
Http & add_headers(pt::Space * headers);
Http & add_headers(pt::Space & headers);
/*
* output headers will be provided in a Space structure as key/value pairs (object)
@ -90,6 +127,15 @@ public:
void initialize_curl_if_needed();
void uninitialize_curl();
/*
* if true set additional information how the curl library works when connection is made
* debug_info will be a table in such a case
*
* if debug is false then debug_info can be nullptr (if not null then will be set to Space null)
*
*/
void use_debug_mode(bool debug, pt::Space * debug_info);
/*
* we do not copy the string but only get a pointer to its c_str()
* so you have to preserve the string until get()/put() is called
@ -124,12 +170,16 @@ private:
size_t read_function_index;
const std::string * read_function_input;
curl_slist * http_headers;
pt::Space * additional_headers_to_send;
pt::Space * additional_space_headers_to_send;
const char * additional_string_headers_to_send;
const wchar_t * additional_wstring_headers_to_send;
pt::Space * output_headers_space;
const wchar_t * bearer_token = nullptr;
pt::TextStream out_headers_stream;
bool change_header_names_to_lower;
std::wstring * output_content_type;
long debug_mode;
pt::Space * debug_info;
std::wstring temp_header;
std::string temp_header_ascii;
@ -139,18 +189,59 @@ private:
bool fetch_internal(const char * url, const std::string * in, pt::TextStream & out);
static size_t fetch_read_function(char * ptr, size_t size, size_t nmemb, void * userdata);
static int fetch_seek_set(Http * http, curl_off_t offset);
static int fetch_seek_cur(Http * http, curl_off_t offset);
static int fetch_seek_end(Http * http, curl_off_t offset);
static int fetch_seek_function(void * userdata, curl_off_t offset, int origin);
static size_t fetch_write_function(char * ptr, size_t size, size_t nmemb, void * userdata);
static size_t fetch_header_function(char * ptr, size_t size, size_t nmemb, void * userdata);
static void debug_function_append(pt::Space & debug, const char * prefix, const char * start, size_t part_size);
static void debug_function_append(Http * http, const char * prefix, char * data, size_t size);
static int debug_function(CURL * handle, curl_infotype type, char * data, size_t size, void * userdata);
void clear_tmp_objects();
void reset_headers();
void add_additional_headers();
void add_additional_space_headers();
void add_additional_string_headers();
void add_bearer_token();
void add_header(const pt::WTextStream & header);
void add_header(const std::wstring & header);
void skip_white(pt::TextStream::iterator & i);
void parse_header_name(pt::TextStream::iterator & i, std::string & name);
void parse_header_value(pt::TextStream::iterator & i, std::string & value);
void parse_headers();
template<typename StringType>
void add_additional_string_headers(StringType * str)
{
StringType * part_start = str;
size_t part_size = 0;
size_t i = 0;
while( str[i] != 0 )
{
if( str[i]== '\r' && str[i+1]=='\n' )
{
add_header(part_start, part_size);
i += 2;
part_start = str + i;
part_size = 0;
}
else
if( str[i]== '\n' )
{
add_header(part_start, part_size);
i += 1;
part_start = str + i;
part_size = 0;
}
else
{
part_size += 1;
i += 1;
}
}
add_header(part_start, part_size);
}
};