/*
 * This file is a part of Winix
 * and is distributed under the 2-Clause BSD licence.
 * Author: Tomasz Sowa <t.sowa@ttmath.org>
 */

/* 
 * Copyright (c) 2008-2022, Tomasz Sowa
 * All rights reserved.
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 * 
 * 1. Redistributions of source code must retain the above copyright notice,
 * this list of conditions and the following disclaimer.
 * 
 * 2. Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 * 
 */

#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include "request.h"
#include "log.h"
#include "misc.h"
#include "functions/functionbase.h"
#include "templates/templates.h"
#include "convert/misc.h"
#include "templates/templates.h"
#include "jobtask.h"


namespace Winix
{





Request::Request()
{
	id = 0;
	config = nullptr;
	templates = nullptr;
	compress = nullptr;
	plugin = nullptr;
	mounts = nullptr;

	// set function to nullptr because is used in Clear()
	function = nullptr;
}


void Request::fields()
{
	field(L"", L"dirs",    				dir_tab);
	field(L"", L"is_item", 				is_item);
	field(L"", L"http_status",			http_status);

	field(L"", L"current_dir", 			&Request::current_dir);
	field(L"", L"last_item",			&Request::last_item_wrapper);
	field(L"", L"http_status_error_title", &Request::http_status_error_title);
	field(L"", L"http_status_error_description", &Request::http_status_error_description);
}



void Request::SetConfig(Config * config)
{
	this->config = config;
}


void Request::SetTemplates(Templates * templates)
{
	this->templates = templates;
}


void Request::SetCompress(Compress * compress)
{
	this->compress = compress;
}


void Request::SetPlugin(Plugin * plugin)
{
	this->plugin = plugin;
}


void Request::SetMounts(Mounts * mounts)
{
	this->mounts = mounts;
}



void Request::ClearOutputStreams()
{
size_t len = 0;

	out_main_stream.clear();

	if( config )
		len = config->ezc_out_streams_size;

	/*
	 *  clearing buffers and setting 'escape' flag to true
	 *  for all streams which were used in the map
	 */
	out_streams.ClearMap();
	out_streams.ResizeTab(len);
}


/*
 * we do not clear config, templates, compress, plugin and mounts pointers
 */
void Request::Clear()
{
	id = -1;
	run_state = RunState::not_assigned;

	RemovePostFileTmp(post_file_tab);
	ClearOutputStreams();

	if( function )
		function->Clear();

	post_file_tab.clear();
	cookie_tab.clear();
	post_in.clear();

	method = unknown_method;

	headers_in.clear();

	out_headers.clear();
	out_cookies.clear();

	env_request_method.clear();
	env_request_uri.clear();
	env_http_cookie.clear();
	env_http_host.clear();
	env_http_user_agent.clear();
	env_http_accept_encoding.clear();
	env_http_accept.clear();
	env_http_accept_language.clear();
	env_fcgi_role.clear();
	env_content_type.clear();
	env_https.clear();

	item_tab.clear();
	item.Clear();
	item.set_connector(nullptr);
	dir_tab.clear();
	last_item = &item;
	is_item = false;
	function = nullptr;
	session = nullptr;
	mount = nullptr;
	param_tab.clear();
	anchor.clear();

	send_bin_stream = false;
	send_main_stream = false;
	send_all_frames = false;
	send_frames.clear();

	use_ezc_engine = false;
	serialize_models = false;

	accept_mime_types.clear();
	container_type = ContainerType::container_raw;
	
	status = WINIX_ERR_OK;
	http_status = Header::status_200_ok;

	browser_msie = false;
	redirect_to.clear();
	redirect_type = 303;
	x_sendfile.clear();
	send_as_attachment = false;

	using_ssl = false;
	is_htmx_request = false;

	start_time = 0;
	start_date.Clear();

	timespec_req_start.tv_sec = 0;
	timespec_req_start.tv_nsec = 0;
	timespec_req_stop.tv_sec = 0;
	timespec_req_stop.tv_nsec = 0;
	timespec_req_diff.tv_sec = 0;
	timespec_req_diff.tv_nsec = 0;

	timespec_ezc_engine_start.tv_sec = 0;
	timespec_ezc_engine_start.tv_nsec = 0;
	timespec_ezc_engine_stop.tv_sec = 0;
	timespec_ezc_engine_stop.tv_nsec = 0;

	subdomain.clear();
	models.Clear();

	out_bin_stream.clear();

	gen_trim_white			= false;
	gen_skip_new_line		= false;
	gen_use_special_chars	= false;

	ip = 0;
	ip_str.clear();
	use_200_status_for_not_found_and_permission_denied = false;

	html_template.clear();
	use_html_filter = false;

	job_id = JobTask::JOB_ID_DEFAULT;
	job_second_id = JobTask::JOB_ID_DEFAULT;
	job.clear();

	accept_gzip = false;
	accept_deflate = false;

	output_8bit.clear();
	compressed_output.clear();
	output_tmp_filtered_stream.clear();
	serialized_model.clear();
	aheader_name.clear();
	aheader_value.clear();
	cookie_id_string.clear();
	send_data_buf.clear();
	http_header_name.clear();
}



void Request::RequestStarts()
{
	// clearing it is better to use at the end of a request
	// so starting is much faster
	clock_gettime(CLOCK_REALTIME, &timespec_req_start);
	timespec_req_stop = timespec_req_start;

	start_time = timespec_req_start.tv_sec;
	start_date = start_time;
}



void Request::RequestEnds()
{
	clock_gettime(CLOCK_REALTIME, &timespec_req_stop);
	calculate_timespec_diff(timespec_req_start, timespec_req_stop, timespec_req_diff);
}



void Request::PrepareAnswerType()
{
	CheckAcceptHeader();
	CheckContainerParameter();

	serialize_models = (container_type != ContainerType::container_raw);
	send_all_frames = (ParamValuep(config->request_all_frames_parameter) != nullptr);
	send_main_stream = (ParamValuep(config->request_main_stream_parameter) != nullptr);
	use_html_filter = config->html_filter;
	PrepareFrameNames();

	if( container_type == ContainerType::container_raw && !send_all_frames && send_frames.empty() )
	{
		send_main_stream = true;
	}

	use_ezc_engine = send_main_stream || send_all_frames || !send_frames.empty();
}


void Request::CheckAcceptHeader()
{
	if( !accept_mime_types.empty() )
	{
		bool found = false;

		for(HeaderValue & h: accept_mime_types)
		{
			if( h.value == Header::text_html || h.value == Header::application_xhtml_xml ||
				h.value == Header::text_all || h.value == Header::all_all)
			{
				container_type = ContainerType::container_raw;
				found = true;
				break;
			}
			else
			if( h.value == Header::application_json || h.value == Header::application_all )
			{
				container_type = ContainerType::container_json;
				found = true;
				break;
			}
			else
			if( h.value == Header::application_xml )
			{
				container_type = ContainerType::container_xml;
				found = true;
				break;
			}
			else
			if( h.value == Header::text_csv )
			{
				container_type = ContainerType::container_csv;
				found = true;
				break;
			}
		}

		if( !found )
		{
			Log * log = get_logger();

			if( log )
			{
				(*log) << log2 << "Request: an unknown " << Header::accept << " headers: ";
				HeaderValue::log_values(accept_mime_types, *log);
				(*log) << " (skipping)" << logend;
			}
		}
	}
}



void Request::CheckContainerParameter()
{
	std::wstring * container = ParamValuep(L"container");

	if( container )
	{
		// IMPROVEME do a plugin call here
		// if a plugin can consume this then don't check text/json/xml/csv and just return true

		if( *container == L"raw" )
		{
			container_type = ContainerType::container_raw;
		}
		else
		if( *container == L"json" )
		{
			container_type = ContainerType::container_json;
		}
		else
		if( *container == L"xml" )
		{
			container_type = ContainerType::container_xml;
		}
		else
		if( *container == L"csv" )
		{
			container_type = ContainerType::container_csv;
		}
		else
		{
			Log * log = get_logger();

			if( log )
			{
				(*log) << log2 << "Request: an unknown container url parameter: " << *container << " (skipping)" << logend;
			}
		}
	}
}


void Request::PrepareFrameNames()
{
	Config * config = get_config();
	Log * log = get_logger();

	if( config && log )
	{
		const std::wstring & frame = ParamValue(config->request_frame_parameter);

		if( frame.size() <= config->request_frame_parameter_max_length )
		{
			send_frames.clear();
			slice_by(frame, ',', send_frames);
			std::sort(send_frames.begin(), send_frames.end());
			auto frames_end = std::unique(send_frames.begin(), send_frames.end());
			send_frames.erase(frames_end, send_frames.end());

			if( send_frames.size() > config->request_frame_parameter_max_frames )
			{
				send_frames.clear();
				(*log) << log2 << "Request: the number of frames exceeds " << config->request_frame_parameter_max_frames
					   << " (skipping frames)" << logend;
			}
		}
		else
		{
			(*log) << log2 << "Request: the length of the frame url parameter exceeds " << config->request_frame_parameter_max_length
				   << " characters (skiping frames)" << logend;
		}
	}
}



// add such a method to Space
pt::Space * Request::AddPostVar(pt::Space & space, const wchar_t * name)
{
	pt::Space * space_value = space.get_space(name);
	pt::Space * new_space = nullptr;
	Log * log = get_logger();

	if( space_value )
	{
		if( space_value->is_table() )
		{
			if( space_value->table_size() < WINIX_POSTTABLE_VALUE_TABLE_MAXSIZE )
			{
				new_space = &space_value->add_empty_space();
			}
			else
			{
				if( log )
				{
					(*log) << log1 << "Request: more than " << WINIX_POSTTABLE_VALUE_TABLE_MAXSIZE << " post variables in a table " << name << " (skipping)" << logend;
				}
			}
		}
		else
		{
			pt::Space new_table;
			new_table.add(std::move(*space_value));
			new_space = &new_table.add_empty_space();
			space_value->set(std::move(new_table));
		}
	}
	else
	{
		if( space.object_size() < WINIX_POSTTABLE_MAXSIZE )
		{
			new_space = &space.add_empty_space(name);
		}
		else
		{
			if( log )
			{
				(*log) << log1 << "Request: more than " << WINIX_POSTTABLE_MAXSIZE << " post variables (skipping)" << logend;
			}
		}
	}

	return new_space;
}



bool Request::AddPostVar(const wchar_t * name, const wchar_t * value)
{
	pt::Space * new_space = AddPostVar(post_in, name);

	if( new_space )
	{
		new_space->set(value);
	}

	return new_space != nullptr;
}


bool Request::AddPostVar(const std::wstring & name, const std::wstring & value)
{
	return AddPostVar(name.c_str(), value.c_str());
}


bool Request::IsPostVar(const wchar_t * var)
{
	return post_in.has_key(var);
}


bool Request::IsPostVar(const std::wstring & var)
{
	return post_in.has_key(var);
}


const std::wstring & Request::PostVar(const wchar_t * var)
{
	std::wstring * value = post_in.get_wstr(var);

	if( value )
		return *value;

	return str_empty;
}


const std::wstring & Request::PostVar(const std::wstring & var)
{
	return PostVar(var.c_str());
}



bool Request::PostVar(const wchar_t * var, std::wstring & result)
{
	std::wstring * value = post_in.get_wstr(var);
	bool found = false;

	if( value )
	{
		result = *value;
		found = true;
	}
	else
	{
		result.clear();
	}

	return found;
}


bool Request::PostVar(const std::wstring & var, std::wstring & result)
{
	return PostVar(var.c_str(), result);
}



std::wstring * Request::PostVarp(const wchar_t * var)
{
	return post_in.get_wstr(var);
}


std::wstring * Request::PostVarp(const std::wstring & var)
{
	return post_in.get_wstr(var.c_str());
}


std::wstring * Request::ParamValuep(const wchar_t * param_name)
{
	ParamTab::iterator i;

	for(i=param_tab.begin() ; i!=param_tab.end() ; ++i)
	{
		if( i->name == param_name )
			return &i->value;
	}

	return nullptr;
}

std::wstring * Request::ParamValuep(const std::wstring & param_name)
{
	ParamTab::iterator i;

	for(i=param_tab.begin() ; i!=param_tab.end() ; ++i)
	{
		if( i->name == param_name )
			return &i->value;
	}

	return nullptr;
}



bool Request::IsParam(const wchar_t * param_name)
{
	return ParamValuep(param_name) != nullptr;
}


bool Request::IsParam(const std::wstring & param_name)
{
	return ParamValuep(param_name) != nullptr;
}


const std::wstring & Request::ParamValue(const wchar_t * param_name)
{
	const std::wstring * val = ParamValuep(param_name);

	if( val != nullptr )
		return *val;
	
	return str_empty;
}


const std::wstring & Request::ParamValue(const std::wstring & param_name)
{
	const std::wstring * val = ParamValuep(param_name);

	if( val != nullptr )
		return *val;

	return str_empty;
}



void Request::AddParam(const std::wstring & param_name, const std::wstring & param_value)
{
	bool found = false;
	ParamTab::iterator i;

	for(i=param_tab.begin() ; i!=param_tab.end() ; ++i)
	{
		if( i->name == param_name )
		{
			i->value = param_value;
			found = true;
		}
	}

	if( !found )
	{
		Param param;
		param.name = param_name;
		param.value = param_value;
		param_tab.push_back(param);
	}
}


void Request::AddParam(const wchar_t * param_name, const wchar_t * param_value)
{
	bool found = false;
	ParamTab::iterator i;

	for(i=param_tab.begin() ; i!=param_tab.end() ; ++i)
	{
		if( i->name == param_name )
		{
			i->value = param_value;
			found = true;
		}
	}

	if( !found )
	{
		Param param;
		param.name = param_name;
		param.value = param_value;
		param_tab.push_back(param);
	}
}


void Request::RemoveParam(const wchar_t * param_name)
{
	for(size_t i=0 ; i < param_tab.size() ; )
	{
		if( param_tab[i].name == param_name )
		{
			param_tab.erase(param_tab.begin() + i);
		}
		else
		{
			++i;
		}
	}
}


void Request::RemoveParam(const std::wstring & param_name)
{
	for(size_t i=0 ; i < param_tab.size() ; )
	{
		if( param_tab[i].name == param_name )
		{
			param_tab.erase(param_tab.begin() + i);
		}
		else
		{
			++i;
		}
	}
}




void Request::current_dir(morm::Wrapper & wrapper)
{
	wrapper.model = dir_tab.back();
}


void Request::last_item_wrapper(morm::Wrapper & wrapper)
{
	wrapper.model = last_item;
}


bool Request::has_frame(const wchar_t * frame)
{
	for(std::wstring & f: send_frames)
	{
		if( f == frame )
			return true;
	}

	return false;
}


bool Request::has_frame(const std::wstring & frame)
{
	for(std::wstring & f: send_frames)
	{
		if( f == frame )
			return true;
	}

	return false;
}


void Request::http_status_error_title(EzcEnv & env)
{
	pt::WTextStream str;
	str << L"http_error_" << http_status << L"_title";

	const std::wstring & msg = TemplatesFunctions::locale.Get(str);
	env.out << msg;
}


void Request::http_status_error_description(EzcEnv & env)
{
	pt::WTextStream str;
	str << L"http_error_" << http_status << L"_msg";

	const std::wstring & msg = TemplatesFunctions::locale.Get(str);
	env.out << msg;
}


void Request::create_job(long job_id, long job_secondary_id)
{
	this->job_id = job_id;
	this->job_second_id = job_secondary_id;
	run_state = RunState::assigned_to_job;
}




/*
 * will be removed in the future
 */
void Request::modify_status_code_if_needed()
{
	// moved from Templates when a pattern was selected
	switch( status )
	{
		case WINIX_ERR_INCORRECT_URI: // !!temporarily
		case WINIX_ERR_INTERNAL_ERROR: // !! temprarily
		case WINIX_ERR_PERMISSION_DENIED:
		case WINIX_ERR_CANT_CHANGE_USER:
		case WINIX_ERR_CANT_CHANGE_GROUP:
		case WINIX_ERR_CANT_CHANGE_PRIVILEGES:
			http_status = Header::status_403_forbidden;
			break;

		case WINIX_ERR_NO_ITEM:
		case WINIX_ERR_NO_FUNCTION:
		case WINIX_ERR_UNKNOWN_PARAM:
			http_status = Header::status_404_not_found;
			break;
	}

	if( use_200_status_for_not_found_and_permission_denied && (
			http_status == Header::status_404_not_found ||
			http_status == Header::status_403_forbidden
		))
	{
		http_status = Header::status_200_ok;
		Log * plog = get_logger();

		if( plog )
		{
			(*plog) << log3 << "Request: changing the http response to: 200 OK" << logend;
		}
	}
}



void Request::ModifyStatusForRedirect()
{
	switch(redirect_type)
	{
	case 300:
		http_status = Header::status_300_multiple_choices;
		break;

	case 301:
		http_status = Header::status_301_moved_permanently;
		break;

	case 302:
		http_status = Header::status_302_found;
		break;

	case 307:
		http_status = Header::status_307_temporary_redirect;
		break;

	case 303:
	default:
		http_status = Header::status_303_see_other;
		break;
	}
}





// may rename to something like PrepareAndSendAnswer()?
void Request::PrepareAndSendAnswer()
{
	output_8bit.clear();
	compressed_output.clear();

	if( !fcgi_request.out )
	{
		Log * plog = get_logger();

		if( plog )
		{
			(*plog) << "Request: I cannot send the answer - fcgi_request.out is null (internal error)" << logend;
			return;
		}
	}

	if( !redirect_to.empty() || !x_sendfile.empty() || method == Request::options )
	{
		Send8bitOutput(output_8bit); // send empty content
		return;
	}

	plugin->Call(WINIX_CONTENT_MAKE);

	if( use_ezc_engine )
	{
		UseEzcGenerator();
	}

	if( container_type == Request::ContainerType::container_raw && send_bin_stream )
	{
		Send8bitOutput(out_bin_stream);
		return;
	}
	else
	if( container_type == Request::ContainerType::container_raw )
	{
		PrepareRawAnswer();
	}
	else
	if( container_type == Request::ContainerType::container_json )
	{
		PrepareJsonAnswer();
	}
	else
	if( container_type == Request::ContainerType::container_xml )
	{
		PrepareXmlAnswer();
	}
	else
	if( container_type == Request::ContainerType::container_csv )
	{
		PrepareCsvAnswer();
	}

	Send8bitOutput(output_8bit);
}


void Request::PrepareRawAnswer()
{
	if( send_main_stream )
	{
		FilterHtmlIfNeeded(out_main_stream.get_buffer(), output_8bit, false);
	}
	else
	if( send_all_frames )
	{
		SerializeAllFrames();
	}
	else
	if( !send_frames.empty() )
	{
		SerializeSpecificFrames();
	}
}


void Request::PrepareJsonAnswer()
{
	output_8bit << '{';
	PrepareContenerizedAnswer();
	output_8bit << '}';
}


void Request::PrepareXmlAnswer()
{
	output_8bit << '<';
	pt::esc_to_xml(config->xml_root, output_8bit);
	output_8bit << '>';

	PrepareContenerizedAnswer();

	output_8bit << "</";
	pt::esc_to_xml(config->xml_root, output_8bit);
	output_8bit << '>';
}


void Request::PrepareCsvAnswer()
{
	PrepareContenerizedAnswer();
}


void Request::PrepareContenerizedAnswer()
{
	bool put_separator = false;

	if( serialize_models )
	{
		SerializeModels();
		put_separator = true;
	}

	if( send_bin_stream )
	{
		PutSeparatorIfNeeded(put_separator);

		// IMPLEMENT ME serialize binary stream as base64 and put in 'bin_stream' field
		pt::WTextStream str;
		str << "NOT IMPLEMENTED YET";
		SerializeStream(str, config->bin_stream_field.c_str());
		put_separator = true;
	}

	if( send_main_stream )
	{
		PutSeparatorIfNeeded(put_separator);
		SerializeStream(out_main_stream.get_buffer(), config->main_stream_field.c_str());
		put_separator = true;
	}

	if( send_all_frames || !send_frames.empty() )
	{
		PutSeparatorIfNeeded(put_separator);
		SerializeFieldJson(config->ezc_frames_field.c_str());
		output_8bit << "{";

		if( send_all_frames )
		{
			SerializeAllFrames();
		}
		else
		if( !send_frames.empty() )
		{
			SerializeSpecificFrames();
		}

		output_8bit << "}";
		put_separator = true;
	}
}


void Request::PutSeparatorIfNeeded(bool put_separator)
{
	if( put_separator )
	{
		switch( container_type )
		{
		case Request::ContainerType::container_json:
			output_8bit << ",";
			break;

		case Request::ContainerType::container_xml:
			break;

		case Request::ContainerType::container_csv:
			output_8bit << ";";
			break;

		case Request::ContainerType::container_raw:
		default:
			break;
		}
	}
}


void Request::SerializeFieldJson(const wchar_t * field_name)
{
	if( field_name )
	{
		output_8bit << '"';
		pt::esc_to_json(field_name, output_8bit);
		output_8bit << "\":";
	}
}



void Request::SerializeStream(const pt::WTextStream & input_stream, const wchar_t * field_name)
{
	switch( container_type )
	{
	case Request::ContainerType::container_json:
		SerializeStreamJson(input_stream, field_name);
		break;

	case Request::ContainerType::container_xml:
		SerializeStreamXml(input_stream, field_name);
		break;

	case Request::ContainerType::container_csv:
		SerializeStreamCsv(input_stream, field_name);
		break;

	case Request::ContainerType::container_raw:
	default:
		FilterHtmlIfNeeded(input_stream, output_8bit, false);
		break;
	}
}


void Request::SerializeStreamJson(const pt::WTextStream & input_stream, const wchar_t * field_name)
{
	SerializeFieldJson(field_name);
	output_8bit << '"';

	if( config->html_filter && use_html_filter )
	{
		TemplatesFunctions::html_filter.filter(input_stream, output_tmp_filtered_stream, true);
		pt::esc_to_json(output_tmp_filtered_stream, output_8bit);
	}
	else
	{
		pt::esc_to_json(input_stream, output_8bit);
	}

	output_8bit << '"';
}


void Request::SerializeStreamXml(const pt::WTextStream & input_stream, const wchar_t * field_name)
{
	if( field_name )
	{
		output_8bit << '<';
		pt::esc_to_xml(field_name, output_8bit);
		output_8bit << '>';
	}

	if( config->html_filter && use_html_filter )
	{
		TemplatesFunctions::html_filter.filter(input_stream, output_tmp_filtered_stream, true);
		pt::esc_to_xml(output_tmp_filtered_stream, output_8bit);
	}
	else
	{
		pt::esc_to_xml(input_stream, output_8bit);
	}

	if( field_name )
	{
		output_8bit << "</";
		pt::esc_to_xml(field_name, output_8bit);
		output_8bit << '>';
	}
}


void Request::SerializeStreamCsv(const pt::WTextStream & input_stream, const wchar_t * field_name)
{
	if( field_name )
	{
		output_8bit << '"';
		pt::esc_to_csv(field_name, output_8bit);
		output_8bit << "\";";
	}

	output_8bit << '"';

	if( config->html_filter && use_html_filter )
	{
		TemplatesFunctions::html_filter.filter(input_stream, output_tmp_filtered_stream, true);
		pt::esc_to_csv(output_tmp_filtered_stream, output_8bit);
	}
	else
	{
		pt::esc_to_csv(input_stream, output_8bit);
	}

	output_8bit << "\";\n";
}


void Request::SerializeAllFrames()
{
	auto i = out_streams.streams_map.begin();
	bool is_first = true;

	for( ; i != out_streams.streams_map.end() ; ++i)
	{
		if( container_type == Request::ContainerType::container_json && !is_first )
		{
			output_8bit << ',';
		}

		if( container_type == Request::ContainerType::container_xml && i->first.empty() )
		{
			Log * plog = get_logger();

			if( plog )
			{
				(*plog) << log2 << "Request: I cannot serialize a frame with an empty name to xml (frame skipped)" << logend;
			}
		}
		else
		{
			SerializeStream(i->second->get_buffer(), i->first.c_str());
		}

		is_first = false;
	}
}


void Request::SerializeSpecificFrames()
{
	bool is_first = true;

	for(std::wstring & frame: send_frames)
	{
		auto i = out_streams.streams_map.find(frame);

		if( i != out_streams.streams_map.end() )
		{
			if( container_type == Request::ContainerType::container_json && !is_first )
			{
				output_8bit << ',';
			}

			SerializeStream(i->second->get_buffer(), frame.c_str());
			is_first = false;
		}
		else
		{
			Log * plog = get_logger();

			if( plog )
			{
				(*plog) << log2 << "Request: there is no such a frame: " << frame << logend;
			}
		}
	}
}


void Request::SerializeModels()
{
	Ezc::Models::ModelsMap models_map = models.GetMap();
	auto i = models_map.begin();
	bool is_first = true;

	for( ; i != models_map.end() ; ++i)
	{
		if( container_type == Request::ContainerType::container_json && !is_first )
		{
			output_8bit << ',';
		}

		if( container_type == Request::ContainerType::container_xml && i->first.empty() )
		{
			Log * plog = get_logger();

			if( plog )
			{
				(*plog) << log2 << "Request: I cannot serialize a model with an empty name to xml (model skipped)" << logend;
			}
		}
		else
		{
			SerializeModel(i->second, i->first.c_str());
		}

		is_first = false;
	}
}


void Request::SerializeModel(morm::Wrapper & wrapper, const wchar_t * field_name)
{
	switch( container_type )
	{
	case Request::ContainerType::container_json:
		SerializeModelJson(wrapper, field_name);
		break;

	case Request::ContainerType::container_xml:
		SerializeModelXml(wrapper, field_name);
		break;

	case Request::ContainerType::container_csv:
		SerializeModelCsv(wrapper, field_name);
		break;

	case Request::ContainerType::container_raw:
	default:
		break;
	}
}


void Request::SerializeModelJson(morm::Wrapper & wrapper, const wchar_t * field_name)
{
	SerializeFieldJson(field_name);

	if( wrapper.model )
	{
		serialized_model.clear();
		wrapper.model->set_connector(model_connector);
		wrapper.model->to_text(serialized_model);
		output_8bit << serialized_model;
	}

	if( wrapper.date )
	{
		output_8bit << '"';
		wrapper.date->SerializeISO(output_8bit);
		output_8bit << '"';
	}

	if( wrapper.space_wrapper )
	{
		wrapper.space_wrapper->get_space()->serialize_to_json_stream(output_8bit, false);
	}

	if( wrapper.model_container_wrapper )
	{
		wrapper.model_container_wrapper->set_iterator_at_first_model();
		bool is_first = true;
		output_8bit << '[';

		while( wrapper.model_container_wrapper->is_iterator_correct() )
		{
			if( !is_first )
				output_8bit << ',';

			morm::Model * model = wrapper.model_container_wrapper->get_model();
			serialized_model.clear();
			model->set_connector(model_connector);
			model->to_text(serialized_model);
			output_8bit << serialized_model;

			wrapper.model_container_wrapper->increment_iterator();
			is_first = false;
		}

		output_8bit << ']';
	}
}


void Request::SerializeModelXml(morm::Wrapper & wrapper, const wchar_t * field_name)
{
	// 	IMPROVEME
	Log * plog = get_logger();

	if( plog )
	{
		(*plog) << log2 << "Request: serializing models to xml not implemented yet" << logend;
	}
}


void Request::SerializeModelCsv(morm::Wrapper & wrapper, const wchar_t * field_name)
{
	// 	IMPROVEME
	Log * plog = get_logger();

	if( plog )
	{
		(*plog) << log2 << "Request: serializing models to csv not implemented yet" << logend;
	}
}




// IMPROVEME
// gime me a better name
void Request::FilterHtmlIfNeeded(const pt::WTextStream & input_stream, BinaryPage & output, bool clear_stream)
{
	if( config->html_filter && use_html_filter )
	{
		TemplatesFunctions::html_filter.filter(input_stream, output, clear_stream);
	}
	else
	{
		pt::wide_stream_to_utf8(input_stream, output, clear_stream);
	}
}



void Request::Send8bitOutput(BinaryPage & output)
{
	bool compressing = false;
	int compress_encoding = 0;
	size_t output_size = 0;
	Log * plog = get_logger();

	SelectCompression(output.size(), compressing, compress_encoding);

	if( config->log_server_answer )
	{
		(*plog) << log1 << "Request: the server's answer is:\n" << output << "\nRequest: end of the server's answer" << logend;
	}

	if( compressing )
	{
		compress->Compressing(output, compressed_output, compress_encoding);
		output_size = compressed_output.size();
	}
	else
	{
		output_size = output.size();
	}

	PrepareHeaders(compressing, compress_encoding, output_size);
	SendHeaders();
	SendCookies();
	FCGX_PutS("\r\n", fcgi_request.out);

	if( CanSendContent() )
	{
		if( compressing )
			SendData(compressed_output, fcgi_request.out);
		else
			SendData(output, fcgi_request.out);
	}
}



// IMPROVEME
// we can send directly from BinaryPage without copying to a temporary buffer
// (but there is no an interface in BinaryPage yet)
void Request::SendData(const BinaryPage & page, FCGX_Stream * out)
{
	const size_t buf_size = 4096;

	if( send_data_buf.size() != buf_size )
		send_data_buf.resize(buf_size);

	BinaryPage::const_iterator i   = page.begin();
	BinaryPage::const_iterator end = page.end();

	while( i != end )
	{
		size_t s = 0;

		for( ; i != end && s < buf_size ; ++i, ++s)
			send_data_buf[s] = *i;

		if( s > 0 )
			FCGX_PutStr(send_data_buf.c_str(), s, out);
	}
}





// !! IMPROVE ME change to a better name
void Request::UseEzcGenerator()
{
//	if( page_generated || !redirect_to.empty() || !x_sendfile.empty() )
//		return;


	clock_gettime(CLOCK_REALTIME, &timespec_ezc_engine_start);

	templates->SetEzcParameters(	gen_trim_white,
								gen_skip_new_line,
								gen_use_special_chars);

	templates->Generate();
	clock_gettime(CLOCK_REALTIME, &timespec_ezc_engine_stop);

	timespec diff;
	calculate_timespec_diff(timespec_ezc_engine_start, timespec_ezc_engine_stop, diff);

	pt::TextStream str;
	timespec_to_stream_with_unit(diff, str); // IMPROVEME in the future Log can be used directly

	Log * plog = get_logger();

	if( plog )
	{
		(*plog) << log3 << "Request: ezc engine took: " << str << logend;
	}
}



int Request::SelectDeflateVersion()
{
	if( browser_msie )
		return 0; // raw deflate
	else
		return 1; // deflate
}



void Request::SelectCompression(size_t source_len, bool & compression_allowed, int & compression_encoding)
{
	compression_allowed  = false;
	compression_encoding = 0;

	if(	config->compression		&&
		redirect_to.empty()		&&
		x_sendfile.empty()		&&
		!browser_konqueror		&&  /* IMPROVEME check whether the Konqueror can use deflate algorithm */
		source_len >= config->compression_page_min_size )
	{
		// IMPROVEME put the constants somewhere (1, 2, 10, 20)
		if( config->compression_encoding == 1 || config->compression_encoding == 10 )
		{
			if( accept_deflate )
			{
				compression_allowed  = true;
				compression_encoding = SelectDeflateVersion();
			}
			else
			if( config->compression_encoding == 10 && accept_gzip )
			{
				compression_allowed  = true;
				compression_encoding = 2; // gzip
			}
		}

		if( config->compression_encoding == 2 || config->compression_encoding == 20 )
		{
			if( accept_gzip )
			{
				compression_allowed  = true;
				compression_encoding = 2; // gzip
			}
			else
			if( config->compression_encoding == 20 && accept_deflate )
			{
				compression_allowed  = true;
				compression_encoding = SelectDeflateVersion();
			}
		}
	}
}




void Request::PrepareSessionCookie()
{
	SessionManager * session_manager = get_session_manager();

	if( !session || session->id==0 )
		return;

	if( config->session_cookie_encode )
	{
		if( !session_manager->EncodeSessionId(session->id, session->id_index, cookie_id_string) )
			Toa(session->id, cookie_id_string);
	}
	else
	{
		Toa(session->id, cookie_id_string);
	}


	if( !session->puser || !session->remember_me )
	{
		AddDefaultSessionCookie(cookie_id_string);
	}
	else
	{
		pt::Date expires = start_time + config->session_remember_max_idle;
		AddDefaultSessionCookie(cookie_id_string, &expires);
	}
}



void Request::PrepareHeaders(bool compressing, int compress_encoding, size_t output_size)
{
	PrepareSessionCookie();

	if( send_as_attachment )
	{
		AddHeader(L"Content-Disposition", L"attachment");
	}

	if( !redirect_to.empty() )
	{
		ModifyStatusForRedirect();
		AddHeader(L"Location", redirect_to);

		Log * plog = get_logger();

		if( plog )
		{
			(*plog) << log2 << "Request: redirect to: " << redirect_to << logend;
		}
	}
	else
	if( mount && mount->type == mounts->MountTypeStatic() )
	{
		PrepareSendFileHeaderForStaticMountpoint();
	}
	else
	if( !x_sendfile.empty() )
	{
		PrepareSendFileHeader();
	}
	else
	{
		PrepareContentLengthHeader(output_size);
	}

	if( compressing )
	{
		PrepareContentEncodingHeader(compress_encoding);
	}

	PrepareHeaderStatus(http_status);
	PrepareHeaderContentType();
}




bool Request::AddHeader(const wchar_t * name, const wchar_t * value)
{
	if( !out_headers.has_key(name) )
	{
		out_headers.add(name, value);
		return true;
	}

return false;
}


bool Request::AddHeader(const wchar_t * name, long value)
{
	if( !out_headers.has_key(name) )
	{
		out_headers.add(name, value);
		return true;
	}

return false;
}



bool Request::AddHeader(const std::wstring & name, const std::wstring & value)
{
	if( !out_headers.has_key(name) )
	{
		out_headers.add(name, value);
		return true;
	}

return false;
}


bool Request::AddHeader(const wchar_t * name, const pt::WTextStream & value)
{
	if( !out_headers.has_key(name) )
	{
		out_headers.add_stream(name, value);
		return true;
	}

return false;
}


bool Request::AddHeader(const std::wstring & name, const pt::WTextStream & value)
{
	if( !out_headers.has_key(name) )
	{
		out_headers.add_stream(name, value);
		return true;
	}

return false;
}


void Request::PrepareSendFileHeaderForStaticMountpoint()
{
	Log * plog = get_logger();

	if( mount )
	{
		if( PathHasUpDir(env_request_uri) )
		{
			if( plog )
			{
				(*plog) << log1 << "Request: incorrect path for a static file" << logend;
			}

			http_status = Header::status_403_forbidden;
			return;
		}

		const std::wstring & index_str = mount->FirstArg(mounts->MountParStatic());
		size_t index = Toi(index_str);

		if( index >= config->static_dirs.size() )
		{
			if( plog )
			{
				(*plog) << log1 << "Request: static dir with index " << index << " is not defined in the config" << logend;
			}

			http_status = Header::status_403_forbidden;
			return;
		}

		pt::WTextStream path;
		path << config->static_dirs[index] << L"/";

		if( !CreateStaticResourcePath(path) )
		{
			http_status = Header::status_403_forbidden;
			return;
		}

		/*
		 * FIX ME now we can send full path (apache, lighttpd) and relative path (nginx)
		 * but this feature for mounting static content probably will be removed
		 */
		if( AddHeader(config->send_file_header, path) )
		{
			if( plog )
			{
				(*plog) << log2 << "Request: sending a file from a static mountpoint: " << path << logend;
			}
		}
	}
}


void Request::PrepareSendFileHeader()
{
	Log * plog = get_logger();

	if( AddHeader(config->send_file_header, x_sendfile) )
	{
		if( plog )
		{
			(*plog) << log2 << "Request: sending file: " << x_sendfile << logend;
		}
	}
}


void Request::PrepareContentEncodingHeader(int compress_encoding)
{
	if( compress_encoding == 0 || compress_encoding == 1 )
	{
		AddHeader(L"Content-Encoding", L"deflate");
	}
	else
	{
		AddHeader(L"Content-Encoding", L"gzip");
	}
}


void Request::PrepareContentLengthHeader(size_t output_size)
{
	if( output_size != static_cast<size_t>(-1) )
	{
		pt::WTextStream buf;
		buf << output_size;
		AddHeader(L"Content-Length", buf);
	}
}


void Request::PrepareHeaderContentType()
{
	if( !out_headers.has_key(Winix::Header::content_type) )
	{
		if( container_type == Request::ContainerType::container_json )
		{
			out_headers.add(Winix::Header::content_type, Winix::Header::application_json_utf8);
		}
		else
		if( container_type == Request::ContainerType::container_xml )
		{
			out_headers.add(Winix::Header::content_type, Winix::Header::application_xml_utf8);
		}
		else
		if( container_type == Request::ContainerType::container_csv )
		{
			out_headers.add(Winix::Header::content_type, Winix::Header::text_csv_utf8);
		}
		else
		if( container_type == Request::ContainerType::container_raw )
		{
			if( send_bin_stream )
			{
				out_headers.add(Winix::Header::content_type, Winix::Header::application_octet_stream);
			}
			else
			{
				switch( config->content_type_header )
				{
					case 1:
					out_headers.add(Winix::Header::content_type, Winix::Header::application_xhtml_xml_utf8);
					break;

					case 2:
					out_headers.add(Winix::Header::content_type, Winix::Header::application_xml_utf8);
					break;

					case 0:
					default:
					out_headers.add(Winix::Header::content_type, Winix::Header::text_html_utf8);
				}
			}
		}
	}
}


void Request::PrepareHeaderStatus(int http_status)
{
	pt::WTextStream value;
	Header::prepare_status_value(http_status, value, false);
	AddHeader(L"Status", value);

	Log * plog = get_logger();

	if( plog )
	{
		(*plog) << log2 << "Request: http status: " << value << logend;
	}
}


// we can improve SendHeaders and SendCookies methods by checking
// whether there is a new line character in either a name or a value
// and if such character exists and is being sent to the client it breaks the http headers and content
// and if compression is enabled the client's browser will not be able to decompress the stream
void Request::SendHeaders()
{
	pt::Space::ObjectType::iterator i;
	pt::Space & headers = out_headers;
	Log * plog = get_logger();

	if( headers.is_object() )
	{
		plugin->Call(WINIX_PREPARE_TO_SEND_HTTP_HEADERS, &headers);

		for(i=headers.value.value_object.begin() ; i != headers.value.value_object.end() ; ++i)
		{
			bool header_prepared = false;

			if( i->second->is_wstr() )
			{
				pt::wide_to_utf8(i->first, aheader_name);
				pt::wide_to_utf8(*i->second->get_wstr(), aheader_value);
				header_prepared = true;
			}
			else
			if( i->second->is_long_long() )
			{
				pt::wide_to_utf8(i->first, aheader_name);
				pt::Toa(*i->second->get_long_long(), aheader_value);
				header_prepared = true;
			}
			else
			{
				if( plog )
				{
					(*plog) << log2 << "Skipping HTTP Header: " << i->first << " - it's neither a wstr nor a long long" << logend;
				}
			}

			if( header_prepared )
			{
				FCGX_PutS(aheader_name.c_str(), 	fcgi_request.out);
				FCGX_PutS(": ", 					fcgi_request.out);
				FCGX_PutS(aheader_value.c_str(), 	fcgi_request.out);
				FCGX_PutS("\r\n", 					fcgi_request.out);

				if( config->log_http_answer_headers && plog )
					(*plog) << log1 << "HTTP Header: " << aheader_name << ": " << aheader_value << logend;
			}
		}
	}
}



void Request::SendCookies()
{
	pt::Space::ObjectType::iterator i;
	pt::Space & cookies = out_cookies;
	Log * plog = get_logger();

	if( cookies.is_object() )
	{
		plugin->Call(WINIX_PREPARE_TO_SEND_HTTP_COOKIES, &cookies);

		for(i=cookies.value.value_object.begin() ; i != cookies.value.value_object.end() ; ++i)
		{
			if( i->second->is_wstr() )
			{
				pt::wide_to_utf8(i->first,     aheader_name);
				pt::wide_to_utf8(*i->second->get_wstr(), aheader_value);

				FCGX_PutS("Set-Cookie: ", 			fcgi_request.out);
				FCGX_PutS(aheader_name.c_str(), 	fcgi_request.out);
				FCGX_PutS("=", 						fcgi_request.out);
				FCGX_PutS(aheader_value.c_str(), 	fcgi_request.out);
				FCGX_PutS("\r\n", 					fcgi_request.out);

				if( config->log_http_answer_headers && plog )
					(*plog) << log1 << "HTTP Header: Set-Cookie: " << aheader_name << "=" << aheader_value << logend;
			}
			else
			{
				if( plog )
				{
					(*plog) << log2 << "Skipping Cookie: " << i->first << " - it's not a wstr" << logend;
				}
			}
		}
	}
}



bool Request::CreateStaticResourcePath(pt::WTextStream & out_path)
{
	bool status = false;
	size_t i = 0;
	Log * plog = get_logger();
	Dirs * dirs = get_dirs();

	if( dirs && mount )
	{
		Item * dir = dirs->GetDir(mount->dir_id);

		if( dir )
		{
			size_t how_many_dirs = dirs->DirLevel(dir->id);
			const wchar_t * path = SkipDirs(env_request_uri.c_str(), how_many_dirs);

			// the path begins with a slash only if how_many_dirs is zero
			while( *path == '/' )
				path += 1;

			while( path[i]!=0 && path[i]!='?' && path[i]!='#' )
				++i;

			if( i > 0 )
				out_path.write(path, i);

			status = true;
		}
		else
		{
			if( plog )
			{
				(*plog) << log1 << "Request: cannot find the mount directory" << logend;
			}
		}
	}

	return status;
}


bool Request::CanSendContent()
{
	if( !x_sendfile.empty() )
	{
		// if there is a file to send then we do not send a content
		return false;
	}

	if( !redirect_to.empty() )
	{
		// if there is a redirect and no json is requred then we do not send the content
		return false;
	}

	if( method == Request::head || method == Request::options )
	{
		return false;
	}

	return true;
}



void Request::LogRequestTime()
{
	pt::TextStream str;
	timespec_to_stream_with_unit(timespec_req_diff, str);

	Log * plog = get_logger();

	if( plog )
	{
		(*plog) << log2 << "Request: request took: " << str << logend;
	}
}


void Request::SetEnv(const char * name, std::wstring & env)
{
	const char * v = FCGX_GetParam(name, fcgi_request.envp);

	if( v )
	{
		pt::utf8_to_wide(v, env);
	}
}


/*
 * IMPROVE ME take it from cur.request.headers_in?
 */
void Request::ReadEnvVariables()
{
	SetEnv("REQUEST_METHOD", 		env_request_method);
	SetEnv("REQUEST_URI", 			env_request_uri);
	SetEnv("FCGI_ROLE", 			env_fcgi_role);
	SetEnv("CONTENT_TYPE", 			env_content_type);
	SetEnv("HTTPS", 				env_https);

	SetEnv("HTTP_HOST", 			env_http_host);
	SetEnv("HTTP_USER_AGENT", 		env_http_user_agent);
	SetEnv("HTTP_COOKIE", 			env_http_cookie);
	SetEnv("HTTP_ACCEPT_ENCODING", 	env_http_accept_encoding);
	SetEnv("HTTP_ACCEPT", 			env_http_accept);
	SetEnv("HTTP_ACCEPT_LANGUAGE",	env_http_accept_language);
}


void Request::ReadEnvRemoteIP()
{
	const char * v = nullptr;

	if( config && config->check_proxy_ip_header )
	{
		http_header_8bit = "HTTP_";
		pt::wide_to_utf8(config->proxy_ip_header, http_header_8bit, false);
		pt::to_upper_emplace(http_header_8bit);
		v = FCGX_GetParam(http_header_8bit.c_str(), fcgi_request.envp);
	}
	else
	{
		v = FCGX_GetParam("REMOTE_ADDR", fcgi_request.envp);
	}

	if( v )
	{
		ip = (int)inet_addr(v);
		pt::utf8_to_wide(v, ip_str);
	}
}


Request::Method Request::CheckRequestMethod(const wchar_t * name)
{
	Method method = Request::unknown_method;

	if( pt::is_equal_nc(name, L"GET") )
		method = Request::get;
	else
	if( pt::is_equal_nc(name, L"HEAD") )
		method = Request::head;
	else
	if( pt::is_equal_nc(name, L"POST") )
		method = Request::post;
	else
	if( pt::is_equal_nc(name, L"PUT") )
		method = Request::put;
	else
	if( pt::is_equal_nc(name, L"DELETE") )
		method = Request::delete_;
	else
	if( pt::is_equal_nc(name, L"CONNECT") )
		method = Request::connect;
	else
	if( pt::is_equal_nc(name, L"OPTIONS") )
		method = Request::options;
	else
	if( pt::is_equal_nc(name, L"TRACE") )
		method = Request::trace;
	else
	if( pt::is_equal_nc(name, L"PATCH") )
		method = Request::patch;

	return method;
}


void Request::CheckRequestMethod()
{
	method = CheckRequestMethod(env_request_method.c_str());
}


void Request::CheckSSL()
{
	// !! CHECK ME
	// value "on" exists in lighttpd server
	// make sure that for other servers is "on" too

	if( config && config->assume_connection_is_through_ssl )
		using_ssl = true;
	else
	if( pt::is_equal_nc(env_https.c_str(), L"on") )
		using_ssl = true;
}


void Request::SetSubdomain()
{
	if( config )
	{
		CreateSubdomain(config->base_url.c_str(), env_http_host.c_str(), subdomain);
	}
}



void Request::PutMethodName(Request::Method method, pt::Stream & stream)
{
	switch(method)
	{
	case get:
		stream << L"GET";
		break;

	case head:
		stream << L"HEAD";
		break;

	case post:
		stream << L"POST";
		break;

	case put:
		stream << L"PUT";
		break;

	case delete_:
		stream << L"DELETE";
		break;

	case connect:
		stream << L"CONNECT";
		break;

	case options:
		stream << L"OPTIONS";
		break;

	case trace:
		stream << L"TRACE";
		break;

	case patch:
		stream << L"PATCH";
		break;

	default:
		stream << L"UNKNOWN";
		break;
	}
}


void Request::PutMethodName(pt::Stream & stream)
{
	PutMethodName(method, stream);
}



void Request::AddCookie(
		const std::wstring & name,
		const std::wstring * value_string,
		const pt::Stream * value_stream,
		pt::Date * expires,
		const std::wstring * path,
		const std::wstring * domain,
		CookieSameSite cookie_same_site,
		bool http_only,
		bool secure)
{
	pt::WTextStream cookie;
	prepare_cookie_string(cookie, value_string, value_stream, expires, path, domain, cookie_same_site, http_only, secure);
	out_cookies.add_stream(name, cookie);
}


void Request::AddDefaultSessionCookie(const std::wstring & value, pt::Date * expires)
{
	AddCookie(
		config->session_cookie_name,
		&value,
		nullptr,
		expires,
		&config->session_cookie_path,
		&config->session_cookie_domain,
		static_cast<CookieSameSite>(config->session_cookie_same_site),
		config->session_cookie_http_only,
		config->session_cookie_secure
	);
}


void Request::FinishRequest()
{
	modify_status_code_if_needed(); // will be removed
	PrepareAndSendAnswer();
	RequestEnds();
	LogRequestTime();

	if( session )
	{
		session->allow_to_delete = true;
	}

	if( plugin )
	{
		plugin->Call(WINIX_END_REQUEST, this);
	}

	run_state = RunState::finished;
	FCGX_Finish_r(&fcgi_request);

	if( templates )
	{
		templates->ClearAfterRequest();
	}
}


} // namespace Winix