added: generating Content-Length header when text answer is sent to the client

added:   now we are able to use the html filter for the whole out_streams (when ajax is used the output is filtered too)
         splitted FilterCompressSend() function -- first we are making the filtering
         (after filtering we know the size of the content to send)
added:   to Compress:
         Compressing(const char * source, size_t source_len, BinaryPage & out_stream, int encoding);
changed: some refactoring in App




git-svn-id: svn://ttmath.org/publicrep/winix/trunk@942 e52654a7-88a9-db11-a3e9-0013d4bc506e
This commit is contained in:
Tomasz Sowa 2013-12-04 01:21:57 +00:00
parent c04874397b
commit ee9c68b04e
12 changed files with 213 additions and 129 deletions

View File

@ -339,7 +339,7 @@ void App::ProcessRequest()
log << log2 << config.log_delimiter << logend;
ProcessRequestThrow();
SaveSessionsIfNeeded(); // !! przerzucic to na watek sesji
SaveSessionsIfNeeded(); // !! IMPROVE ME move to the session's thread
system.load_avg.StopRequest();
}
@ -356,25 +356,48 @@ void App::ProcessRequest()
log << log1 << "App: there was an unknown exception" << logend;
}
ClearAfterRequest();
}
void App::ClearAfterRequest()
{
try
{
plugin.Call(WINIX_END_REQUEST);
}
catch(...)
{
log << log1 << "App: an exception when clearing after a request (exception from a plugin)" << logend;
log << log1 << "App: an exception from a plugin when clearing after a request" << logend;
}
// simple operations which should not throw an exception
templates.RequestEnd();
cur.request->Clear();
cur.session->ClearOnEndRequest();
cur.session = session_manager.GetTmpSession();
log << logendrequest;
try
{
// simple operations which should not throw an exception
json_out_stream.Clear();
templates.ClearAfterRequest();
cur.request->Clear();
cur.session->ClearAfterRequest();
cur.session = session_manager.GetTmpSession();
output_8bit.clear();
compressed_output.clear();
html_filtered.clear();
aheader_name.clear();
aheader_value.clear();
// send_data_buf doesn't have to be cleared and it is better to not clear it (optimizing)
log << logendrequest;
}
catch(...)
{
log << log1 << "App: an exception when clearing after a request" << logend;
}
}
void App::Start()
{
while( !synchro.was_stop_signal && FCGX_Accept_r(&fcgi_request) == 0 )
@ -475,12 +498,6 @@ bool sent = false;
{
templates.Generate();
}
if( cur.request->return_json )
{
CreateJSONAnswer();
}
}
@ -915,7 +932,7 @@ void App::PrepareHeadersCompression(int compress_encoding)
}
void App::PrepareHeadersNormal(Header header)
void App::PrepareHeadersNormal(Header header, size_t output_size)
{
switch( header )
{
@ -932,6 +949,9 @@ void App::PrepareHeadersNormal(Header header)
cur.request->out_headers.Add(L"Status", L"200 OK");
PrepareHeaderContentType();
}
if( output_size != static_cast<size_t>(-1) )
cur.request->out_headers.Add(L"Content-Length", output_size);
}
@ -986,7 +1006,7 @@ void App::SendCookies()
}
void App::PrepareHeaders(bool compressing, int compress_encoding, Header header)
void App::PrepareHeaders(bool compressing, int compress_encoding, Header header, size_t output_size)
{
PrepareSessionCookie();
@ -1009,7 +1029,7 @@ void App::PrepareHeaders(bool compressing, int compress_encoding, Header header)
}
else
{
PrepareHeadersNormal(header);
PrepareHeadersNormal(header, output_size);
}
if( compressing )
@ -1018,41 +1038,42 @@ void App::PrepareHeaders(bool compressing, int compress_encoding, Header header)
void App::FilterCompressSend(bool compressing, int compress_encoding, const std::wstring & source_ref)
void App::FilterContent()
{
const std::wstring * source = &source_ref;
Request & req = *cur.request;
bool raw = req.is_item && req.item.content_type == Item::ct_raw &&
req.status == WINIX_ERR_OK && req.function &&
(req.function == &functions.fun_cat || req.function == &functions.fun_run);
bool raw = cur.request->is_item && cur.request->item.content_type == Item::ct_raw && cur.request->status == WINIX_ERR_OK &&
cur.request->function && (cur.request->function == &functions.fun_cat || cur.request->function == &functions.fun_run);
size_t start = req.out_streams.size(); // default nothing should be filtered
size_t end = req.out_streams.size();
if( config.html_filter && cur.request->use_html_filter && !raw && !cur.request->return_json )
if( config.html_filter && !req.send_bin_stream && !raw )
{
TemplatesFunctions::html_filter.Filter(*source, clean_html);
source = &clean_html;
}
else
{
html_with_debug = *source;
source = &html_with_debug;
if( req.return_json )
{
if( !req.return_info_only )
{
start = 1;
}
}
else
{
start = 0;
end = 1;
}
}
if( config.utf8 )
PT::WideToUTF8(*source, source_a);
else
AssignString(*source, source_a);
// !! IMPROVE ME add to log the binary stream as well
if( config.log_server_answer )
log << log1 << "App: the server's answer is:\n" << source_a << "\nApp: end of the server's answer" << logend;
if( compressing )
compress.CompressAndPut(source_a.c_str(), source_a.length(), fcgi_request.out, compress_encoding);
else
FCGX_PutS(source_a.c_str(), fcgi_request.out);
if( cur.request->return_json )
json_out_stream.Clear();
for(size_t i=start ; i<end ; ++i)
{
if( req.use_html_filter[i] )
{
// !! IMPROVE ME may some kind of html_filtered.reserve() here? (optimization)
TemplatesFunctions::html_filter.Filter(req.out_streams[i].Str(), html_filtered);
req.out_streams[i].Str(std::move(html_filtered));
}
}
}
@ -1067,6 +1088,7 @@ int App::SelectDeflateVersion()
}
void App::SelectCompression(size_t source_len, bool & compression_allowed, int & compression_encoding)
{
compression_allowed = false;
@ -1114,7 +1136,7 @@ void App::SelectCompression(size_t source_len, bool & compression_allowed, int &
bool App::CanSendContent(Header header)
bool App::CanSendContent()
{
if( !cur.request->redirect_to.empty() || !cur.request->x_sendfile.empty() )
// if there is a redirect or a file to send then we do not send a content
@ -1132,22 +1154,10 @@ return true;
void App::SendTextAnswer()
App::Header App::GetHTTPStatusCode()
{
const std::wstring * source;
Header header = h_200;
Error status = cur.request->status;
bool compressing;
int compress_encoding;
if( cur.request->return_json )
source = &json_out_stream.Str();
else
source = &cur.request->out_streams[0].Str();
SelectCompression(source->length(), compressing, compress_encoding);
Error status = cur.request->status;
Header header = h_200;
if( status == WINIX_ERR_NO_ITEM || status == WINIX_ERR_NO_FUNCTION || status == WINIX_ERR_UNKNOWN_PARAM )
{
@ -1167,15 +1177,67 @@ int compress_encoding;
header = h_200;
}
PrepareHeaders(compressing, compress_encoding, header);
return header;
}
void App::SendTextAnswer()
{
const std::wstring * source;
bool compressing = false;
int compress_encoding = 0;
size_t output_size = 0;
Header header = GetHTTPStatusCode();
if( CanSendContent() )
{
FilterContent();
if( cur.request->return_json )
{
CreateJSONAnswer();
source = &json_out_stream.Str(); // json_out_stream was prepared by CreateJSONAnswer()
}
else
{
source = &cur.request->out_streams[0].Str();
}
SelectCompression(source->length(), compressing, compress_encoding);
if( config.utf8 )
PT::WideToUTF8(*source, output_8bit);
else
AssignString(*source, output_8bit);
// !! IMPROVE ME add to log the binary stream as well
if( config.log_server_answer )
log << log1 << "App: the server's answer is:\n" << output_8bit << "\nApp: end of the server's answer" << logend;
if( compressing )
{
compress.Compressing(output_8bit.c_str(), output_8bit.length(), compressed_output, compress_encoding);
output_size = compressed_output.size();
}
else
{
output_size = output_8bit.size();
}
}
PrepareHeaders(compressing, compress_encoding, header, output_size);
SendHeaders();
SendCookies();
FCGX_PutS("\r\n", fcgi_request.out);
if( CanSendContent(header) )
if( CanSendContent() )
{
// filtering (html), compressing and sending back to the web browser
FilterCompressSend(compressing, compress_encoding, *source);
if( compressing )
SendData(compressed_output, fcgi_request.out);
else
FCGX_PutStr(output_8bit.c_str(), output_8bit.size(), fcgi_request.out);
}
}
@ -1187,7 +1249,7 @@ void App::SendData(const BinaryPage & page, FCGX_Stream * out)
if( send_data_buf.size() != buf_size )
send_data_buf.resize(buf_size);
BinaryPage::const_iterator i = page.begin();
BinaryPage::const_iterator i = page.begin();
BinaryPage::const_iterator end = page.end();
while( i != end )
@ -1219,18 +1281,19 @@ int compress_encoding;
if( status == WINIX_ERR_PERMISSION_DENIED || status == WINIX_ERR_CANT_CHANGE_USER || status == WINIX_ERR_CANT_CHANGE_GROUP )
header = h_403;
// !! IMPROVE ME add header: content-size
PrepareHeaders(compressing, compress_encoding, header);
// !! IMPROVE ME add header: content-length (size from Item struct)
// warning: if someone changed a file on the disk (in the real os)
// then winix would send an incorrect content-lenght header,
// we are waiting for the fsck winix function to be implemented
PrepareHeaders(compressing, compress_encoding, header, static_cast<size_t>(-1));
if( CanSendContent(header) )
if( CanSendContent() )
{
if( compressing )
{
out_bin_stream_compressed.clear(); // !! IMPROVE ME add to the end of a request
compress.Compressing(source, out_bin_stream_compressed, compress_encoding);
SendData(out_bin_stream_compressed, fcgi_request.out);
out_bin_stream_compressed.clear();
compress.Compressing(source, compressed_output, compress_encoding);
SendData(compressed_output, fcgi_request.out);
}
else
{

View File

@ -87,7 +87,7 @@ public:
Functions functions;
// false at the beginning
// !! moze to do loggera dac?
// !! IMPROVE ME moze to do loggera dac?
bool stdout_is_closed;
@ -112,18 +112,18 @@ private:
CookieParser cookie_parser;
AcceptEncodingParser accept_encoding_parser;
Compress compress;
std::wstring clean_html, html_with_debug;
FCGX_Request fcgi_request;
int fcgi_socket;
Synchro synchro;
pthread_t signal_thread;
std::string url_to_fetch_on_exit;
std::string source_a;
std::string send_data_buf;
PT::SpaceToJSON json_generic_serializer;
TextStream<std::wstring> json_out_stream;
BinaryPage out_bin_stream_compressed;
std::string aheader_name, aheader_value;
std::wstring html_filtered;
std::string output_8bit;
BinaryPage compressed_output;
bool CheckAccessFromPlugins();
void ProcessRequestThrow();
@ -135,7 +135,7 @@ private:
void CheckPostRedirect();
void MakePage();
void Make();
void SaveSessionsIfNeeded(); // !! wywalic do menagera sesji??
void SaveSessionsIfNeeded(); // !! IMPROVE ME wywalic do menagera sesji??
void LogAccess();
void SendData(const BinaryPage & page, FCGX_Stream * out);
void CreateJSONAnswer();
@ -158,8 +158,9 @@ private:
void SetSubdomain();
Header GetHTTPStatusCode();
void PrepareSessionCookie();
void FilterCompressSend(bool compressing, int compress_encoding, const std::wstring & source_ref);
void FilterContent();
void SendHeaders();
void SendCookies();
bool PrepareHeadersStaticCreateResource(PT::WTextStream & out_path);
@ -169,11 +170,12 @@ private:
void PrepareHeadersRedirect();
void PrepareHeadersSendFile();
void PrepareHeadersCompression(int compress_encoding);
void PrepareHeadersNormal(Header header);
void PrepareHeaders(bool compressing, int compress_encoding, Header header);
void PrepareHeadersNormal(Header header, size_t output_size);
void PrepareHeaders(bool compressing, int compress_encoding, Header header, size_t output_size);
int SelectDeflateVersion();
void SelectCompression(size_t source_len, bool & compression_allowed, int & compression_encoding);
bool CanSendContent(Header header);
bool CanSendContent();
void ClearAfterRequest();
void LogUser(const char * msg, uid_t id);
void LogGroup(const char * msg, gid_t id, bool put_logend = true);
@ -187,7 +189,8 @@ private:
void CreateStaticTree();
// !! dodac do session managera?
// !! IMPROVE ME
// !! move to the session manager?
time_t last_sessions_save;
};

View File

@ -2,7 +2,7 @@
* This file is a part of Winix
* and is not publicly distributed
*
* Copyright (c) 2008-2012, Tomasz Sowa
* Copyright (c) 2008-2013, Tomasz Sowa
* All rights reserved.
*
*/
@ -153,7 +153,9 @@ return 0;
int Compress::MakeCompress(z_stream & strm, const char * source, size_t source_len, FCGX_Stream * out_stream, int encoding)
int Compress::MakeCompress(z_stream & strm, const char * source, size_t source_len, BinaryPage & out_stream, int encoding)
{
int ret, flush;
size_t have;
@ -180,7 +182,7 @@ size_t have;
have = buffer_max_len - strm.avail_out;
last_out_size += have;
FCGX_PutStr(buffer_out, have, out_stream);
out_stream.write(buffer_out, have);
}
while( strm.avail_out == 0 );
@ -203,6 +205,7 @@ return 0;
}
void Compress::CopyToInputBuffer(BinaryPage::const_iterator & i, size_t len)
{
for(size_t a=0 ; a<len ; ++a, ++i)
@ -331,12 +334,13 @@ void Compress::PutLog(size_t source_len, int encoding)
3 - not inited (use Init() first)
100 - unknown
*/
int Compress::CompressAndPut(const char * source, size_t source_len, FCGX_Stream * out_stream, int encoding)
int Compress::Compressing(const char * source, size_t source_len, BinaryPage & out_stream, int encoding)
{
int ret;
z_stream * pstrm;
last_out_size = 0;
out_stream.clear();
if( !ready_for_compress )
{
@ -344,6 +348,8 @@ z_stream * pstrm;
return 3;
}
// !! CHECK ME
// it is correct to immediately return? what about headers in the compressed page?
if( source_len == 0 )
return 0;
@ -379,6 +385,8 @@ z_stream * pstrm;
return 3;
}
// !! CHECK ME
// it is correct to immediately return? what about headers in the compressed page?
if( in.empty() )
return 0;

View File

@ -2,7 +2,7 @@
* This file is a part of Winix
* and is not publicly distributed
*
* Copyright (c) 2008-2012, Tomasz Sowa
* Copyright (c) 2008-2013, Tomasz Sowa
* All rights reserved.
*
*/
@ -11,7 +11,6 @@
#define headerfile_winix_core_compress
#include <cstring>
#include <fcgiapp.h>
#include <zlib.h>
#include "requesttypes.h"
@ -34,7 +33,7 @@ public:
1 - deflate
2 - gzip
*/
int CompressAndPut(const char * source, size_t source_len, FCGX_Stream * out_stream, int encoding = 2);
int Compressing(const char * source, size_t source_len, BinaryPage & out_stream, int encoding = 2);
int Compressing(const BinaryPage & in, BinaryPage & out, int encoding = 2);
@ -45,7 +44,7 @@ private:
bool InitDeflate();
bool InitGzip();
int MakeCompress(z_stream & strm, const char * source, size_t source_len, FCGX_Stream * out_stream, int encoding);
int MakeCompress(z_stream & strm, const char * source, size_t source_len, BinaryPage & out_stream, int encoding);
int MakeCompress(z_stream & strm, const BinaryPage & page, BinaryPage & out, int encoding);
z_stream * SelectStream(int encoding);
void ResetStream(z_stream * pstrm, int encoding);

View File

@ -40,9 +40,13 @@ void Request::ClearOutputStreams()
len = 16;
out_streams.resize(len);
use_html_filter.resize(len);
for(size_t i=0 ; i<out_streams.size() ; ++i)
{
out_streams[i].Clear();
use_html_filter[i] = true;
}
}
@ -65,7 +69,6 @@ void Request::Clear()
out_cookies.Clear();
page_generated = false;
use_html_filter = true;
env_request_method = &char_empty;
env_request_uri = &char_empty;

View File

@ -139,7 +139,6 @@ struct Request
FunctionBase * function;
// parameters (name:value)
// !! CHECK ME may it should be changed to PT::Space?
ParamTab param_tab;
// this is a pointer either to the item (if exists) or to the last directory
@ -163,12 +162,10 @@ struct Request
// !! CHANGE ME it'll be better to use ordinary http result codes
Error status;
// if not empty means an address for redirecting to
// it should be url-encoded
std::wstring redirect_to;
// a redirect type
// following redirect types are supported:
// 300 Multiple Choices
@ -197,22 +194,24 @@ struct Request
// default: false
//
//
//
// winix answer send to the client's browser
// |
// |
// depending on send_bin_stream
// -------------------------------------------------
// (if false) ------------------------------------------------- (if true)
// | |
// text answer binary answer
// | |
// depending on return_json sending out_bin_stream
// ------------------------------------
// (if false) ------------------------------------ (if true)
// | |
// normal request ajax request
// | |
// sending out_streams[0] depending on return_info_only
// ------------------------------------------------------
// sending out_streams[0] |
// |
// |
// depending on return_info_only
// (if false) ------------------------------------------------------ (if true)
// | |
// generating JSON object from: generating JSON object only from info
// out_streams and info, e.g.: e.g.:
@ -227,24 +226,9 @@ struct Request
//
bool send_bin_stream;
// -------------------------------------------------------------------------------------
// binary answer
//
// binary page sent to the client if send_bin_stream is true
BinaryPage out_bin_stream;
//
// -------------------------------------------------------------------------------------
// -------------------------------------------------------------------------------------
// text answer
//
// when returning the text answer we can either return the whole html page (normal requests)
// or a JSON object (for requests generated from AJAX)
// if return_json is false then we return the whole html page (which is in out_streams[0])
@ -277,17 +261,19 @@ struct Request
// default: null (json_generic_serializer used)
PT::SpaceToJSON * info_serializer;
//
// -------------------------------------------------------------------------------------
// if set to true then the standard template system will not be generated
// if set to true then the standard template system will not be used
// default: false
bool page_generated;
// whether or not the html filter should be used
// default: true
bool use_html_filter;
// the size of the table is the same as out_streams
// default: all items true
std::vector<bool> use_html_filter;
// if this variable is true then winix always return 200 OK header

View File

@ -2,7 +2,7 @@
* This file is a part of Winix
* and is not publicly distributed
*
* Copyright (c) 2008-2012, Tomasz Sowa
* Copyright (c) 2008-2013, Tomasz Sowa
* All rights reserved.
*
*/
@ -86,7 +86,7 @@ void Session::Clear(bool clear_plugin_data)
// clearing some variables when a request is ended (just for safety)
void Session::ClearOnEndRequest()
void Session::ClearAfterRequest()
{
// ip_ban list can be sorted by SessionManager (in the special thread)
ip_ban = 0;

View File

@ -2,7 +2,7 @@
* This file is a part of Winix
* and is not publicly distributed
*
* Copyright (c) 2008-2012, Tomasz Sowa
* Copyright (c) 2008-2013, Tomasz Sowa
* All rights reserved.
*
*/
@ -32,7 +32,7 @@ struct Session
void SetTimesTo(time_t time);
void Clear(bool clear_plugin_data = true);
void ClearOnEndRequest();
void ClearAfterRequest();
// 0 - means that there is a temporary session

View File

@ -2,7 +2,7 @@
* This file is a part of Winix
* and is not publicly distributed
*
* Copyright (c) 2010-2012, Tomasz Sowa
* Copyright (c) 2010-2013, Tomasz Sowa
* All rights reserved.
*
*/
@ -42,6 +42,9 @@ public:
const StringType & Str() const;
const CharType * CStr() const;
void Str(const StringType & str);
void Str(const StringType && str);
CharType operator[](size_t index);
TextStream & operator<<(const char * str);
@ -71,6 +74,7 @@ public:
TextStream & write(const char * buf, size_t len); // for compatibility with standard library (Ezc uses it)
TextStream & write(const wchar_t * buf, size_t len);
protected:
StringType buffer;
@ -118,6 +122,21 @@ const typename TextStream<StringType>::CharType * TextStream<StringType>::CStr()
}
template<class StringType>
void TextStream<StringType>::Str(const StringType & str)
{
buffer = str;
}
template<class StringType>
void TextStream<StringType>::Str(const StringType && str)
{
buffer = str;
}
template<class StringType>
typename TextStream<StringType>::CharType TextStream<StringType>::operator[](size_t index)
{
@ -345,5 +364,8 @@ return *this;
}
#endif

View File

@ -468,8 +468,8 @@ void Rm::CreateJSON(bool status)
else
cur->request->out_streams[0] << "[false]\n";
cur->request->page_generated = true;
cur->request->use_html_filter = false;
cur->request->page_generated = true;
cur->request->use_html_filter[0] = false;
}

View File

@ -857,7 +857,7 @@ using namespace TemplatesFunctions;
// clearing at the end of a request
void Templates::RequestEnd()
void Templates::ClearAfterRequest()
{
using namespace TemplatesFunctions;

View File

@ -586,7 +586,7 @@ public:
void SetSessionManager(SessionManager * psession_manager);
void Init();
void RequestEnd();
void ClearAfterRequest();
void ReadTemplates();
void ReadNewIndexTemplates();
void ReadNewChangeTemplates();