added support for gzip compression

new config option: compression_encoding (integer)
 1  - use deflate if available (or raw deflate for Internet Explorer) or don't compress
 2  - use gzip if available or don't compress
 10 - prefer deflate -- use deflate (or raw deflate for IE) if both deflate and gzip are available
 20 - prefer gzip    -- use gzip if both deflate and gzip are available
 default: 20

		   


git-svn-id: svn://ttmath.org/publicrep/winix/trunk@727 e52654a7-88a9-db11-a3e9-0013d4bc506e
This commit is contained in:
Tomasz Sowa 2011-04-02 00:10:16 +00:00
parent 1b60935d08
commit aadf12c7b3
9 changed files with 316 additions and 86 deletions

View File

@ -2,7 +2,7 @@
* This file is a part of Winix
* and is not publicly distributed
*
* Copyright (c) 2008-2010, Tomasz Sowa
* Copyright (c) 2008-2011, Tomasz Sowa
* All rights reserved.
*
*/

View File

@ -2,7 +2,7 @@
* This file is a part of Winix
* and is not publicly distributed
*
* Copyright (c) 2008-2010, Tomasz Sowa
* Copyright (c) 2008-2011, Tomasz Sowa
* All rights reserved.
*
*/

View File

@ -2,7 +2,7 @@
* This file is a part of Winix
* and is not publicly distributed
*
* Copyright (c) 2008-2010, Tomasz Sowa
* Copyright (c) 2008-2011, Tomasz Sowa
* All rights reserved.
*
*/
@ -24,24 +24,60 @@ public:
}
bool AcceptGzip()
{
return accept_gzip;
}
void ParseAndLog(const char * str)
{
Parse(str);
if( accept_deflate || accept_gzip )
{
log << log3 << "AEP: ";
if( accept_deflate )
{
log << "accept deflate";
if( accept_gzip )
log << ", ";
}
if( accept_gzip )
log << "accept gzip";
log << logend;
}
}
private:
void Init()
{
accept_deflate = false;
accept_gzip = false;
}
void Param(const std::string & param, double q)
{
if( param=="deflate" && q!=0 )
if( param=="deflate" && q!=0.0 )
{
accept_deflate = true;
log << log3 << "AEP: accept deflate" << logend;
}
if( param=="gzip" && q!=0.0 )
{
accept_gzip = true;
}
}
bool accept_deflate;
bool accept_gzip;
};

View File

@ -2,7 +2,7 @@
* This file is a part of Winix
* and is not publicly distributed
*
* Copyright (c) 2010, Tomasz Sowa
* Copyright (c) 2010-2011, Tomasz Sowa
* All rights reserved.
*
*/
@ -385,7 +385,7 @@ void App::ReadRequest()
ReadGetPostVars();
cookie_parser.Parse(cur.request->env_http_cookie, cur.request->cookie_tab);
accept_encoding_parser.Parse(cur.request->env_http_accept_encoding);
accept_encoding_parser.ParseAndLog(cur.request->env_http_accept_encoding);
CheckIE();
CheckKonqueror();
@ -540,7 +540,7 @@ void App::PrepareSessionCookie()
void App::SendHeaders(bool compressing, Header header)
void App::SendHeaders(bool compressing, int compress_encoding, Header header)
{
PrepareSessionCookie();
@ -590,8 +590,13 @@ void App::SendHeaders(bool compressing, Header header)
}
}
if( compressing )
FCGX_PutS("Content-Encoding: deflate\r\n", fcgi_request.out);
if( compressing )
{
if( compress_encoding == 0 || compress_encoding == 1 )
FCGX_PutS("Content-Encoding: deflate\r\n", fcgi_request.out);
else
FCGX_PutS("Content-Encoding: gzip\r\n", fcgi_request.out);
}
FCGX_PutS(cur.request->headers.CStr(), fcgi_request.out);
FCGX_PutS("\r\n", fcgi_request.out);
@ -610,14 +615,12 @@ void App::SetHtmlFilterConf()
}
// !! kopiowanie tych stringow bedzie zmienione
// gdy bedziemy korzystac w przyszlosci z wlasnego stringstream
void App::FilterCompressSend(bool compressing, const std::wstring & source_ref)
void App::FilterCompressSend(bool compressing, int compress_encoding, const std::wstring & source_ref)
{
const std::wstring * source = &source_ref;
bool raw = cur.request->is_item && cur.request->item.content_type == Item::ct_raw && cur.request->status == WINIX_ERR_OK &&
cur.request->function && (cur.request->function == &functions.fun_cat || cur.request->function == &functions.fun_run);
cur.request->function && (cur.request->function == &functions.fun_cat || cur.request->function == &functions.fun_run);
if( config.html_filter && !raw )
{
@ -633,32 +636,76 @@ void App::FilterCompressSend(bool compressing, const std::wstring & source_ref)
source = &html_with_debug;
}
// !! zrobic z tym porzadek
std::string temp;
Ezc::WideToUTF8(*source, temp);
if( config.utf8 )
Ezc::WideToUTF8(*source, source_a);
else
AssignString(*source, source_a);
if( compressing )
compress.CompressAndPut(temp.c_str(), temp.length(), fcgi_request.out);
compress.CompressAndPut(source_a.c_str(), source_a.length(), fcgi_request.out, compress_encoding);
else
FCGX_PutS(temp.c_str(), fcgi_request.out);
FCGX_PutS(source_a.c_str(), fcgi_request.out);
}
bool App::IsCompressionAllowed(const std::wstring & source)
int App::SelectDeflateVersion()
{
return( config.compression &&
cur.request->role == Request::responder &&
cur.request->redirect_to.empty() &&
cur.request->x_sendfile.empty() &&
!cur.request->browser_msie &&
!cur.request->browser_konqueror &&
accept_encoding_parser.AcceptDeflate() &&
source.size() >= (size_t)config.compression_page_min_size );
if( cur.request->browser_msie )
return 0; // raw deflate
else
return 1; // deflate
}
void App::SelectCompression(size_t source_len, bool & compression_allowed, int & compression_encoding)
{
compression_allowed = false;
compression_encoding = 0;
if( config.compression &&
cur.request->role == Request::responder &&
cur.request->redirect_to.empty() &&
cur.request->x_sendfile.empty() &&
!cur.request->browser_konqueror && /* !! sprawdzic czy Konqueror bedzie obslugiwal raw deflate */
source_len >= config.compression_page_min_size )
{
if( config.compression_encoding == 1 || config.compression_encoding == 10 )
{
if( accept_encoding_parser.AcceptDeflate() )
{
compression_allowed = true;
compression_encoding = SelectDeflateVersion();
}
else
if( config.compression_encoding == 10 && accept_encoding_parser.AcceptGzip() )
{
compression_allowed = true;
compression_encoding = 2; // gzip
}
}
if( config.compression_encoding == 2 || config.compression_encoding == 20 )
{
if( accept_encoding_parser.AcceptGzip() )
{
compression_allowed = true;
compression_encoding = 2; // gzip
}
else
if( config.compression_encoding == 20 && accept_encoding_parser.AcceptDeflate() )
{
compression_allowed = true;
compression_encoding = SelectDeflateVersion();
}
}
}
}
bool App::CanSendContent(Header header)
{
if( !cur.request->redirect_to.empty() || !cur.request->x_sendfile.empty() )
@ -698,9 +745,12 @@ void App::AddDebugInfo(std::wstring & out)
void App::SendAnswer()
{
const std::wstring & source = cur.request->page.Str();
Header header = h_200;
bool compressing = IsCompressionAllowed(source);
Error status = cur.request->status;
Header header = h_200;
Error status = cur.request->status;
bool compressing;
int compress_encoding;
SelectCompression(source.length(), compressing, compress_encoding);
if( status == WINIX_ERR_NO_ITEM || status == WINIX_ERR_NO_FUNCTION || status == WINIX_ERR_UNKNOWN_PARAM )
header = h_404;
@ -708,12 +758,12 @@ Error status = cur.request->status;
if( status == WINIX_ERR_PERMISSION_DENIED || status == WINIX_ERR_CANT_CHANGE_USER || status == WINIX_ERR_CANT_CHANGE_GROUP )
header = h_403;
SendHeaders(compressing, header);
SendHeaders(compressing, compress_encoding, header);
if( CanSendContent(header) )
{
// filtering (html), compressing (deflate) and sending back to the web browser
FilterCompressSend(compressing, source);
// filtering (html), compressing and sending back to the web browser
FilterCompressSend(compressing, compress_encoding, source);
}
}

View File

@ -2,7 +2,7 @@
* This file is a part of Winix
* and is not publicly distributed
*
* Copyright (c) 2010, Tomasz Sowa
* Copyright (c) 2010-2011, Tomasz Sowa
* All rights reserved.
*
*/
@ -131,7 +131,7 @@ private:
Synchro synchro;
pthread_t signal_thread;
std::string url_to_fetch_on_exit;
std::string source_a;
void ProcessRequestThrow();
void ProcessRequest();
@ -156,9 +156,10 @@ private:
void SetHtmlFilterConf();
void PrepareSessionCookie();
void AddDebugInfo(std::wstring & out);
void FilterCompressSend(bool compressing, const std::wstring & source_ref);
void SendHeaders(bool compressing, Header header);
bool IsCompressionAllowed(const std::wstring & source);
void FilterCompressSend(bool compressing, int compress_encoding, const std::wstring & source_ref);
void SendHeaders(bool compressing, int compress_encoding, Header header);
int SelectDeflateVersion();
void SelectCompression(size_t source_len, bool & compression_allowed, int & compression_encoding);
bool CanSendContent(Header header);
void LogUser(const char * msg, uid_t id);

View File

@ -2,7 +2,7 @@
* This file is a part of Winix
* and is not publicly distributed
*
* Copyright (c) 2008-2010, Tomasz Sowa
* Copyright (c) 2008-2011, Tomasz Sowa
* All rights reserved.
*
*/
@ -15,9 +15,13 @@
Compress::Compress()
{
buffer = 0;
buffer_max_len = 65536; // 64KB
buffer = 0;
buffer_max_len = 65536; // 64KB
ready_for_compress = false;
compress_level = 6;
raw_deflate_inited = false;
deflate_inited = false;
gzip_inited = false;
}
@ -25,6 +29,15 @@ Compress::~Compress()
{
if( buffer )
delete [] buffer;
if( raw_deflate_inited )
deflateEnd(&strm_raw_deflate);
if( deflate_inited )
deflateEnd(&strm_deflate);
if( gzip_inited )
deflateEnd(&strm_gzip);
}
@ -33,7 +46,6 @@ bool Compress::AllocateMemory()
if( buffer )
delete [] buffer;
try
{
buffer = new char[buffer_max_len];
@ -41,7 +53,6 @@ bool Compress::AllocateMemory()
catch(const std::bad_alloc &)
{
log << log1 << "Compress: can't allocate memory" << logend;
buffer = 0;
return false;
}
@ -50,39 +61,93 @@ return true;
}
bool Compress::InitRawDeflate()
{
raw_deflate_inited = false;
strm_raw_deflate.next_in = 0;
strm_raw_deflate.zalloc = Z_NULL;
strm_raw_deflate.zfree = Z_NULL;
strm_raw_deflate.opaque = Z_NULL;
int ret = deflateInit2(&strm_raw_deflate, compress_level, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY);
if( ret != Z_OK )
log << log1 << "Compress: problem with deflateInit2() for raw deflate" << logend;
else
raw_deflate_inited = true;
return ret == Z_OK;
}
bool Compress::InitDeflate()
{
deflate_inited = false;
strm_deflate.next_in = 0;
strm_deflate.zalloc = Z_NULL;
strm_deflate.zfree = Z_NULL;
strm_deflate.opaque = Z_NULL;
int ret = deflateInit2(&strm_deflate, compress_level, Z_DEFLATED, 15, 8, Z_DEFAULT_STRATEGY);
if( ret != Z_OK )
log << log1 << "Compress: problem with deflateInit2() for deflate" << logend;
else
deflate_inited = true;
return ret == Z_OK;
}
bool Compress::InitGzip()
{
gzip_inited = false;
strm_gzip.next_in = 0;
strm_gzip.zalloc = Z_NULL;
strm_gzip.zfree = Z_NULL;
strm_gzip.opaque = Z_NULL;
int ret = deflateInit2(&strm_gzip, compress_level, Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY);
if( ret != Z_OK )
log << log1 << "Compress: problem with deflateInit2() for gzip" << logend;
else
gzip_inited = true;
return ret == Z_OK;
}
/*
return:
0 - ok;
1 - can't allocate memory
100 - unknown
0 - ok
1 - can't allocate memory
100 - unknown error
*/
int Compress::Init(int compress_level)
int Compress::Init(int compress_level_)
{
compress_level = compress_level_;
if( buffer == 0 )
if( !AllocateMemory() )
return 1;
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
int ret = deflateInit(&strm, compress_level);
if( ret != Z_OK )
log << log1 << "Compress: problem with deflateInit()" << logend;
if( ret == Z_MEM_ERROR )
return 1;
if( ret != Z_OK )
return 100;
ready_for_compress = true;
if( InitRawDeflate() && InitDeflate() && InitGzip() )
ready_for_compress = true;
else
return 100;
return 0;
}
int Compress::MakeCompress(const char * source, size_t source_len, FCGX_Stream * out_stream)
int Compress::MakeCompress(z_stream & strm, const char * source, size_t source_len, FCGX_Stream * out_stream, int encoding)
{
int ret, flush;
size_t have;
@ -133,6 +198,62 @@ return 0;
z_stream * Compress::SelectStream(int encoding)
{
z_stream * pstrm;
if( encoding == 0 )
pstrm = &strm_raw_deflate;
else
if( encoding == 1 )
pstrm = &strm_deflate;
else
pstrm = &strm_gzip;
return pstrm;
}
void Compress::ResetStream(z_stream * pstrm, int encoding)
{
if( deflateReset(pstrm) != Z_OK )
{
log << log1 << "Compress: problem with deflateReset()" << logend;
deflateEnd(pstrm);
if( encoding == 0 )
InitRawDeflate();
else
if( encoding == 1 )
InitDeflate();
else
InitGzip();
}
}
void Compress::PutLog(size_t source_len, int encoding)
{
double ratio = 100.0 - (double(last_out_size) / double(source_len) * 100.0);
char buffer[30];
sprintf(buffer, "%.1f", ratio);
log << log2 << "Compress: ";
if( encoding == 0 )
log << "raw deflate";
else
if( encoding == 1 )
log << "deflate";
else
log << "gzip";
log << ", original size: " << source_len << ", size after compressing: "
<< (int)last_out_size << ", ratio: " << buffer << "%" << logend;
}
/*
return:
@ -142,10 +263,11 @@ return 0;
3 - not inited (use Init() first)
100 - unknown
*/
int Compress::CompressAndPut(const char * source, size_t source_len, FCGX_Stream * out_stream, int level)
int Compress::CompressAndPut(const char * source, size_t source_len, FCGX_Stream * out_stream, int encoding)
{
int ret;
z_stream * pstrm;
last_out_size = 0;
if( !ready_for_compress )
@ -156,17 +278,11 @@ int ret;
if( source_len == 0 )
return 0;
ret = MakeCompress(source, source_len, out_stream);
if( deflateReset(&strm) != Z_OK )
log << log1 << "Compress: problem with deflateReset()" << logend;
double ratio = 100.0 - (double(last_out_size) / double(source_len) * 100.0);
char buffer[30];
sprintf(buffer, "%.1f", ratio);
log << log2 << "Compress: original size: " << source_len << ", compress size: " << (int)last_out_size << ", ratio: " << buffer << "%" << logend;
pstrm = SelectStream(encoding);
ret = MakeCompress(*pstrm, source, source_len, out_stream, encoding);
ResetStream(pstrm, encoding);
PutLog(source_len, encoding);
return ret;
}

View File

@ -2,7 +2,7 @@
* This file is a part of Winix
* and is not publicly distributed
*
* Copyright (c) 2008-2010, Tomasz Sowa
* Copyright (c) 2008-2011, Tomasz Sowa
* All rights reserved.
*
*/
@ -23,18 +23,35 @@ public:
Compress();
~Compress();
int Init(int compress_level = 6);
int CompressAndPut(const char * source, size_t source_len, FCGX_Stream * out_stream, int level = 6);
int Init(int compress_level_ = 6);
/*
encoding:
0 - raw deflate data with no zlib header or trailer, and will not compute an adler32 check value
(for Internet Explorer)
1 - deflate
2 - gzip
*/
int CompressAndPut(const char * source, size_t source_len, FCGX_Stream * out_stream, int encoding = 2);
size_t last_out_size;
private:
bool AllocateMemory();
int MakeCompress(const char * source, size_t source_len, FCGX_Stream * out_stream);
bool InitRawDeflate();
bool InitDeflate();
bool InitGzip();
int MakeCompress(z_stream & strm, const char * source, size_t source_len, FCGX_Stream * out_stream, int encoding);
z_stream * SelectStream(int encoding);
void ResetStream(z_stream * pstrm, int encoding);
void PutLog(size_t source_len, int encoding);
int compress_level;
size_t buffer_max_len;
char * buffer;
z_stream strm;
z_stream strm_raw_deflate, strm_deflate, strm_gzip;
bool raw_deflate_inited, deflate_inited, gzip_inited;
bool ready_for_compress;
};

View File

@ -153,8 +153,9 @@ void Config::AssignValues(bool stdout_is_closed)
session_file = AText(L"session_file");
session_max = Size(L"session_max", 1000000);
compression = Bool(L"compression", true);
compression_page_min_size = Int(L"compression_page_min_size", 512);
compression = Bool(L"compression", true);
compression_page_min_size = Size(L"compression_page_min_size", 512);
compression_encoding = Int(L"compression_encoding", 20);
html_filter = Bool(L"html_filter", true);
html_filter_trim_white = Bool(L"html_filter_trim_white", true);

View File

@ -137,11 +137,20 @@ public:
// default: 1000000 (one milion)
size_t session_max;
// allow the html ouput to be compressed
// allow the html output to be compressed
// default: true
bool compression;
// if the output is shorter than this value then it will not be compressed
int compression_page_min_size;
// default: 512 bytes
size_t compression_page_min_size;
// 1 - use deflate if available (or raw deflate for Internet Explorer) or don't compress
// 2 - use gzip if available or don't compress
// 10 - prefer deflate -- use deflate (or raw deflate for IE) if both deflate and gzip are available
// 20 - prefer gzip -- use gzip if both deflate and gzip are available
// default: 20
int compression_encoding;
// plugins directory
// default: /usr/local/winix/plugins