diff --git a/core/acceptbaseparser.cpp b/core/acceptbaseparser.cpp index 83d3a84..4f22792 100755 --- a/core/acceptbaseparser.cpp +++ b/core/acceptbaseparser.cpp @@ -2,7 +2,7 @@ * This file is a part of Winix * and is not publicly distributed * - * Copyright (c) 2008-2010, Tomasz Sowa + * Copyright (c) 2008-2011, Tomasz Sowa * All rights reserved. * */ diff --git a/core/acceptbaseparser.h b/core/acceptbaseparser.h index d78b7cc..e94ae39 100755 --- a/core/acceptbaseparser.h +++ b/core/acceptbaseparser.h @@ -2,7 +2,7 @@ * This file is a part of Winix * and is not publicly distributed * - * Copyright (c) 2008-2010, Tomasz Sowa + * Copyright (c) 2008-2011, Tomasz Sowa * All rights reserved. * */ diff --git a/core/acceptencodingparser.h b/core/acceptencodingparser.h index 043fc4a..75cea30 100755 --- a/core/acceptencodingparser.h +++ b/core/acceptencodingparser.h @@ -2,7 +2,7 @@ * This file is a part of Winix * and is not publicly distributed * - * Copyright (c) 2008-2010, Tomasz Sowa + * Copyright (c) 2008-2011, Tomasz Sowa * All rights reserved. * */ @@ -24,24 +24,60 @@ public: } + bool AcceptGzip() + { + return accept_gzip; + } + + + void ParseAndLog(const char * str) + { + Parse(str); + + if( accept_deflate || accept_gzip ) + { + log << log3 << "AEP: "; + + if( accept_deflate ) + { + log << "accept deflate"; + + if( accept_gzip ) + log << ", "; + } + + if( accept_gzip ) + log << "accept gzip"; + + log << logend; + } + } + + private: void Init() { accept_deflate = false; + accept_gzip = false; } void Param(const std::string & param, double q) { - if( param=="deflate" && q!=0 ) + if( param=="deflate" && q!=0.0 ) { accept_deflate = true; - log << log3 << "AEP: accept deflate" << logend; + } + + if( param=="gzip" && q!=0.0 ) + { + accept_gzip = true; } } bool accept_deflate; + bool accept_gzip; }; diff --git a/core/app.cpp b/core/app.cpp index 4b42d29..a0f34c1 100755 --- a/core/app.cpp +++ b/core/app.cpp @@ -2,7 +2,7 @@ * This file is a part of Winix * and is not publicly distributed * - * Copyright (c) 2010, Tomasz Sowa + * Copyright (c) 2010-2011, Tomasz Sowa * All rights reserved. * */ @@ -385,7 +385,7 @@ void App::ReadRequest() ReadGetPostVars(); cookie_parser.Parse(cur.request->env_http_cookie, cur.request->cookie_tab); - accept_encoding_parser.Parse(cur.request->env_http_accept_encoding); + accept_encoding_parser.ParseAndLog(cur.request->env_http_accept_encoding); CheckIE(); CheckKonqueror(); @@ -540,7 +540,7 @@ void App::PrepareSessionCookie() -void App::SendHeaders(bool compressing, Header header) +void App::SendHeaders(bool compressing, int compress_encoding, Header header) { PrepareSessionCookie(); @@ -590,8 +590,13 @@ void App::SendHeaders(bool compressing, Header header) } } - if( compressing ) - FCGX_PutS("Content-Encoding: deflate\r\n", fcgi_request.out); + if( compressing ) + { + if( compress_encoding == 0 || compress_encoding == 1 ) + FCGX_PutS("Content-Encoding: deflate\r\n", fcgi_request.out); + else + FCGX_PutS("Content-Encoding: gzip\r\n", fcgi_request.out); + } FCGX_PutS(cur.request->headers.CStr(), fcgi_request.out); FCGX_PutS("\r\n", fcgi_request.out); @@ -610,14 +615,12 @@ void App::SetHtmlFilterConf() } -// !! kopiowanie tych stringow bedzie zmienione -// gdy bedziemy korzystac w przyszlosci z wlasnego stringstream -void App::FilterCompressSend(bool compressing, const std::wstring & source_ref) +void App::FilterCompressSend(bool compressing, int compress_encoding, const std::wstring & source_ref) { const std::wstring * source = &source_ref; bool raw = cur.request->is_item && cur.request->item.content_type == Item::ct_raw && cur.request->status == WINIX_ERR_OK && - cur.request->function && (cur.request->function == &functions.fun_cat || cur.request->function == &functions.fun_run); + cur.request->function && (cur.request->function == &functions.fun_cat || cur.request->function == &functions.fun_run); if( config.html_filter && !raw ) { @@ -633,32 +636,76 @@ void App::FilterCompressSend(bool compressing, const std::wstring & source_ref) source = &html_with_debug; } - - // !! zrobic z tym porzadek - std::string temp; - Ezc::WideToUTF8(*source, temp); + if( config.utf8 ) + Ezc::WideToUTF8(*source, source_a); + else + AssignString(*source, source_a); if( compressing ) - compress.CompressAndPut(temp.c_str(), temp.length(), fcgi_request.out); + compress.CompressAndPut(source_a.c_str(), source_a.length(), fcgi_request.out, compress_encoding); else - FCGX_PutS(temp.c_str(), fcgi_request.out); + FCGX_PutS(source_a.c_str(), fcgi_request.out); } -bool App::IsCompressionAllowed(const std::wstring & source) +int App::SelectDeflateVersion() { - return( config.compression && - cur.request->role == Request::responder && - cur.request->redirect_to.empty() && - cur.request->x_sendfile.empty() && - !cur.request->browser_msie && - !cur.request->browser_konqueror && - accept_encoding_parser.AcceptDeflate() && - source.size() >= (size_t)config.compression_page_min_size ); + if( cur.request->browser_msie ) + return 0; // raw deflate + else + return 1; // deflate } +void App::SelectCompression(size_t source_len, bool & compression_allowed, int & compression_encoding) +{ + compression_allowed = false; + compression_encoding = 0; + + if( config.compression && + cur.request->role == Request::responder && + cur.request->redirect_to.empty() && + cur.request->x_sendfile.empty() && + !cur.request->browser_konqueror && /* !! sprawdzic czy Konqueror bedzie obslugiwal raw deflate */ + source_len >= config.compression_page_min_size ) + { + if( config.compression_encoding == 1 || config.compression_encoding == 10 ) + { + if( accept_encoding_parser.AcceptDeflate() ) + { + compression_allowed = true; + compression_encoding = SelectDeflateVersion(); + } + else + if( config.compression_encoding == 10 && accept_encoding_parser.AcceptGzip() ) + { + compression_allowed = true; + compression_encoding = 2; // gzip + } + } + + + if( config.compression_encoding == 2 || config.compression_encoding == 20 ) + { + if( accept_encoding_parser.AcceptGzip() ) + { + compression_allowed = true; + compression_encoding = 2; // gzip + } + else + if( config.compression_encoding == 20 && accept_encoding_parser.AcceptDeflate() ) + { + compression_allowed = true; + compression_encoding = SelectDeflateVersion(); + } + } + } +} + + + + bool App::CanSendContent(Header header) { if( !cur.request->redirect_to.empty() || !cur.request->x_sendfile.empty() ) @@ -698,9 +745,12 @@ void App::AddDebugInfo(std::wstring & out) void App::SendAnswer() { const std::wstring & source = cur.request->page.Str(); -Header header = h_200; -bool compressing = IsCompressionAllowed(source); -Error status = cur.request->status; +Header header = h_200; +Error status = cur.request->status; +bool compressing; +int compress_encoding; + + SelectCompression(source.length(), compressing, compress_encoding); if( status == WINIX_ERR_NO_ITEM || status == WINIX_ERR_NO_FUNCTION || status == WINIX_ERR_UNKNOWN_PARAM ) header = h_404; @@ -708,12 +758,12 @@ Error status = cur.request->status; if( status == WINIX_ERR_PERMISSION_DENIED || status == WINIX_ERR_CANT_CHANGE_USER || status == WINIX_ERR_CANT_CHANGE_GROUP ) header = h_403; - SendHeaders(compressing, header); + SendHeaders(compressing, compress_encoding, header); if( CanSendContent(header) ) { - // filtering (html), compressing (deflate) and sending back to the web browser - FilterCompressSend(compressing, source); + // filtering (html), compressing and sending back to the web browser + FilterCompressSend(compressing, compress_encoding, source); } } diff --git a/core/app.h b/core/app.h index 7b164de..4eb2fdf 100755 --- a/core/app.h +++ b/core/app.h @@ -2,7 +2,7 @@ * This file is a part of Winix * and is not publicly distributed * - * Copyright (c) 2010, Tomasz Sowa + * Copyright (c) 2010-2011, Tomasz Sowa * All rights reserved. * */ @@ -131,7 +131,7 @@ private: Synchro synchro; pthread_t signal_thread; std::string url_to_fetch_on_exit; - + std::string source_a; void ProcessRequestThrow(); void ProcessRequest(); @@ -156,9 +156,10 @@ private: void SetHtmlFilterConf(); void PrepareSessionCookie(); void AddDebugInfo(std::wstring & out); - void FilterCompressSend(bool compressing, const std::wstring & source_ref); - void SendHeaders(bool compressing, Header header); - bool IsCompressionAllowed(const std::wstring & source); + void FilterCompressSend(bool compressing, int compress_encoding, const std::wstring & source_ref); + void SendHeaders(bool compressing, int compress_encoding, Header header); + int SelectDeflateVersion(); + void SelectCompression(size_t source_len, bool & compression_allowed, int & compression_encoding); bool CanSendContent(Header header); void LogUser(const char * msg, uid_t id); diff --git a/core/compress.cpp b/core/compress.cpp index bb237f5..7a3d7d5 100755 --- a/core/compress.cpp +++ b/core/compress.cpp @@ -2,7 +2,7 @@ * This file is a part of Winix * and is not publicly distributed * - * Copyright (c) 2008-2010, Tomasz Sowa + * Copyright (c) 2008-2011, Tomasz Sowa * All rights reserved. * */ @@ -15,9 +15,13 @@ Compress::Compress() { - buffer = 0; - buffer_max_len = 65536; // 64KB + buffer = 0; + buffer_max_len = 65536; // 64KB ready_for_compress = false; + compress_level = 6; + raw_deflate_inited = false; + deflate_inited = false; + gzip_inited = false; } @@ -25,6 +29,15 @@ Compress::~Compress() { if( buffer ) delete [] buffer; + + if( raw_deflate_inited ) + deflateEnd(&strm_raw_deflate); + + if( deflate_inited ) + deflateEnd(&strm_deflate); + + if( gzip_inited ) + deflateEnd(&strm_gzip); } @@ -33,7 +46,6 @@ bool Compress::AllocateMemory() if( buffer ) delete [] buffer; - try { buffer = new char[buffer_max_len]; @@ -41,7 +53,6 @@ bool Compress::AllocateMemory() catch(const std::bad_alloc &) { log << log1 << "Compress: can't allocate memory" << logend; - buffer = 0; return false; } @@ -50,39 +61,93 @@ return true; } + +bool Compress::InitRawDeflate() +{ + raw_deflate_inited = false; + + strm_raw_deflate.next_in = 0; + strm_raw_deflate.zalloc = Z_NULL; + strm_raw_deflate.zfree = Z_NULL; + strm_raw_deflate.opaque = Z_NULL; + + int ret = deflateInit2(&strm_raw_deflate, compress_level, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY); + + if( ret != Z_OK ) + log << log1 << "Compress: problem with deflateInit2() for raw deflate" << logend; + else + raw_deflate_inited = true; + +return ret == Z_OK; +} + + +bool Compress::InitDeflate() +{ + deflate_inited = false; + + strm_deflate.next_in = 0; + strm_deflate.zalloc = Z_NULL; + strm_deflate.zfree = Z_NULL; + strm_deflate.opaque = Z_NULL; + + int ret = deflateInit2(&strm_deflate, compress_level, Z_DEFLATED, 15, 8, Z_DEFAULT_STRATEGY); + + if( ret != Z_OK ) + log << log1 << "Compress: problem with deflateInit2() for deflate" << logend; + else + deflate_inited = true; + +return ret == Z_OK; +} + + +bool Compress::InitGzip() +{ + gzip_inited = false; + + strm_gzip.next_in = 0; + strm_gzip.zalloc = Z_NULL; + strm_gzip.zfree = Z_NULL; + strm_gzip.opaque = Z_NULL; + + int ret = deflateInit2(&strm_gzip, compress_level, Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY); + + if( ret != Z_OK ) + log << log1 << "Compress: problem with deflateInit2() for gzip" << logend; + else + gzip_inited = true; + +return ret == Z_OK; +} + + + /* return: - 0 - ok; - 1 - can't allocate memory - 100 - unknown + 0 - ok + 1 - can't allocate memory + 100 - unknown error */ -int Compress::Init(int compress_level) +int Compress::Init(int compress_level_) { + compress_level = compress_level_; + if( buffer == 0 ) if( !AllocateMemory() ) return 1; - - strm.zalloc = Z_NULL; - strm.zfree = Z_NULL; - strm.opaque = Z_NULL; - int ret = deflateInit(&strm, compress_level); - - if( ret != Z_OK ) - log << log1 << "Compress: problem with deflateInit()" << logend; - - if( ret == Z_MEM_ERROR ) - return 1; - - if( ret != Z_OK ) - return 100; - ready_for_compress = true; + if( InitRawDeflate() && InitDeflate() && InitGzip() ) + ready_for_compress = true; + else + return 100; return 0; } -int Compress::MakeCompress(const char * source, size_t source_len, FCGX_Stream * out_stream) + +int Compress::MakeCompress(z_stream & strm, const char * source, size_t source_len, FCGX_Stream * out_stream, int encoding) { int ret, flush; size_t have; @@ -133,6 +198,62 @@ return 0; +z_stream * Compress::SelectStream(int encoding) +{ +z_stream * pstrm; + + if( encoding == 0 ) + pstrm = &strm_raw_deflate; + else + if( encoding == 1 ) + pstrm = &strm_deflate; + else + pstrm = &strm_gzip; + +return pstrm; +} + + +void Compress::ResetStream(z_stream * pstrm, int encoding) +{ + if( deflateReset(pstrm) != Z_OK ) + { + log << log1 << "Compress: problem with deflateReset()" << logend; + + deflateEnd(pstrm); + + if( encoding == 0 ) + InitRawDeflate(); + else + if( encoding == 1 ) + InitDeflate(); + else + InitGzip(); + } +} + + +void Compress::PutLog(size_t source_len, int encoding) +{ + double ratio = 100.0 - (double(last_out_size) / double(source_len) * 100.0); + char buffer[30]; + sprintf(buffer, "%.1f", ratio); + + log << log2 << "Compress: "; + + if( encoding == 0 ) + log << "raw deflate"; + else + if( encoding == 1 ) + log << "deflate"; + else + log << "gzip"; + + log << ", original size: " << source_len << ", size after compressing: " + << (int)last_out_size << ", ratio: " << buffer << "%" << logend; +} + + /* return: @@ -142,10 +263,11 @@ return 0; 3 - not inited (use Init() first) 100 - unknown */ -int Compress::CompressAndPut(const char * source, size_t source_len, FCGX_Stream * out_stream, int level) +int Compress::CompressAndPut(const char * source, size_t source_len, FCGX_Stream * out_stream, int encoding) { int ret; - +z_stream * pstrm; + last_out_size = 0; if( !ready_for_compress ) @@ -156,17 +278,11 @@ int ret; if( source_len == 0 ) return 0; - - ret = MakeCompress(source, source_len, out_stream); - - if( deflateReset(&strm) != Z_OK ) - log << log1 << "Compress: problem with deflateReset()" << logend; - double ratio = 100.0 - (double(last_out_size) / double(source_len) * 100.0); - char buffer[30]; - sprintf(buffer, "%.1f", ratio); - - log << log2 << "Compress: original size: " << source_len << ", compress size: " << (int)last_out_size << ", ratio: " << buffer << "%" << logend; + pstrm = SelectStream(encoding); + ret = MakeCompress(*pstrm, source, source_len, out_stream, encoding); + ResetStream(pstrm, encoding); + PutLog(source_len, encoding); return ret; } diff --git a/core/compress.h b/core/compress.h index 7511094..318cb62 100755 --- a/core/compress.h +++ b/core/compress.h @@ -2,7 +2,7 @@ * This file is a part of Winix * and is not publicly distributed * - * Copyright (c) 2008-2010, Tomasz Sowa + * Copyright (c) 2008-2011, Tomasz Sowa * All rights reserved. * */ @@ -23,18 +23,35 @@ public: Compress(); ~Compress(); - int Init(int compress_level = 6); - int CompressAndPut(const char * source, size_t source_len, FCGX_Stream * out_stream, int level = 6); + int Init(int compress_level_ = 6); + + /* + encoding: + 0 - raw deflate data with no zlib header or trailer, and will not compute an adler32 check value + (for Internet Explorer) + 1 - deflate + 2 - gzip + */ + int CompressAndPut(const char * source, size_t source_len, FCGX_Stream * out_stream, int encoding = 2); size_t last_out_size; private: bool AllocateMemory(); - int MakeCompress(const char * source, size_t source_len, FCGX_Stream * out_stream); + bool InitRawDeflate(); + bool InitDeflate(); + bool InitGzip(); + int MakeCompress(z_stream & strm, const char * source, size_t source_len, FCGX_Stream * out_stream, int encoding); + z_stream * SelectStream(int encoding); + void ResetStream(z_stream * pstrm, int encoding); + void PutLog(size_t source_len, int encoding); + + int compress_level; size_t buffer_max_len; char * buffer; - z_stream strm; + z_stream strm_raw_deflate, strm_deflate, strm_gzip; + bool raw_deflate_inited, deflate_inited, gzip_inited; bool ready_for_compress; }; diff --git a/core/config.cpp b/core/config.cpp index 4b2c33c..9522f49 100755 --- a/core/config.cpp +++ b/core/config.cpp @@ -153,8 +153,9 @@ void Config::AssignValues(bool stdout_is_closed) session_file = AText(L"session_file"); session_max = Size(L"session_max", 1000000); - compression = Bool(L"compression", true); - compression_page_min_size = Int(L"compression_page_min_size", 512); + compression = Bool(L"compression", true); + compression_page_min_size = Size(L"compression_page_min_size", 512); + compression_encoding = Int(L"compression_encoding", 20); html_filter = Bool(L"html_filter", true); html_filter_trim_white = Bool(L"html_filter_trim_white", true); diff --git a/core/config.h b/core/config.h index a2265e8..ec33bf1 100755 --- a/core/config.h +++ b/core/config.h @@ -137,11 +137,20 @@ public: // default: 1000000 (one milion) size_t session_max; - // allow the html ouput to be compressed + // allow the html output to be compressed + // default: true bool compression; // if the output is shorter than this value then it will not be compressed - int compression_page_min_size; + // default: 512 bytes + size_t compression_page_min_size; + + // 1 - use deflate if available (or raw deflate for Internet Explorer) or don't compress + // 2 - use gzip if available or don't compress + // 10 - prefer deflate -- use deflate (or raw deflate for IE) if both deflate and gzip are available + // 20 - prefer gzip -- use gzip if both deflate and gzip are available + // default: 20 + int compression_encoding; // plugins directory // default: /usr/local/winix/plugins