You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
winix/winixd/core/compress.cpp

436 lines
8.9 KiB

/*
* This file is a part of Winix
* and is distributed under the 2-Clause BSD licence.
* Author: Tomasz Sowa <t.sowa@ttmath.org>
*/
/*
* Copyright (c) 2008-2014, Tomasz Sowa
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
*/
#include "compress.h"
#include "log.h"
namespace Winix
{
Compress::Compress()
{
buffer_in = 0;
buffer_out = 0;
buffer_max_len = 65536; // 64KB
ready_for_compress = false;
compress_level = 6;
raw_deflate_inited = false;
deflate_inited = false;
gzip_inited = false;
}
Compress::~Compress()
{
delete [] buffer_in;
delete [] buffer_out;
if( raw_deflate_inited )
deflateEnd(&strm_raw_deflate);
if( deflate_inited )
deflateEnd(&strm_deflate);
if( gzip_inited )
deflateEnd(&strm_gzip);
}
bool Compress::AllocateMemory()
{
if( buffer_in )
delete [] buffer_in;
if( buffer_out )
delete [] buffer_out;
buffer_in = 0;
buffer_out = 0;
try
{
buffer_in = new char[buffer_max_len];
buffer_out = new char[buffer_max_len];
}
catch(const std::bad_alloc &)
{
log << log1 << "Compress: can't allocate memory" << logend;
return false;
}
return true;
}
bool Compress::InitRawDeflate()
{
raw_deflate_inited = false;
strm_raw_deflate.next_in = 0;
strm_raw_deflate.zalloc = Z_NULL;
strm_raw_deflate.zfree = Z_NULL;
strm_raw_deflate.opaque = Z_NULL;
int ret = deflateInit2(&strm_raw_deflate, compress_level, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY);
if( ret != Z_OK )
log << log1 << "Compress: problem with deflateInit2() for raw deflate" << logend;
else
raw_deflate_inited = true;
return ret == Z_OK;
}
bool Compress::InitDeflate()
{
deflate_inited = false;
strm_deflate.next_in = 0;
strm_deflate.zalloc = Z_NULL;
strm_deflate.zfree = Z_NULL;
strm_deflate.opaque = Z_NULL;
int ret = deflateInit2(&strm_deflate, compress_level, Z_DEFLATED, 15, 8, Z_DEFAULT_STRATEGY);
if( ret != Z_OK )
log << log1 << "Compress: problem with deflateInit2() for deflate" << logend;
else
deflate_inited = true;
return ret == Z_OK;
}
bool Compress::InitGzip()
{
gzip_inited = false;
strm_gzip.next_in = 0;
strm_gzip.zalloc = Z_NULL;
strm_gzip.zfree = Z_NULL;
strm_gzip.opaque = Z_NULL;
int ret = deflateInit2(&strm_gzip, compress_level, Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY);
if( ret != Z_OK )
log << log1 << "Compress: problem with deflateInit2() for gzip" << logend;
else
gzip_inited = true;
return ret == Z_OK;
}
/*
return:
0 - ok
1 - can't allocate memory
100 - unknown error
*/
int Compress::Init(int compress_level_)
{
compress_level = compress_level_;
if( !AllocateMemory() )
return 1;
if( InitRawDeflate() && InitDeflate() && InitGzip() )
ready_for_compress = true;
else
return 100;
return 0;
}
int Compress::MakeCompress(z_stream & strm, const char * source, size_t source_len, BinaryPage & out_stream, int encoding)
{
int ret, flush;
size_t have;
do
{
strm.avail_in = (source_len > buffer_max_len) ? buffer_max_len : source_len;
source_len -= strm.avail_in;
flush = (source_len == 0) ? Z_FINISH : Z_NO_FLUSH;
strm.next_in = (Bytef*)source;
source += strm.avail_in;
do
{
strm.avail_out = buffer_max_len;
strm.next_out = (Bytef*)buffer_out;
ret = deflate(&strm, flush);
if( ret == Z_STREAM_ERROR || ret == Z_BUF_ERROR )
{
log << log1 << "Compress: problem with deflate()" << logend;
return 2;
}
have = buffer_max_len - strm.avail_out;
last_out_size += have;
out_stream.write(buffer_out, have);
}
while( strm.avail_out == 0 );
if( strm.avail_in != 0 )
{
log << log1 << "Compress: problem with deflate() - not all input is used" << logend;
return 2;
}
}
while( flush != Z_FINISH );
if( ret != Z_STREAM_END )
{
log << log1 << "Compress: problem with deflate() - stream not complete" << logend;
return 2;
}
return 0;
}
void Compress::CopyToInputBuffer(BinaryPage::const_iterator & i, size_t len)
{
for(size_t a=0 ; a<len ; ++a, ++i)
buffer_in[a] = *i;
}
// new way
int Compress::MakeCompress(z_stream & strm, const BinaryPage & page, BinaryPage & out, int encoding)
{
int ret, flush;
size_t have;
BinaryPage::const_iterator i = page.begin();
size_t source_len = page.size();
do
{
strm.avail_in = (source_len > buffer_max_len) ? buffer_max_len : source_len;
source_len -= strm.avail_in;
flush = (source_len == 0) ? Z_FINISH : Z_NO_FLUSH;
strm.next_in = (Bytef*)buffer_in;
changed the way how the request's answer is created, now winix can return json, xml, csv from out_main_stream or from frames and json from models removed from Request: bool send_bin_stream bool return_json bool return_info_only pt::Space info bool page_generated bool out_main_stream_use_html_filter bool out_streams_use_html_filter added to Request: enum AnswerSource enum AnswerContainer AnswerSource answer_source AnswerContainer answer_container bool use_ezc_engine std::wstring frame bool send_all_frames bool use_html_filter added to Config: // the name of the url parameter for returning all frames, e.g. https://domain.tld/mydir/myfunction/allframes // default: allframes std::wstring request_all_frames_parameter; // the name of the root element when serializing request answer to xml // default: winix std::wstring xml_root; algorithm (the whole algorithm is described in core/request.h): at the beginning of a request winix sets answer_source to models answer_container to text use_ezc_engine to true next answer_container and use_ezc_engine can be changed in the following way: 1. winix will look for 'Accept' http header and depending on the header winix will set: (not implemented yet) Accept | answer_container | use_ezc_engine ------------------------------------|----------------- application/json | json | false application/xml | xml | false text/csv | csv | false 2. next answer_container is set depending on 'container' url parameter container | answer_container --------------------------------------------------------- not present | don't change the value text | text json | json xml | xml csv | csv use_ezc_engine is set depending on 'answer' url parameter: answer | use_ezc_engine --------------------------------- not present | don't change the value html | true data | false if 'answer' is html then we take into account two more parameters: frame: frame_name (empty default) - if set then winix returns this specific frame allframes: (if present then winix returns all frames)
1 year ago
// IMPROVE ME we can add an interface to pt::TextStreamBase<> and get all chunks of memory
CopyToInputBuffer(i, strm.avail_in);
do
{
strm.avail_out = buffer_max_len;
strm.next_out = (Bytef*)buffer_out;
ret = deflate(&strm, flush);
if( ret == Z_STREAM_ERROR || ret == Z_BUF_ERROR )
{
log << log1 << "Compress: problem with deflate()" << logend;
return 2;
}
have = buffer_max_len - strm.avail_out;
last_out_size += have;
out.write(buffer_out, have);
}
while( strm.avail_out == 0 );
if( strm.avail_in != 0 )
{
log << log1 << "Compress: problem with deflate() - not all input is used" << logend;
return 2;
}
}
while( flush != Z_FINISH );
if( ret != Z_STREAM_END )
{
log << log1 << "Compress: problem with deflate() - stream not complete" << logend;
return 2;
}
return 0;
}
z_stream * Compress::SelectStream(int encoding)
{
z_stream * pstrm;
if( encoding == 0 )
pstrm = &strm_raw_deflate;
else
if( encoding == 1 )
pstrm = &strm_deflate;
else
pstrm = &strm_gzip;
return pstrm;
}
void Compress::ResetStream(z_stream * pstrm, int encoding)
{
if( deflateReset(pstrm) != Z_OK )
{
log << log1 << "Compress: problem with deflateReset()" << logend;
deflateEnd(pstrm);
if( encoding == 0 )
InitRawDeflate();
else
if( encoding == 1 )
InitDeflate();
else
InitGzip();
}
}
void Compress::PutLog(size_t source_len, int encoding)
{
double ratio = 100.0 - (double(last_out_size) / double(source_len) * 100.0);
char buffer[30];
sprintf(buffer, "%.1f", ratio);
log << log2 << "Compress: ";
if( encoding == 0 )
log << "raw deflate";
else
if( encoding == 1 )
log << "deflate";
else
log << "gzip";
log << ", original size: " << source_len << ", size after compressing: "
<< (int)last_out_size << ", ratio: " << buffer << "%" << logend;
}
/*
return:
0 - ok;
1 - can't allocate memory
2 - error during compressing
3 - not inited (use Init() first)
100 - unknown
*/
int Compress::Compressing(const char * source, size_t source_len, BinaryPage & out_stream, int encoding)
{
int ret;
z_stream * pstrm;
last_out_size = 0;
out_stream.clear();
if( !ready_for_compress )
{
log << log1 << "Compress: not ready yet" << logend;
return 3;
}
// !! CHECK ME
// it is correct to immediately return? what about headers in the compressed page?
if( source_len == 0 )
return 0;
pstrm = SelectStream(encoding);
ret = MakeCompress(*pstrm, source, source_len, out_stream, encoding);
ResetStream(pstrm, encoding);
PutLog(source_len, encoding);
return ret;
}
/*
return:
0 - ok;
1 - can't allocate memory
2 - error during compressing
3 - not inited (use Init() first)
100 - unknown
*/
int Compress::Compressing(const BinaryPage & in, BinaryPage & out, int encoding)
{
int ret;
z_stream * pstrm;
last_out_size = 0;
out.clear();
if( !ready_for_compress )
{
log << log1 << "Compress: not ready yet" << logend;
return 3;
}
// !! CHECK ME
// it is correct to immediately return? what about headers in the compressed page?
if( in.empty() )
return 0;
pstrm = SelectStream(encoding);
ret = MakeCompress(*pstrm, in, out, encoding);
ResetStream(pstrm, encoding);
PutLog(in.size(), encoding);
return ret;
}
} // namespace Winix