575 lines
11 KiB
C++
Executable File
575 lines
11 KiB
C++
Executable File
/*
|
|
* This file is a part of Winix
|
|
* and is not publicly distributed
|
|
*
|
|
* Copyright (c) 2011-2014, Tomasz Sowa
|
|
* All rights reserved.
|
|
*
|
|
*/
|
|
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <unistd.h>
|
|
#include <string.h>
|
|
#include "exportthread.h"
|
|
#include "core/log.h"
|
|
#include "core/misc.h"
|
|
#include "utf8/utf8.h"
|
|
|
|
namespace Winix
|
|
{
|
|
|
|
|
|
|
|
namespace Export
|
|
{
|
|
ExportThread * ExportThread::exp_thread;
|
|
|
|
|
|
ExportThread::ExportThread()
|
|
{
|
|
exp_thread = 0;
|
|
utf8 = false;
|
|
browser_name = "Winix Export";
|
|
conn_timeout = 5;
|
|
conn_max_errors = 3;
|
|
}
|
|
|
|
|
|
|
|
void ExportThread::SetUTF8(bool use_utf8)
|
|
{
|
|
utf8 = use_utf8;
|
|
}
|
|
|
|
|
|
void ExportThread::SetBaseUrl(const std::wstring & url)
|
|
{
|
|
// SetBaseUrl() is called before the this thread
|
|
// is started so we can use 'base_url' later without locking
|
|
// the same is for 'utf8'
|
|
base_url = url;
|
|
}
|
|
|
|
void ExportThread::AddMessage(const Message & message)
|
|
{
|
|
log << log1 << "export thread: a" << logsave << logend;
|
|
|
|
message_tab.insert(message_tab.end(), message);
|
|
WakeUpThread();
|
|
|
|
log << log1 << "export thread: b" << logsave << logend;
|
|
}
|
|
|
|
|
|
// first thread (objects locked)
|
|
void ExportThread::AddMessage(int type, const std::wstring & url, const std::wstring & path)
|
|
{
|
|
message_add_temp.type = type;
|
|
message_add_temp.url = url;
|
|
message_add_temp.path = path;
|
|
|
|
AddMessage(message_add_temp);
|
|
}
|
|
|
|
|
|
|
|
// second thread
|
|
// objects are locked
|
|
bool ExportThread::SignalReceived()
|
|
{
|
|
return !message_tab.empty();
|
|
}
|
|
|
|
|
|
|
|
// second thread
|
|
// objects are not locked
|
|
void ExportThread::Do()
|
|
{
|
|
MessageTab::iterator i;
|
|
bool end;
|
|
|
|
Lock();
|
|
i = message_tab.begin();
|
|
Unlock();
|
|
|
|
do
|
|
{
|
|
Lock();
|
|
|
|
if( i != message_tab.end() )
|
|
{
|
|
message_work = *i;
|
|
|
|
Unlock();
|
|
DoMessage();
|
|
Lock();
|
|
|
|
// although there was Unlock() used we can use the same iterator 'i' here
|
|
// it will *not* be invalidated (MessageTab is a std::list)
|
|
// and we are deleting only here
|
|
i->errors = message_work.errors;
|
|
|
|
if( message_work.can_remove )
|
|
message_tab.erase(i++);
|
|
|
|
end = false;
|
|
}
|
|
else
|
|
{
|
|
end = true;
|
|
}
|
|
|
|
WaitForSignalSleep(1);
|
|
Unlock();
|
|
}
|
|
while( !end && !IsExitSignal() );
|
|
}
|
|
|
|
|
|
// second thread
|
|
// objects are not locked
|
|
// current message we have in 'message_work'
|
|
void ExportThread::DoMessage()
|
|
{
|
|
bool sent_ok = false;
|
|
|
|
Convert(message_work.url, url_a);
|
|
|
|
if( message_work.type == WINIX_PL_EXPORT_TYPE_CREATE_FILE )
|
|
{
|
|
if( Fetch(url_a.c_str()) )
|
|
{
|
|
ChangeAdresses(buffer);
|
|
|
|
if( Put() )
|
|
sent_ok = true;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if( Put() )
|
|
sent_ok = true;
|
|
}
|
|
|
|
|
|
if( sent_ok )
|
|
{
|
|
message_work.can_remove = true;
|
|
}
|
|
else
|
|
{
|
|
message_work.errors += 1;
|
|
message_work.can_remove = false;
|
|
|
|
if( message_work.errors > conn_max_errors )
|
|
{
|
|
message_work.can_remove = true;
|
|
|
|
Lock();
|
|
log << log1 << "Export: too many errors for uploading " << message_work.path << " (skipping)" << logend << logsave;
|
|
Unlock();
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// second thread
|
|
// objects are not locked
|
|
bool ExportThread::Fetch(const char * url)
|
|
{
|
|
Lock();
|
|
CURL * curl = curl_easy_init();
|
|
Unlock();
|
|
|
|
if( !curl )
|
|
{
|
|
Lock();
|
|
log << log1 << "Export: I can't use curl" << logend;
|
|
Unlock();
|
|
return false;
|
|
}
|
|
|
|
error_buf[0] = 0;
|
|
exp_thread = this;
|
|
buffer.clear();
|
|
|
|
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, StaticSaveFunction);
|
|
curl_easy_setopt(curl, CURLOPT_URL, url);
|
|
curl_easy_setopt(curl, CURLOPT_USERAGENT, browser_name.c_str());
|
|
curl_easy_setopt(curl, CURLOPT_TIMEOUT, conn_timeout);
|
|
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, conn_timeout);
|
|
curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, error_buf);
|
|
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
|
|
curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 20);
|
|
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
|
|
|
|
CURLcode res = curl_easy_perform(curl);
|
|
|
|
curl_easy_cleanup(curl);
|
|
|
|
if( res != 0 )
|
|
{
|
|
Lock();
|
|
log << log1 << "Export: download failed: " << error_buf << logend << logsave;
|
|
Unlock();
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
|
|
|
|
size_t ExportThread::StaticSaveFunction(char * ptr, size_t size, size_t nmemb, void *userdata)
|
|
{
|
|
if( exp_thread )
|
|
return exp_thread->SaveFunction(ptr, size, nmemb, userdata);
|
|
|
|
return 0;
|
|
}
|
|
|
|
|
|
size_t ExportThread::SaveFunction(char * ptr, size_t size, size_t nmemb, void *userdata)
|
|
{
|
|
size_t len = size * nmemb;
|
|
|
|
if( len > 0 )
|
|
buffer.append(ptr, len);
|
|
|
|
return len;
|
|
}
|
|
|
|
|
|
|
|
// second thread
|
|
// objects are not locked
|
|
void ExportThread::Convert(const std::wstring & in, std::string & out, bool clear)
|
|
{
|
|
Lock();
|
|
|
|
if( utf8 )
|
|
PT::WideToUTF8(in, out, clear);
|
|
else
|
|
AssignString(in, out, clear);
|
|
|
|
Unlock();
|
|
}
|
|
|
|
|
|
|
|
// second thread
|
|
// objects are not locked
|
|
bool ExportThread::Put()
|
|
{
|
|
FILE * file = 0;
|
|
|
|
if( message_work.type == WINIX_PL_EXPORT_TYPE_CREATE_FILE_STATIC )
|
|
{
|
|
Convert(message_work.url, local_path);
|
|
file = fopen(local_path.c_str(), "r");
|
|
|
|
if( !file )
|
|
{
|
|
Lock();
|
|
log << log1 << "Export: I cannot open the file: " << local_path << logend;
|
|
Unlock();
|
|
return false;
|
|
}
|
|
else
|
|
{
|
|
log << log3 << "Export: sending a static file: " << local_path << logend;
|
|
}
|
|
}
|
|
|
|
|
|
Lock();
|
|
CURL * curl = curl_easy_init();
|
|
Unlock();
|
|
|
|
|
|
if( !curl )
|
|
{
|
|
Lock();
|
|
log << log1 << "Export: I can't use curl (sending)" << logend;
|
|
Unlock();
|
|
|
|
if( file )
|
|
fclose(file);
|
|
|
|
return false;
|
|
}
|
|
|
|
exp_thread = this;
|
|
error_buf[0] = 0;
|
|
buffer_read_index = 0;
|
|
|
|
ftp_server = "ftp://";
|
|
Convert(message_work.ftp_server, ftp_server, false);
|
|
Convert(message_work.path, ftp_server, false);
|
|
Convert(message_work.ftp_login, ftp_login);
|
|
Convert(message_work.ftp_pass, ftp_pass);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_URL, ftp_server.c_str());
|
|
curl_easy_setopt(curl, CURLOPT_USERNAME, ftp_login.c_str());
|
|
curl_easy_setopt(curl, CURLOPT_PASSWORD, ftp_pass.c_str());
|
|
curl_easy_setopt(curl, CURLOPT_FTP_CREATE_MISSING_DIRS, 2);
|
|
curl_easy_setopt(curl, CURLOPT_USERAGENT, browser_name.c_str());
|
|
curl_easy_setopt(curl, CURLOPT_UPLOAD, 1);
|
|
curl_easy_setopt(curl, CURLOPT_FTP_RESPONSE_TIMEOUT, conn_timeout);
|
|
curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, error_buf);
|
|
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
|
|
|
|
if( file )
|
|
{
|
|
curl_easy_setopt(curl, CURLOPT_READDATA, file);
|
|
}
|
|
else
|
|
{
|
|
curl_easy_setopt(curl, CURLOPT_READFUNCTION, StaticReadFunction);
|
|
curl_easy_setopt(curl, CURLOPT_INFILESIZE, buffer.size());
|
|
}
|
|
|
|
CURLcode res = curl_easy_perform(curl);
|
|
curl_easy_cleanup(curl);
|
|
|
|
if( file )
|
|
fclose(file);
|
|
|
|
if( res != 0 )
|
|
{
|
|
Lock();
|
|
log << log1 << "Export: upload failed: " << error_buf << " (" << ftp_server << ")" << logend << logsave;
|
|
Unlock();
|
|
return false;
|
|
}
|
|
else
|
|
{
|
|
Lock();
|
|
log << log2 << "Export: uploaded: " << ftp_server << logend << logsave;
|
|
Unlock();
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
|
|
|
|
|
|
size_t ExportThread::StaticReadFunction(char * ptr, size_t size, size_t nmemb, void *userdata)
|
|
{
|
|
if( exp_thread )
|
|
return exp_thread->ReadFunction(ptr, size, nmemb, userdata);
|
|
|
|
return 0;
|
|
}
|
|
|
|
|
|
size_t ExportThread::ReadFunction(char * ptr, size_t size, size_t nmemb, void *userdata)
|
|
{
|
|
size_t max_len = size * nmemb;
|
|
size_t i;
|
|
|
|
for(i=0 ; i<max_len && buffer_read_index < buffer.size() ; ++i, ++buffer_read_index )
|
|
{
|
|
ptr[i] = buffer[buffer_read_index];
|
|
}
|
|
|
|
return i;
|
|
}
|
|
|
|
|
|
// second thread
|
|
// objects locked
|
|
bool ExportThread::HasThumbInAdress(std::string & buf, size_t i)
|
|
{
|
|
const char * thumb1 = "/-/thumb";
|
|
const char * thumb2 = "/download/thumb";
|
|
|
|
size_t len1 = strlen(thumb1);
|
|
size_t len2 = strlen(thumb2);
|
|
|
|
for( ; i<buf.size() ; ++i)
|
|
{
|
|
if( IsSubStringp(thumb1, &buf[i]) )
|
|
{
|
|
buf.erase(i, len1);
|
|
return true;
|
|
}
|
|
|
|
if( IsSubStringp(thumb2, &buf[i]) )
|
|
{
|
|
buf.erase(i, len2);
|
|
return true;
|
|
}
|
|
|
|
if( buf[i] == 10 || buf[i] == '"' || buf[i] == ' ' || buf[i] == '\t' ||
|
|
buf[i] == '>' ||buf[i] == '<' )
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
|
|
|
|
// second thread
|
|
// objects locked
|
|
void ExportThread::ChangeAdressesThumb(std::string & buf, const char * http_prefix)
|
|
{
|
|
look_for_url = http_prefix;
|
|
Convert(base_url, look_for_url, false);
|
|
Convert(message_work.src_dir, look_for_url, false);
|
|
|
|
for(size_t i=0 ; i<buf.size() ; ++i)
|
|
{
|
|
if( IsSubStringp(look_for_url.c_str(), &buf[i]) )
|
|
{
|
|
i += look_for_url.size() - 1; // without skipping the last slash
|
|
|
|
if( HasThumbInAdress(buf, i) )
|
|
{
|
|
if( i <= buf.size() )
|
|
buf.insert(i, "/download"); //!! do konfiga (prefiks katalogu z miniaturami na serwerze docelowym)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
|
|
// second thread
|
|
// objects not locked
|
|
void ExportThread::ChangeAdressesThumb(std::string & buf)
|
|
{
|
|
Lock();
|
|
ChangeAdressesThumb(buf, "http://");
|
|
ChangeAdressesThumb(buf, "https://");
|
|
Unlock();
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// second thread
|
|
// objects locked
|
|
void ExportThread::ChangeLogStrings()
|
|
{
|
|
log << log3 << "Export: changing string: |" << look_for_url << "|, to: |" << repl_url << "|" << logend;
|
|
}
|
|
|
|
|
|
|
|
// second thread
|
|
// objects locked
|
|
void ExportThread::ChangeBaseAdress(std::string & buf,
|
|
const char * http_prefix, const char * dir_prefix, const char * dir_postfix,
|
|
const char * repl_dir_postfix, bool skip_dir_last_slash)
|
|
{
|
|
look_for_url = http_prefix;
|
|
Convert(base_url, look_for_url, false);
|
|
look_for_url += dir_prefix;
|
|
Convert(message_work.src_dir, look_for_url, false);
|
|
|
|
if( skip_dir_last_slash )
|
|
NoLastSlash(look_for_url);
|
|
|
|
look_for_url += dir_postfix;
|
|
|
|
Convert(message_work.http_server, repl_url);
|
|
repl_url += repl_dir_postfix;
|
|
|
|
ChangeLogStrings();
|
|
ReplaceString(buf, look_for_url, repl_url);
|
|
}
|
|
|
|
|
|
|
|
// second thread
|
|
// objects not locked
|
|
void ExportThread::ChangeBaseAdress(std::string & buf,
|
|
const char * dir_prefix, const char * dir_postfix,
|
|
const char * repl_dir_postfix, bool skip_dir_last_slash)
|
|
{
|
|
Lock();
|
|
ChangeBaseAdress(buf, "http://", dir_prefix, dir_postfix, repl_dir_postfix, skip_dir_last_slash);
|
|
ChangeBaseAdress(buf, "https://", dir_prefix, dir_postfix, repl_dir_postfix, skip_dir_last_slash);
|
|
Unlock();
|
|
}
|
|
|
|
|
|
// second thread
|
|
// objects not locked
|
|
void ExportThread::ChangeSiteNames(std::string & buf)
|
|
{
|
|
Lock();
|
|
|
|
// changing:
|
|
// /sitename/ -> /
|
|
Convert(message_work.src_dir, look_for_url);
|
|
repl_url = '/';
|
|
ChangeLogStrings();
|
|
ReplaceString(buf, look_for_url, repl_url);
|
|
|
|
// changing:
|
|
// /sitename" -> /"
|
|
NoLastSlash(look_for_url);
|
|
look_for_url += '\"';
|
|
repl_url = "/\"";
|
|
ChangeLogStrings();
|
|
ReplaceString(buf, look_for_url, repl_url);
|
|
|
|
Unlock();
|
|
}
|
|
|
|
|
|
// second thread
|
|
// objects not locked
|
|
void ExportThread::ChangeAdresses(std::string & buf)
|
|
{
|
|
// http:// in sources here means either http:// or https://
|
|
|
|
// changing:
|
|
// http://domain.tld/sitename/[...]/file.jpg/-/thumb -> http://otherdomain.tld/download/[...]/file.jpg
|
|
// http://domain.tld/sitename/[...]/file.jpg/download/thumb -> http://otherdomain.tld/download/[...]/file.jpg
|
|
ChangeAdressesThumb(buf);
|
|
|
|
// changing:
|
|
// http://domain.tld/static/sitename/ -> http://otherdomain.tld/static/
|
|
ChangeBaseAdress(buf, "/static", "", "/static/");
|
|
|
|
// changing:
|
|
// http://domain.tld/common/sitename/ -> http://otherdomain.tld/common/
|
|
//ChangeAdresss(buf, "/common", "", "/common/");
|
|
|
|
// changing:
|
|
// http://domain.tld/sitename/ -> http://otherdomain.tld/
|
|
ChangeBaseAdress(buf, "", "", "/");
|
|
|
|
// changing:
|
|
// http://domain.tld/sitename" -> http://otherdomain.tld"
|
|
ChangeBaseAdress(buf, "", "\"", "\"", true);
|
|
|
|
// changing:
|
|
// /sitename/ -> /
|
|
// /sitename" -> /"
|
|
ChangeSiteNames(buf);
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
} // namespace
|
|
|
|
} // namespace Winix
|
|
|