tomasz.sowa
/
winix
1
0
Fork 0
Code Issues 5 Pull Requests Releases Activity

added support for Model migrations 

now we have a table core.migration and each model (User, Group, Item, ItemContent and a new Migration) have its own
row in the table with a version number

added to config:
db_make_migration_if_needed and db_stop_if_migration_fails (need description yet)
This commit is contained in:
Tomasz Sowa 2021-05-14 03:31:29 +02:00
parent 4df10de6b7
commit a94e09f0aa
16 changed files with 674 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
winixd/core/app.cpp
View File

@ -50,7 +50,7 @@
#include "functions/functions.h"
#include "utf8/utf8.h"
#include "convert/convert.h"
#include "models/migration.h"
namespace Winix
@ -272,6 +272,52 @@ return true;
}
bool App::DoDatabaseMigration()
{
bool ok = true;
Migration migration;
User user;
ItemContent item_content;
Item item;
Group group;
ok = ok && Migration::do_migration(&model_connector, migration);
ok = ok && Migration::do_migration(&model_connector, user);
/*
* do migration of ItemContent before Item
* Item::do_migration_to_3() requires that ItemContent should have new columns
*
*/
ok = ok && Migration::do_migration(&model_connector, item_content);
ok = ok && Migration::do_migration(&model_connector, item);
ok = ok && Migration::do_migration(&model_connector, group);
return ok;
}
bool App::TryToMakeDatabaseMigration()
{
if( config.db_make_migration_if_needed )
{
if( !DoDatabaseMigration() )
{
if( config.db_stop_if_migration_fails )
{
log << log1 << "App: database migration failed, stopping winix" << logend;
return false;
}
}
}
return true;
}
bool App::Init()
{
postgresql_connector.set_conn_param(config.db_database, config.db_user, config.db_pass);
@ -283,20 +329,11 @@ bool App::Init()
model_connector.set_db_connector(postgresql_connector);
model_connector.set_logger(log);
// temporary
if( config.space.to_bool(L"do_migration_to_winix_fullmorm", false) )
{
Item item_temp;
item_temp.set_connector(model_connector);
item_temp.do_migration(&model_connector, log);
log << log1 << "Migrations complete, now remove do_migration_to_winix_fullmorm from the config" << logend;
std::exit(0);
}
if( !TryToMakeDatabaseMigration() )
return false;
db_conn.SetConnParam(config.db_database, config.db_user, config.db_pass);
db_conn.WaitForConnection();
db.PostgreSQLsmallerThan10(config.db_postgresql_smaller_than_10);
db.LogQueries(config.log_db_query);
cur.request->Clear();

4
winixd/core/app.h
View File

@ -266,6 +266,10 @@ private:
void CreateStaticTree();
bool DoDatabaseMigration();
bool TryToMakeDatabaseMigration();
// !! IMPROVE ME
// !! move to the session manager?
time_t last_sessions_save;

3
winixd/core/config.cpp
View File

@ -194,7 +194,8 @@ void Config::AssignValues(bool stdout_is_closed)
db_database = Text(L"db_database");
db_user = Text(L"db_user");
db_pass = Text(L"db_pass");
db_postgresql_smaller_than_10 = Bool(L"db_postgresql_smaller_than_10", false);
db_make_migration_if_needed = Bool(L"db_make_migration_if_needed", true);
db_stop_if_migration_fails = Bool(L"db_stop_if_migration_fails", true);
item_url_empty = Text(L"item_url_empty");

9
winixd/core/config.h
View File

@ -198,10 +198,11 @@ public:
std::wstring db_user;
std::wstring db_pass;
// is the PostgreSQL later than 10
// default false
// if true then we are not using ROW() statements in sql query
bool db_postgresql_smaller_than_10;
// make database migration if needed
//
bool db_make_migration_if_needed;
bool db_stop_if_migration_fails;
// the name of the cookie which has the session identifier
std::wstring http_session_id_name;

26
winixd/db/db.cpp
View File

@ -40,19 +40,19 @@
namespace Winix
{
void Db::PostgreSQLsmallerThan10(bool is_smaller_than_10)
{
is_postgresql_smaller_than_10 = is_smaller_than_10;
if( is_postgresql_smaller_than_10 )
{
postgrsql_row_statement.clear();
}
else
{
postgrsql_row_statement = L"ROW";
}
}
//void Db::PostgreSQLsmallerThan10(bool is_smaller_than_10)
//{
// is_postgresql_smaller_than_10 = is_smaller_than_10;
//
// if( is_postgresql_smaller_than_10 )
// {
// postgrsql_row_statement.clear();
// }
// else
// {
// postgrsql_row_statement = L"ROW";
// }
//}
/*

2
winixd/db/db.h
View File

@ -61,7 +61,7 @@ public:
is_postgresql_smaller_than_10 = false;
}
void PostgreSQLsmallerThan10(bool is_smaller_than_10);
//void PostgreSQLsmallerThan10(bool is_smaller_than_10);
/*
bool GetUserPass(const std::wstring & login, long & user_id, UserPass & up);

81
winixd/models/group.cpp Normal file
View File

@ -0,0 +1,81 @@
/*
* This file is a part of Winix
* and is distributed under the 2-Clause BSD licence.
* Author: Tomasz Sowa <t.sowa@ttmath.org>
*/
/*
* Copyright (c) 2021, Tomasz Sowa
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
*/
#include "group.h"
namespace Winix
{
bool Group::do_migration(int & current_table_version)
{
bool ok = true;
ok = ok && morm::Model::do_migration(current_table_version, 1, this, &Group::do_migration_to_1);
ok = ok && morm::Model::do_migration(current_table_version, 2, this, &Group::do_migration_to_2);
return ok;
}
bool Group::do_migration_to_1()
{
const char * str = R"sql(
CREATE TABLE core."group" (
id serial,
"group" character varying(20)
);
)sql";
db_query(str);
return true;
}
bool Group::do_migration_to_2()
{
const char * str = R"sql(
alter table core.group rename column "group" to "name";
)sql";
return db_query(str);
}
}

10
winixd/models/group.h
View File

@ -82,6 +82,16 @@ public:
name.clear();
//members.clear();
}
bool do_migration(int & current_table_version);
protected:
bool do_migration_to_1();
bool do_migration_to_2();
};

126
winixd/models/item.cpp
View File

@ -223,7 +223,64 @@ return !is_that_url;
}
void Item::do_migration(morm::ModelConnector * model_connector, Log & log)
void Item::propagate_connector()
{
item_content.set_connector(model_connector);
}
bool Item::do_migration(int & current_table_version)
{
bool ok = true;
ok = ok && morm::Model::do_migration(current_table_version, 1, this, &Item::do_migration_to_1);
ok = ok && morm::Model::do_migration(current_table_version, 2, this, &Item::do_migration_to_2);
ok = ok && morm::Model::do_migration(current_table_version, 3, this, &Item::do_migration_to_3);
ok = ok && morm::Model::do_migration(current_table_version, 4, this, &Item::do_migration_to_4);
return ok;
}
bool Item::do_migration_to_1()
{
const char * str = R"sql(
CREATE TABLE core.item (
user_id integer,
group_id integer,
privileges integer,
date_creation timestamp without time zone,
date_modification timestamp without time zone,
parent_id bigint,
type smallint,
id integer NOT NULL,
url character varying(255),
content_id bigint,
subject character varying(255),
guest_name character varying(20),
modification_user_id integer,
template character varying(255),
link_to character varying(2048),
link_redirect smallint,
sort_index integer,
meta text,
ameta text
);
)sql";
db_query(str);
return true; // IMPROVEME remove me in the future: this is only for a moment until we do migration on all our sites
}
/*
* directories didn't have an ItemContent object, so now we create such objects
*/
bool Item::do_migration_to_2()
{
morm::Finder<Item> finder(model_connector);
@ -234,17 +291,76 @@ void Item::do_migration(morm::ModelConnector * model_connector, Log & log)
eq(L"content_id", -1).
get_list();
PT::Log * log = model_connector->get_logger();
for(Item & item : list)
{
log << "updating item id: " << item.id << ", type: " << (int)item.type << ", url: " << item.url << ", subject: " << item.subject << logend << logsave;
if( log )
{
(*log) << "Item: adding a content row corresponding to item id: " << item.id << ", type: " << (int)item.type << ", url: " << item.url
<< ", subject: " << item.subject << PT::Log::logend;
}
item.item_content.set_save_mode(morm::Model::DO_INSERT_ON_SAVE);
item.save();
if( !item.save() )
return false;
}
return true;
}
void Item::propagate_connector()
bool Item::do_migration_to_3()
{
item_content.set_connector(model_connector);
const char * str[] = {
"update core.content set user_id = (select user_id from core.item where item.content_id = content.id limit 1);",
"update core.content set group_id = (select group_id from core.item where item.content_id = content.id limit 1);",
"update core.content set guest_name = (select guest_name from core.item where item.content_id = content.id limit 1);",
"update core.content set modification_user_id = (select modification_user_id from core.item where item.content_id = content.id limit 1);",
"update core.content set privileges = (select privileges from core.item where item.content_id = content.id limit 1);",
"update core.content set date_creation = (select date_creation from core.item where item.content_id = content.id limit 1);",
"update core.content set date_modification = (select date_modification from core.item where item.content_id = content.id limit 1);",
"update core.content set link_to = (select link_to from core.item where item.content_id = content.id limit 1);",
"update core.content set link_redirect = (select link_redirect from core.item where item.content_id = content.id limit 1);",
"update core.content set meta = (select meta from core.item where item.content_id = content.id limit 1);",
"update core.content set meta_admin = (select ameta from core.item where item.content_id = content.id limit 1);",
};
size_t len = sizeof(str) / sizeof(const char*);
for(size_t i=0 ; i < len ; ++i)
{
if( !db_query(str[i]) )
{
return false;
}
}
return true;
}
bool Item::do_migration_to_4()
{
const char * str = R"sql(
alter table core.item
drop column user_id,
drop column group_id,
drop column guest_name,
drop column modification_user_id,
drop column privileges,
drop column date_creation,
drop column date_modification,
drop column link_to,
drop column link_redirect,
drop column meta,
drop column ameta;
)sql";
return db_query(str);
}

11
winixd/models/item.h
View File

@ -175,17 +175,20 @@ public:
*/
/*
* temporary
*/
void do_migration(morm::ModelConnector * model_connector, Log & log);
bool do_migration(int & current_table_version);
void propagate_connector();
protected:
CalcItemsHelper calc_items_by_url(long parent_id, const std::wstring & url);
bool do_migration_to_1();
bool do_migration_to_2();
bool do_migration_to_3();
bool do_migration_to_4();
};

93
winixd/models/itemcontent.cpp
View File

@ -165,5 +165,98 @@ bool ItemContent::CanContentBeHtmlFiltered()
}
bool ItemContent::do_migration(int & current_table_version)
{
bool ok = true;
ok = ok && morm::Model::do_migration(current_table_version, 1, this, &ItemContent::do_migration_to_1);
ok = ok && morm::Model::do_migration(current_table_version, 2, this, &ItemContent::do_migration_to_2);
ok = ok && morm::Model::do_migration(current_table_version, 3, this, &ItemContent::do_migration_to_3);
return ok;
}
bool ItemContent::do_migration_to_1()
{
const char * str = R"sql(
CREATE TABLE core.content (
id serial,
content text,
content_type smallint,
file_path character varying(2048),
file_fs smallint,
file_type smallint,
has_thumb smallint,
ref integer,
modify_index smallint,
hash character varying(255),
hash_type smallint,
file_size bigint
);
)sql";
db_query(str);
return true; // IMPROVEME remove me in the future: this is only for a moment until we do migration on all our sites
}
bool ItemContent::do_migration_to_2()
{
const char * str = R"sql(
alter table core.content
add column user_id integer,
add column group_id integer,
add column guest_name character varying(20),
add column modification_user_id integer,
add column privileges integer,
add column date_creation timestamp without time zone,
add column date_modification timestamp without time zone,
add column link_to character varying(2048),
add column link_redirect smallint,
add column meta text,
add column meta_admin text,
add column content_parsed text,
add column content_parsed_type smallint;
)sql";
return db_query(str);
}
bool ItemContent::do_migration_to_3()
{
const char * str[] = {
"alter table core.content rename column ref to \"references\";",
"alter table core.content rename column content to content_raw;",
"alter table core.content rename column content_type to content_raw_type;",
"alter table core.content rename column has_thumb to file_has_thumb;",
"alter table core.content rename column hash to file_hash;",
"alter table core.content rename column hash_type to file_hash_type;",
"alter table core.content drop column modify_index;",
"alter table core.content add column file_has_thumb_new boolean;",
"update core.content as c1 set file_has_thumb_new = (select case when file_has_thumb <> 0 then true else false end from core.content as c2 where c1.id = c2.id);",
"alter table core.content drop column file_has_thumb;",
"alter table core.content rename file_has_thumb_new to file_has_thumb;",
};
size_t len = sizeof(str) / sizeof(const char*);
for(size_t i=0 ; i < len ; ++i)
{
if( !db_query(str[i]) )
{
return false;
}
}
return true;
}
} // namespace Winix

9
winixd/models/itemcontent.h
View File

@ -230,6 +230,15 @@ public:
bool CanContentBeHtmlFiltered();
bool do_migration(int & current_table_version);
protected:
bool do_migration_to_1();
bool do_migration_to_2();
bool do_migration_to_3();
};

133
winixd/models/migration.cpp Normal file
View File

@ -0,0 +1,133 @@
/*
* This file is a part of Winix
* and is distributed under the 2-Clause BSD licence.
* Author: Tomasz Sowa <t.sowa@ttmath.org>
*/
/*
* Copyright (c) 2021, Tomasz Sowa
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
*/
#include "migration.h"
#include "core/log.h"
#include "finder.h"
#include <ctime>
namespace Winix
{
bool Migration::do_migration(morm::ModelConnector * model_connector, morm::Model & model)
{
model.set_connector(model_connector);
std::wstring table_name;
model.get_table_name(table_name);
morm::Finder<Migration> finder(model_connector);
Migration migration = finder.
select().
where().
eq(L"table_name", table_name).
get();
int current_table_version = 0;
int old_table_version = 0;
if( migration.found() )
{
old_table_version = current_table_version = migration.table_version;
}
else
{
migration.set_save_mode(morm::Model::DO_INSERT_ON_SAVE);
migration.table_name = table_name;
}
migration.migration_date.FromTime(std::time(0));
bool migration_status = model.do_migration(current_table_version);
if( current_table_version != old_table_version )
{
migration.table_version = current_table_version;
migration_status = migration.save();
PT::Log * log = model_connector->get_logger();
if( log )
{
(*log) << PT::Log::log2 << "Migration: table " << table_name << " has been migrated to version " << current_table_version << PT::Log::logend;
}
if( !migration_status && log )
{
(*log) << PT::Log::log1 << "Migration: table " << table_name << " has been migrated to version " << current_table_version;
(*log) << " but there was a problem with saving this information in the migration table." << PT::Log::logend;
(*log) << "Make sure the migration table is created and save/update following record there:" << PT::Log::logend;
(*log) << migration << PT::Log::logend;
}
}
return migration_status;
}
bool Migration::do_migration(int & current_table_version)
{
bool ok = true;
ok = ok && morm::Model::do_migration(current_table_version, 1, this, &Migration::do_migration_to_1);
return ok;
}
bool Migration::do_migration_to_1()
{
// IMPROVEME
// what about 'create schema core'?
// may the name of a schema (core) should be a parameter in the config?
const char * str = R"sql(
create table core.migration (
id serial,
table_name varchar(255),
table_version int,
migration_date timestamp without time zone,
primary key(id)
);
)sql";
return db_query(str);
}
}

96
winixd/models/migration.h Normal file
View File

@ -0,0 +1,96 @@
/*
* This file is a part of Winix
* and is distributed under the 2-Clause BSD licence.
* Author: Tomasz Sowa <t.sowa@ttmath.org>
*/
/*
* Copyright (c) 2021, Tomasz Sowa
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
*/
#ifndef headerfile_winix_models_migration
#define headerfile_winix_models_migration
#include <string>
#include <vector>
#include "model.h"
namespace Winix
{
class Migration : public morm::Model
{
public:
long id;
std::wstring table_name;
int table_version;
PT::Date migration_date;
void fields()
{
field(L"id", id, morm::FT::no_insertable | morm::FT::no_updatable | morm::FT::primary_key);
field(L"table_name", table_name);
field(L"table_version", table_version);
field(L"migration_date", migration_date);
}
void table()
{
morm::Model::table_name(L"core", L"migration");
}
void after_insert()
{
get_last_sequence_for_primary_key(L"core.migration_id_seq", id);
}
static bool do_migration(morm::ModelConnector * model_connector, morm::Model & model);
bool do_migration(int & current_table_version);
private:
bool do_migration_to_1();
};
} // namespace Winix
#endif

41
winixd/models/user.cpp
View File

@ -137,6 +137,47 @@ return false;
}
bool User::do_migration(int & current_table_version)
{
bool ok = true;
ok = ok && morm::Model::do_migration(current_table_version, 1, this, &User::do_migration_to_1);
return ok;
}
bool User::do_migration_to_1()
{
const char * str = R"sql(
CREATE TABLE core."user" (
id serial,
login character varying(255),
password character varying(255),
email character varying(255),
notify integer,
pass_type integer,
pass_hash_salted boolean,
pass_encrypted bytea,
super_user boolean,
env text,
aenv text,
status integer,
locale_id integer,
time_zone_id integer,
has_pass boolean
);
)sql";
db_query(str);
return true; // IMPROVEME remove me in the future: this is only for a moment until we do migration on all our sites
}

9
winixd/models/user.h
View File

@ -142,6 +142,15 @@ public:
bool SetTzFromEnv();
void clear_passwords();
bool do_migration(int & current_table_version);
private:
bool do_migration_to_1();
};