add a two ways algorithm for building a hints file for the otp backend

This commit is contained in:
Tomasz Sowa 2024-04-16 15:41:49 +02:00
parent e8ac579fcb
commit a652a9d230
No known key found for this signature in database
GPG Key ID: 58A472D1C9399FED
9 changed files with 256 additions and 76 deletions

3
.gitignore vendored
View File

@ -9,3 +9,6 @@ lookup-dir/
m
.qtcreator/
build.sh
map_test.osm
.clangd
.qtc_clangd/

View File

@ -1,5 +1,5 @@
# DO NOT DELETE
./finder.o: finder.h params.h misc.h
./params.o: params.h
./misc.o: misc.h
./params.o: params.h

View File

@ -47,10 +47,18 @@ namespace allplacefinder
Finder::Finder()
{
params = nullptr;
clear();
}
void Finder::clear()
{
params = nullptr;
second_phase = false;
nodes = 0;
ways = 0;
ways_table.clear();
}
bool Finder::create_lookup_dir(Params & params)
@ -67,7 +75,7 @@ bool Finder::create_lookup_dir(Params & params)
return false;
}
if( !open_file(params.osm_file, osm_file_str, "I cannot open the osm file for reading") )
if( !open_file_for_reading(params.osm_file, osm_file_str, "I cannot open the osm file for reading") )
return false;
if( !dir_exists(params.lookup_dir ) )
@ -77,11 +85,44 @@ bool Finder::create_lookup_dir(Params & params)
}
this->params = &params;
return create_lookup_dir();
return parse_osm_file();
}
bool Finder::create_lookup_dir()
bool Finder::create_hints_file(Params & params)
{
if( params.osm_file.empty() )
{
std::cerr << "you have to provide a path to osm file in --osm-file parameter" << std::endl;
return false;
}
if( params.hints_file_name.empty() )
{
std::cerr << "you have to provide a path to the hints file in --hints_file_name parameter" << std::endl;
return false;
}
if( !open_file_for_writing(params.hints_file_name, hints_file, "I cannot open the hints file for writing") )
return false;
this->params = &params;
bool status = parse_osm_file();
if( status && params.use_two_ways_hints_file_algorithm )
{
std::cout << "places found: " << ways_table.size() << std::endl;
std::cout << "doing a second phase" << std::endl;
second_phase = true;
status = parse_osm_file();
}
return status;
}
bool Finder::parse_osm_file()
{
pt::HTMLParser parser;
pt::Space space;
@ -108,34 +149,32 @@ bool Finder::create_lookup_dir()
break;
}
//std::cout << "nodes: " << nodes << std::endl;
//std::cout << "ways: " << ways << std::endl;
std::cout << "nodes: " << nodes << std::endl;
std::cout << "ways: " << ways << std::endl;
return true;
}
bool Finder::item_parsed(const pt::HTMLParser::Item & item)
void Finder::item_parsed(const pt::HTMLParser::Item & item)
{
std::string name_utf8;
pt::wide_to_utf8(item.name, name_utf8);
//std::cout << name_utf8 << std::endl;
if( item.name == L"node" )
{
nodes += 1;
parse_node(*item.space);
return false;
}
if( item.name == L"way" )
{
ways += 1;
parse_way(*item.space);
return false;
}
}
return true;
bool Finder::should_remove(const pt::HTMLParser::Item & item)
{
return (item.name == L"node" || item.name == L"way" || item.name == L"relation" || item.name == L"bounds" );
}
@ -157,15 +196,29 @@ void Finder::parse_node(pt::Space & node)
if( id_long >= 0 )
{
out.add(L"id", id_long);
out.add(L"lat", lat->to_wstr());
out.add(L"lon", lon->to_wstr());
if( !save_file(L"node", id_long, out) )
std::wstring lat_str = lat->to_wstr();
std::wstring lon_str = lon->to_wstr();
out.add(L"lat", lat_str);
out.add(L"lon", lon_str);
if( params->create_lookup_dir )
{
std::cout << "I cannot save node file with id: " << id_long << std::endl;
if( !save_file(L"node", id_long, out) )
{
std::cout << "I cannot save node file with id: " << id_long << std::endl;
}
}
if( params->create_hints_file )
{
if( params->use_two_ways_hints_file_algorithm && second_phase )
{
check_way(id_long, lat_str, lon_str);
}
}
//pt::Space * childs = node.get_space(L"childs");
}
else
{
@ -188,7 +241,6 @@ void Finder::parse_way(pt::Space & way)
pt::Space * attr = way.get_space(L"attr");
pt::Space out;
if( attr )
{
pt::Space * id = attr->get_space(L"id");
@ -207,17 +259,9 @@ void Finder::parse_way(pt::Space & way)
{
if( parse_way_childs(out, *childs) )
{
std::string str;
str = out.serialize_to_json_str(false);
std::cout << str << std::endl;
// if( !save_file(L"way", id_long, out) )
// {
// std::cout << "I cannot save way file with id: " << id_long << std::endl;
// }
}
}
}
@ -231,9 +275,22 @@ void Finder::parse_way(pt::Space & way)
bool Finder::parse_way_childs(pt::Space & out, pt::Space & childs)
{
size_t nd_len = 0;
bool has_name = false;
bool has_name = parse_way_name(out, childs, nd_len);
bool has_node = false;
if( nd_len > 0 && has_name )
{
has_node = parse_way_middle_node(out, childs, nd_len);
}
return nd_len > 0 && has_name && has_node;
}
bool Finder::parse_way_name(pt::Space & out, pt::Space & childs, size_t & nd_len)
{
bool has_name = false;
if( childs.is_table() )
{
for(pt::Space * child : childs.value.value_table)
@ -265,44 +322,62 @@ bool Finder::parse_way_childs(pt::Space & out, pt::Space & childs)
}
}
}
}
if( nd_len > 0 && has_name )
return has_name;
}
bool Finder::parse_way_middle_node(pt::Space & out, pt::Space & childs, size_t nd_len)
{
bool has_node = false;
size_t nd_mid = (nd_len / 2);
size_t index = 0;
if( childs.is_table() )
{
for(pt::Space * child : childs.value.value_table)
{
size_t nd_mid = (nd_len / 2);
size_t index = 0;
std::wstring * name = child->get_wstr(L"name");
for(pt::Space * child : childs.value.value_table)
if( name && *name == L"nd" )
{
std::wstring * name = child->get_wstr(L"name");
if( name && *name == L"nd" )
if( index == nd_mid )
{
if( index == nd_mid )
pt::Space * attr = child->get_space(L"attr");
if( attr )
{
pt::Space * attr = child->get_space(L"attr");
std::wstring * ref = attr->get_wstr(L"ref");
if( attr )
if( ref )
{
std::wstring * ref = attr->get_wstr(L"ref");
out.add(L"node_id", *ref);
long node_id = pt::to_l(ref->c_str());
if( ref )
if( params->create_hints_file && params->use_two_ways_hints_file_algorithm )
{
if( !second_phase )
{
ways_table.insert(std::make_pair(node_id, out));
}
}
else
{
out.add(L"node_id", *ref);
long node_id = pt::to_l(ref->c_str());
has_node = get_node(node_id, out);
}
}
break;
}
index += 1;
break;
}
index += 1;
}
}
}
return nd_len > 0 && has_name && has_node;
return has_node;
}
@ -411,7 +486,43 @@ bool Finder::save_file(const wchar_t * subdir, long id, pt::Space & space)
}
bool Finder::check_way(long node_id, std::wstring & lat, std::wstring & lon)
{
bool found = false;
std::map<long, pt::Space>::iterator i = ways_table.find(node_id);
if( i != ways_table.end() )
{
pt::Space & way = i->second;
long id = way.to_long(L"id");
std::wstring * name = way.get_wstr(L"name");
long node_id = way.to_long(L"node_id");
if( name )
{
pt::TextStream str;
str << "{\"id\":" << id;
str << ",\"lat\":\"";
pt::esc_to_json(lat, str);
str << "\",\"lon\":\"";
pt::esc_to_json(lon, str);
str << "\",\"name\":\"";
pt::esc_to_json(*name, str);
str << "\",\"node_id\":" << node_id;
str << "}";
std::string str_utf8;
str.to_str(str_utf8);
hints_file << str_utf8 << std::endl;
}
found = true;
}
return found;
}
} // namespace

View File

@ -5,7 +5,7 @@
*/
/*
* Copyright (c) 2021, Tomasz Sowa
* Copyright (c) 2021-2024, Tomasz Sowa
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -43,32 +43,41 @@
namespace allplacefinder
{
class Finder : public pt::HTMLParser::ItemParsedListener
class Finder : public pt::HTMLParser::Listener
{
public:
Finder();
bool create_lookup_dir(Params & params);
bool create_hints_file(Params & params);
void clear();
private:
std::ifstream osm_file_str;
std::ifstream lookup_dir_str;
std::ofstream hints_file;
Params * params;
pt::TextStream text_stream;
size_t nodes;
size_t ways;
bool second_phase;
std::map<long, pt::Space> ways_table;
bool create_lookup_dir();
bool item_parsed(const pt::HTMLParser::Item & item) override;
bool parse_osm_file();
void item_parsed(const pt::HTMLParser::Item & item) override;
bool should_remove(const pt::HTMLParser::Item & item) override;
void parse_node(pt::Space & node);
void parse_way(pt::Space & way);
bool parse_way_childs(pt::Space & out, pt::Space & childs);
bool parse_way_name(pt::Space & out, pt::Space & childs, size_t & nd_len);
bool parse_way_middle_node(pt::Space & out, pt::Space & childs, size_t nd_len);
bool save_file(const wchar_t * subdir, long id, pt::Space & space);
bool get_node(long node_id, pt::Space & out);
bool check_way(long node_id, std::wstring & lat, std::wstring & lon);
};

View File

@ -5,7 +5,7 @@
*/
/*
* Copyright (c) 2021, Tomasz Sowa
* Copyright (c) 2021-2024, Tomasz Sowa
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -69,7 +69,7 @@ bool dir_exists(const std::wstring & file_path)
bool open_file(const std::wstring & file_path, std::ifstream & str, const char * err_msg)
bool open_file_for_reading(const std::wstring & file_path, std::ifstream & str, const char * err_msg)
{
std::string file_path_utf8;
pt::wide_to_utf8(file_path, file_path_utf8);
@ -87,6 +87,24 @@ bool open_file(const std::wstring & file_path, std::ifstream & str, const char *
}
bool open_file_for_writing(const std::wstring & file_path, std::ofstream & str, const char * err_msg)
{
std::string file_path_utf8;
pt::wide_to_utf8(file_path, file_path_utf8);
str.clear();
str.open(file_path_utf8.c_str(), std::ios_base::out | std::ios_base::binary);
if( !str )
{
std::cerr << err_msg << std::endl;
return false;
}
return true;
}
void create_id_str(long id, std::vector<std::wstring> & path)
{

View File

@ -5,7 +5,7 @@
*/
/*
* Copyright (c) 2021, Tomasz Sowa
* Copyright (c) 2021-2024, Tomasz Sowa
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -45,7 +45,8 @@ namespace allplacefinder
bool file_exists(const std::wstring & file_path);
bool dir_exists(const std::wstring & file_path);
bool open_file(const std::wstring & file_path, std::ifstream & str, const char * err_msg);
bool open_file_for_reading(const std::wstring & file_path, std::ifstream & str, const char * err_msg);
bool open_file_for_writing(const std::wstring & file_path, std::ofstream & str, const char * err_msg);
void create_id_str(long id, std::vector<std::wstring> & path);

View File

@ -52,6 +52,10 @@ void Params::clear()
osm_file.clear();
lookup_dir.clear();
create_hints_file = false;
hints_file_name.clear();
use_two_ways_hints_file_algorithm = false;
}

View File

@ -54,7 +54,9 @@ public:
std::wstring osm_file;
std::wstring lookup_dir;
bool create_hints_file;
std::wstring hints_file_name;
bool use_two_ways_hints_file_algorithm;
};

View File

@ -5,7 +5,7 @@
*/
/*
* Copyright (c) 2021, Tomasz Sowa
* Copyright (c) 2021-2024, Tomasz Sowa
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -38,13 +38,30 @@
#include "params.h"
#define ALLPLACEFINDER_VERSION_MAJOR 0
#define ALLPLACEFINDER_VERSION_MINOR 2
#define ALLPLACEFINDER_VERSION_PATCH 0
#define ALLPLACEFINDER_COPYRIGHT_YEAR_FROM 2021
#define ALLPLACEFINDER_COPYRIGHT_YEAR_TO 2024
void print_syntax()
{
std::cout << "placefinder options:" << std::endl;
std::cout << " -h print this help" << std::endl;
std::cout << " --osm-file file_name - a path to osm file name" << std::endl;
std::cout << " --lookup-dir dir_name - a path to lookup dir" << std::endl;
std::cout << " --create-lookup-dir - create a lookup dir, you have to provide a path in --lookup-dir parameter" << std::endl;
std::cout << "placefinder " << ALLPLACEFINDER_VERSION_MAJOR << "." << ALLPLACEFINDER_VERSION_MINOR << "." << ALLPLACEFINDER_VERSION_PATCH << std::endl;
std::cout << "Copyright (c) " << ALLPLACEFINDER_COPYRIGHT_YEAR_FROM << "-" << ALLPLACEFINDER_COPYRIGHT_YEAR_TO << ", Tomasz Sowa" << std::endl;
std::cout << std::endl;
std::cout << "Usage: placefinder [options]" << std::endl;
std::cout << "Options:" << std::endl;
std::cout << " -h print this help" << std::endl;
std::cout << " --osm-file file_name - a path to osm file name" << std::endl;
std::cout << " --lookup-dir dir_name - a path to lookup dir" << std::endl;
std::cout << " --create-lookup-dir - create a lookup dir, you have to provide a path in --lookup-dir parameter" << std::endl;
std::cout << " --create-hints-file - create a hints file used by the backend-winix-otp" << std::endl;
std::cout << " --hints-file-name file_name - a hints file name used when using --create-hints-file" << std::endl;
std::cout << " --use-two-ways-hints-file-alg - use a two way algorithm when building the hints file" << std::endl;
}
@ -76,6 +93,9 @@ bool parse_parameters(const pt::Space & parameters, allplacefinder::Params & par
const pt::Space * osm_file = parameters.get_space(L"osm-file");
const pt::Space * lookup_dir = parameters.get_space(L"lookup-dir");
const pt::Space * create_lookup_dir = parameters.get_space(L"create-lookup-dir");
const pt::Space * create_hints_file = parameters.get_space(L"create-hints-file");
const pt::Space * hints_file_name = parameters.get_space(L"hints-file-name");
const pt::Space * use_two_ways_hints_file_alg = parameters.get_space(L"use-two-ways-hints-file-alg");
bool ok = true;
if( osm_file )
@ -88,7 +108,14 @@ bool parse_parameters(const pt::Space & parameters, allplacefinder::Params & par
ok = ok && parse_parametr(lookup_dir, params.lookup_dir, "--lookup-dir parametr can be used only once");
}
if( hints_file_name )
{
ok = ok && parse_parametr(hints_file_name, params.hints_file_name, "--hints-file-name parametr can be used only once");
}
params.create_lookup_dir = (create_lookup_dir != nullptr);
params.create_hints_file = (create_hints_file != nullptr);
params.use_two_ways_hints_file_algorithm = (use_two_ways_hints_file_alg != nullptr);
return ok;
}
@ -97,17 +124,32 @@ int make_action(allplacefinder::Params & params)
{
int status = 3;
if( params.create_lookup_dir && params.create_hints_file )
{
std::cerr << "You cannot use both --create-lookup-dir and --create-hints-file parameters" << std::endl;
return 5; // IMPROVEME put these constants somewhere
}
if( params.create_lookup_dir )
{
allplacefinder::Finder finder;
status = finder.create_lookup_dir(params) ? 0 : 4;
}
else
if( params.create_hints_file )
{
allplacefinder::Finder finder;
status = finder.create_hints_file(params) ? 0 : 4;
}
else
{
print_syntax();
}
return status;
}
int main(int argc, const char ** argv)
{
pt::Space options, arguments;
@ -116,6 +158,7 @@ allplacefinder::Params params;
arguments.add(L"osm-file", 1);
arguments.add(L"lookup-dir", 1);
arguments.add(L"hints-file-name", 1);
pt::MainOptionsParser::Status status = options_parser.parse(argc, argv, options, arguments);
@ -133,17 +176,6 @@ allplacefinder::Params params;
if( parse_parameters(options, params) )
{
// ------------------
// for debug purposes
//params.create_lookup_dir = true;
//params.osm_file = L"/home/tomek/data/test.xml";
//params.osm_file = L"/home/tomek/data/wroclaw_map.xml";
//params.lookup_dir = L"/home/tomek/data/lookup-dir";
//params.lookup_dir = L"/data/lookup-dir";
// ------------------
return make_action(params);
}