added: Finder is able to create a lookup-dir with nodes, as an input we use Open Street Map xml file

placefinder tool support now: --osm-file file_path, --lookup-dir dir_path, --create-lookup-dir parameters
This commit is contained in:
Tomasz Sowa 2021-11-30 16:37:59 +01:00
parent 38659815c7
commit e63b13ab0c
11 changed files with 782 additions and 13 deletions

3
.gitignore vendored
View File

@ -2,3 +2,6 @@
.project .project
.settings .settings
data data
*.o
*.a
placefinder/placefinder

View File

@ -1,7 +1,10 @@
CXX = g++10 CXX = g++10
#CXX = clang++
CXXFLAGS = -Wl,-rpath=/usr/local/lib/gcc10 -Wfatal-errors -fPIC -Wall -pedantic -O0 -g3 -pthread -std=c++20 -I../../pikotools/src -I../lib -I/usr/local/include CXXFLAGS = -Wl,-rpath=/usr/local/lib/gcc10 -Wfatal-errors -fPIC -Wall -pedantic -O0 -g3 -pthread -std=c++20 -I../../pikotools/src -I../lib -I/usr/local/include
#CXXFLAGS = -fsanitize=address -Wfatal-errors -fPIC -Wall -pedantic -O0 -g3 -pthread -std=c++20 -I../../pikotools/src -I../lib -I/usr/local/include
LDFLAGS = -L/usr/local/lib LDFLAGS = -L/usr/local/lib
#-fsanitize=address #-fsanitize=address
@ -12,14 +15,14 @@ export CXXFLAGS
export LDFLAGS export LDFLAGS
export AR export AR
all: lib placefinder pikotools all: lib placefinder
lib: FORCE pikotools lib: FORCE pikotools
$(MAKE) -C lib $(MAKE) -C lib
placefinder: FORCE pikotools placefinder: FORCE pikotools lib
$(MAKE) -C placefinder $(MAKE) -C placefinder
@ -30,6 +33,7 @@ pikotools: FORCE
clean: FORCE clean: FORCE
$(MAKE) -C lib clean $(MAKE) -C lib clean
$(MAKE) -C placefinder clean $(MAKE) -C placefinder clean
$(MAKE) -C ../pikotools clean
depend: FORCE depend: FORCE

View File

@ -1,3 +1,5 @@
# DO NOT DELETE # DO NOT DELETE
./finder.o: finder.h ./finder.o: finder.h params.h misc.h
./params.o: params.h
./misc.o: misc.h

View File

@ -32,7 +32,13 @@
* *
*/ */
#include <iostream>
#include <sys/stat.h>
#include "finder.h" #include "finder.h"
#include "utf8/utf8.h"
#include "misc.h"
#include "convert/convert.h"
#include "space/spaceparser.h"
namespace allplacefinder namespace allplacefinder
@ -41,12 +47,370 @@ namespace allplacefinder
Finder::Finder() Finder::Finder()
{ {
params = nullptr;
} }
bool Finder::create_lookup_dir(Params & params)
{
if( params.osm_file.empty() )
{
std::cerr << "you have to provide a path to osm file in --osm-file parameter" << std::endl;
return false;
}
if( params.lookup_dir.empty() )
{
std::cerr << "you have to provide a path to lookup dir in --lookup-dir parameter" << std::endl;
return false;
}
if( !open_file(params.osm_file, osm_file_str, "I cannot open the osm file for reading") )
return false;
if( !dir_exists(params.lookup_dir ) )
{
std::cerr << "There is no the lookup directory" << std::endl;
return false;
}
this->params = &params;
return create_lookup_dir();
}
bool Finder::create_lookup_dir()
{
pt::HTMLParser parser;
pt::Space space;
std::string file_path;
pt::wide_to_utf8(params->osm_file, file_path);
nodes = 0;
ways = 0;
parser.set_item_parsed_listener(this);
pt::HTMLParser::Status status = parser.parse_xml_file(file_path, space, false);
switch(status)
{
case pt::HTMLParser::cant_open_file:
std::cerr << "I cannot open the input file: " << file_path << std::endl;
break;
case pt::HTMLParser::syntax_error:
std::cerr << "Syntax error: " << file_path << ":" << parser.get_last_parsed_line() << std::endl;
break;
case pt::HTMLParser::ok:
//std::cout << space.serialize_to_json_str(true) << std::endl;
break;
}
//std::cout << "nodes: " << nodes << std::endl;
//std::cout << "ways: " << ways << std::endl;
return true;
}
bool Finder::item_parsed(const pt::HTMLParser::Item & item)
{
std::string name_utf8;
pt::wide_to_utf8(item.name, name_utf8);
//std::cout << name_utf8 << std::endl;
if( item.name == L"node" )
{
nodes += 1;
parse_node(*item.space);
return false;
}
if( item.name == L"way" )
{
ways += 1;
parse_way(*item.space);
return false;
}
return true;
}
void Finder::parse_node(pt::Space & node)
{
pt::Space * attr = node.get_space(L"attr");
pt::Space out;
if( attr )
{
pt::Space * id = attr->get_space(L"id");
pt::Space * lat = attr->get_space(L"lat");
pt::Space * lon = attr->get_space(L"lon");
if( id && lat && lon )
{
long id_long = id->to_long();
if( id_long >= 0 )
{
out.add(L"id", id_long);
out.add(L"lat", lat->to_wstr());
out.add(L"lon", lon->to_wstr());
if( !save_file(L"node", id_long, out) )
{
std::cout << "I cannot save node file with id: " << id_long << std::endl;
}
//pt::Space * childs = node.get_space(L"childs");
}
else
{
std::cout << "Skipping node with negative id" << std::endl;
}
}
else
{
std::cout << "Skipping node" << std::endl;
}
}
//std::cout << out.serialize_to_json_str(true) << std::endl;
}
void Finder::parse_way(pt::Space & way)
{
pt::Space * attr = way.get_space(L"attr");
pt::Space out;
if( attr )
{
pt::Space * id = attr->get_space(L"id");
if( id )
{
long id_long = id->to_long();
if( id_long >= 0 )
{
out.add(L"id", id_long);
pt::Space * childs = way.get_space(L"childs");
if( childs )
{
if( parse_way_childs(out, *childs) )
{
std::string str;
str = out.serialize_to_json_str(false);
std::cout << str << std::endl;
// if( !save_file(L"way", id_long, out) )
// {
// std::cout << "I cannot save way file with id: " << id_long << std::endl;
// }
}
}
}
}
}
//std::cout << out.serialize_to_json_str(true) << std::endl;
}
bool Finder::parse_way_childs(pt::Space & out, pt::Space & childs)
{
size_t nd_len = 0;
bool has_name = false;
bool has_node = false;
if( childs.is_table() )
{
for(pt::Space * child : childs.value.value_table)
{
std::wstring * name = child->get_wstr(L"name");
if( name )
{
if( *name == L"nd" )
{
nd_len += 1;
}
if( *name == L"tag" )
{
pt::Space * attr = child->get_space(L"attr");
if( attr )
{
std::wstring * k = attr->get_wstr(L"k");
std::wstring * v = attr->get_wstr(L"v");
if( k && v && *k == L"name" )
{
has_name = true;
out.add(L"name", *v);
}
}
}
}
}
if( nd_len > 0 && has_name )
{
size_t nd_mid = (nd_len / 2);
size_t index = 0;
for(pt::Space * child : childs.value.value_table)
{
std::wstring * name = child->get_wstr(L"name");
if( name && *name == L"nd" )
{
if( index == nd_mid )
{
pt::Space * attr = child->get_space(L"attr");
if( attr )
{
std::wstring * ref = attr->get_wstr(L"ref");
if( ref )
{
out.add(L"node_id", *ref);
long node_id = pt::Tol(ref->c_str());
has_node = get_node(node_id, out);
}
}
break;
}
index += 1;
}
}
}
}
return nd_len > 0 && has_name && has_node;
}
bool Finder::get_node(long node_id, pt::Space & out)
{
std::vector<std::wstring> subdirs;
create_id_str(node_id, subdirs);
std::wstring path;
path = params->lookup_dir;
path += L"/";
path += L"node";
for(std::wstring & str : subdirs)
{
path += L"/";
path += str;
}
pt::SpaceParser parser;
pt::Space node;
pt::SpaceParser::Status status = parser.parse_json_file(path, node);
if( status == pt::SpaceParser::ok )
{
std::wstring * lat = node.get_wstr(L"lat");
std::wstring * lon = node.get_wstr(L"lon");
if( lat && lon )
{
out.add(L"lat", *lat);
out.add(L"lon", *lon);
return true;
}
}
return false;
}
bool Finder::save_file(const wchar_t * subdir, long id, pt::Space & space)
{
std::vector<std::wstring> subdirs;
//std::string p;
create_id_str(id, subdirs);
// for(size_t i = 0 ; i < path.size() ; ++i)
// {
// pt::wide_to_utf8(path[i], p);
// std::cout << p << " / ";
// }
//
// std::cout << std::endl;
std::wstring path;
path = params->lookup_dir;
path += L"/";
path += subdir;
if( !create_dir_if_not_exists(path) )
return false;
for(size_t i = 0 ; i + 1 < subdirs.size() ; ++i)
{
path += L"/";
path += subdirs[i];
if( !create_dir_if_not_exists(path) )
return false;
}
if( !subdirs.empty() )
{
path += L"/";
path += subdirs.back();
std::string path_utf8;
pt::wide_to_utf8(path, path_utf8);
std::ofstream str(path_utf8, std::ios_base::out | std::ios_base::binary);
if( str )
{
text_stream.clear();
space.serialize_to_json_stream(text_stream, true);
text_stream << '\n';
save_stream(text_stream, str);
}
else
{
return false;
}
str.close();
if( !str )
return false;
}
return true;
}

View File

@ -35,19 +35,41 @@
#ifndef headerfile_allplacefinder_lib_finder #ifndef headerfile_allplacefinder_lib_finder
#define headerfile_allplacefinder_lib_finder #define headerfile_allplacefinder_lib_finder
#include <fstream>
#include "html/htmlparser.h"
#include "params.h"
namespace allplacefinder namespace allplacefinder
{ {
class Finder class Finder : public pt::HTMLParser::ItemParsedListener
{ {
public: public:
Finder(); Finder();
bool create_lookup_dir(Params & params);
private:
std::ifstream osm_file_str;
std::ifstream lookup_dir_str;
Params * params;
pt::TextStream text_stream;
size_t nodes;
size_t ways;
bool create_lookup_dir();
bool item_parsed(const pt::HTMLParser::Item & item) override;
void parse_node(pt::Space & node);
void parse_way(pt::Space & way);
bool parse_way_childs(pt::Space & out, pt::Space & childs);
bool save_file(const wchar_t * subdir, long id, pt::Space & space);
bool get_node(long node_id, pt::Space & out);
}; };
} // namespace } // namespace

187
lib/misc.cpp Normal file
View File

@ -0,0 +1,187 @@
/*
* This file is a part of AllPlaceFinder library
* and is distributed under the 2-Clause BSD licence.
* Author: Tomasz Sowa <t.sowa@ttmath.org>
*/
/*
* Copyright (c) 2021, Tomasz Sowa
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
*/
#include <iostream>
#include <sys/stat.h>
#include "utf8/utf8.h"
#include "misc.h"
#include "convert/convert.h"
namespace allplacefinder
{
bool file_exists(const std::wstring & file_path)
{
struct stat sb;
std::string file_path_utf8;
pt::wide_to_utf8(file_path, file_path_utf8);
if( stat(file_path_utf8.c_str(), &sb) != 0 )
return false;
return (sb.st_mode & S_IFREG) != 0;
}
bool dir_exists(const std::wstring & file_path)
{
struct stat sb;
std::string file_path_utf8;
pt::wide_to_utf8(file_path, file_path_utf8);
if( stat(file_path_utf8.c_str(), &sb) != 0 )
return false;
return (sb.st_mode & S_IFDIR) != 0;
}
bool open_file(const std::wstring & file_path, std::ifstream & str, const char * err_msg)
{
std::string file_path_utf8;
pt::wide_to_utf8(file_path, file_path_utf8);
str.clear();
str.open(file_path_utf8.c_str(), std::ios_base::in | std::ios_base::binary);
if( !str )
{
std::cerr << err_msg << std::endl;
return false;
}
return true;
}
void create_id_str(long id, std::vector<std::wstring> & path)
{
wchar_t buffer[64];
size_t len = sizeof(buffer) / sizeof(wchar_t);
size_t str_len = 0;
std::wstring d;
path.clear();
pt::Toa(id, buffer, len, 16, &str_len);
if( str_len == 0 )
return;
size_t padding = 0;
if( str_len < 16 )
padding = 16 - str_len;
d = L"00";
padding = padding & (~1);
for(size_t i=0 ; i < padding ; i += 2)
{
path.push_back(d);
}
d.clear();
size_t i = 0;
if( (str_len & 1) != 0 )
{
d = L"0";
d += buffer[0];
i = 1;
path.push_back(d);
}
for( ; i + 1 < str_len ; i += 2)
{
d = buffer[i];
d += buffer[i+1];
path.push_back(d);
}
}
bool create_dir(const std::wstring & path)
{
std::string path_utf8;
pt::wide_to_utf8(path, path_utf8);
return mkdir(path_utf8.c_str(), 0755) == 0;
}
bool create_dir_if_not_exists(const std::wstring & path)
{
if( !dir_exists(path) )
{
if( !create_dir(path) )
{
std::cout << "I cannot create directory: ";
print(path);
std::cout << std::endl;
return false;
}
}
return true;
}
void print(const std::wstring & str)
{
std::string str_utf8;
pt::wide_to_utf8(str, str_utf8);
std::cout << str_utf8;
}
void save_stream(const pt::TextStream & in, std::ofstream & out)
{
pt::TextStream::const_iterator i = in.begin();
for( ; i != in.end() ; ++i)
{
out << *i;
}
}
} // namespace

62
lib/misc.h Normal file
View File

@ -0,0 +1,62 @@
/*
* This file is a part of AllPlaceFinder library
* and is distributed under the 2-Clause BSD licence.
* Author: Tomasz Sowa <t.sowa@ttmath.org>
*/
/*
* Copyright (c) 2021, Tomasz Sowa
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
*/
#ifndef headerfile_allplacefinder_lib_misc
#define headerfile_allplacefinder_lib_misc
#include <fstream>
#include <string>
#include <vector>
#include "textstream/textstream.h"
namespace allplacefinder
{
bool file_exists(const std::wstring & file_path);
bool dir_exists(const std::wstring & file_path);
bool open_file(const std::wstring & file_path, std::ifstream & str, const char * err_msg);
void create_id_str(long id, std::vector<std::wstring> & path);
bool create_dir(const std::wstring & path);
bool create_dir_if_not_exists(const std::wstring & path);
void print(const std::wstring & str);
void save_stream(const pt::TextStream & in, std::ofstream & out);
} // namespace
#endif

63
lib/params.cpp Normal file
View File

@ -0,0 +1,63 @@
/*
* This file is a part of AllPlaceFinder library
* and is distributed under the 2-Clause BSD licence.
* Author: Tomasz Sowa <t.sowa@ttmath.org>
*/
/*
* Copyright (c) 2021, Tomasz Sowa
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
*/
#include "params.h"
namespace allplacefinder
{
Params::Params()
{
clear();
}
void Params::clear()
{
create_lookup_dir = false;
osm_file.clear();
lookup_dir.clear();
}
} // namespace

View File

@ -35,6 +35,8 @@
#ifndef headerfile_allplacefinder_lib_params #ifndef headerfile_allplacefinder_lib_params
#define headerfile_allplacefinder_lib_params #define headerfile_allplacefinder_lib_params
#include <string>
namespace allplacefinder namespace allplacefinder
{ {
@ -43,7 +45,14 @@ class Params
{ {
public: public:
Params();
void clear();
bool create_lookup_dir;
std::wstring osm_file;
std::wstring lookup_dir;

View File

@ -24,7 +24,7 @@ progname = placefinder
all: $(progname) all: $(progname)
$(progname): $(objfiles) $(progname): $(objfiles) ../lib/allplacefinder.a ../../pikotools/src/pikotools.a
$(CXX) -o $(progname) $(CXXFLAGS) $(LDFLAGS) $(objfiles) ../lib/allplacefinder.a ../../pikotools/src/pikotools.a $(CXX) -o $(progname) $(CXXFLAGS) $(LDFLAGS) $(objfiles) ../lib/allplacefinder.a ../../pikotools/src/pikotools.a

View File

@ -42,15 +42,62 @@ void print_syntax()
{ {
std::cout << "placefinder options:" << std::endl; std::cout << "placefinder options:" << std::endl;
std::cout << " -h print this help" << std::endl; std::cout << " -h print this help" << std::endl;
std::cout << " --osm-file file_name - a path to osm file name" << std::endl;
std::cout << " --lookup-dir dir_name - a path to lookup dir" << std::endl;
std::cout << " --create-lookup-dir - create a lookup dir, you have to provide a path in --lookup-dir parameter" << std::endl;
}
bool parse_parametr(const pt::Space * input_par, std::wstring & param, const char * error_msg)
{
if( input_par )
{
if( input_par->table_size() != 1 )
{
std::cout << error_msg << std::endl;
return false;
param = input_par->value.value_table.back()->to_wstr();
}
}
return true;
} }
bool parse_parameters(const pt::Space & parameters, allplacefinder::Params & params) bool parse_parameters(const pt::Space & parameters, allplacefinder::Params & params)
{ {
const pt::Space * osm_file = parameters.get_space(L"osm-file");
const pt::Space * lookup_dir = parameters.get_space(L"lookup-dir");
const pt::Space * create_lookup_dir = parameters.get_space(L"create-lookup-dir");
bool ok = true;
return true; if( osm_file )
{
ok = ok && parse_parametr(osm_file, params.osm_file, "--osm-file parametr can be used only once");
}
if( lookup_dir )
{
ok = ok && parse_parametr(lookup_dir, params.lookup_dir, "--lookup-dir parametr can be used only once");
}
params.create_lookup_dir = (create_lookup_dir != nullptr);
return ok;
}
int make_action(allplacefinder::Params & params)
{
int status = 3;
if( params.create_lookup_dir )
{
allplacefinder::Finder finder;
status = finder.create_lookup_dir(params) ? 0 : 4;
}
return status;
} }
@ -61,8 +108,8 @@ pt::Space options, arguments;
pt::MainOptionsParser options_parser; pt::MainOptionsParser options_parser;
allplacefinder::Params params; allplacefinder::Params params;
arguments.add(L"osmfile", 1); arguments.add(L"osm-file", 1);
arguments.add(L"lookupdir", 1); arguments.add(L"lookup-dir", 1);
pt::MainOptionsParser::Status status = options_parser.parse(argc, argv, options, arguments); pt::MainOptionsParser::Status status = options_parser.parse(argc, argv, options, arguments);
@ -80,16 +127,22 @@ allplacefinder::Params params;
if( parse_parameters(options, params) ) if( parse_parameters(options, params) )
{ {
// ------------------
// for debug purposes
//params.create_lookup_dir = true;
//params.osm_file = L"/home/tomek/data/test.xml";
//params.osm_file = L"/home/tomek/data/wroclaw_map.xml";
//allplacefinder::Finder finder; //params.lookup_dir = L"/home/tomek/data/lookup-dir";
//params.lookup_dir = L"/data/lookup-dir";
// ------------------
return make_action(params);
} }
//std::cout << options.serialize_to_json_str(true) << std::endl; //std::cout << options.serialize_to_json_str(true) << std::endl;
return 0; return 2;
} }