diff --git a/.gitignore b/.gitignore index 8b4257c..c03642d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ .project .settings data +*.o +*.a +placefinder/placefinder diff --git a/Makefile b/Makefile index c063d68..675a98d 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,10 @@ CXX = g++10 +#CXX = clang++ + CXXFLAGS = -Wl,-rpath=/usr/local/lib/gcc10 -Wfatal-errors -fPIC -Wall -pedantic -O0 -g3 -pthread -std=c++20 -I../../pikotools/src -I../lib -I/usr/local/include +#CXXFLAGS = -fsanitize=address -Wfatal-errors -fPIC -Wall -pedantic -O0 -g3 -pthread -std=c++20 -I../../pikotools/src -I../lib -I/usr/local/include LDFLAGS = -L/usr/local/lib #-fsanitize=address @@ -12,14 +15,14 @@ export CXXFLAGS export LDFLAGS export AR -all: lib placefinder pikotools +all: lib placefinder lib: FORCE pikotools $(MAKE) -C lib -placefinder: FORCE pikotools +placefinder: FORCE pikotools lib $(MAKE) -C placefinder @@ -30,6 +33,7 @@ pikotools: FORCE clean: FORCE $(MAKE) -C lib clean $(MAKE) -C placefinder clean + $(MAKE) -C ../pikotools clean depend: FORCE diff --git a/lib/Makefile.dep b/lib/Makefile.dep index ad87197..cb6bfe3 100644 --- a/lib/Makefile.dep +++ b/lib/Makefile.dep @@ -1,3 +1,5 @@ # DO NOT DELETE -./finder.o: finder.h +./finder.o: finder.h params.h misc.h +./params.o: params.h +./misc.o: misc.h diff --git a/lib/finder.cpp b/lib/finder.cpp index 95ae866..b51ccf0 100644 --- a/lib/finder.cpp +++ b/lib/finder.cpp @@ -32,7 +32,13 @@ * */ +#include +#include #include "finder.h" +#include "utf8/utf8.h" +#include "misc.h" +#include "convert/convert.h" +#include "space/spaceparser.h" namespace allplacefinder @@ -41,12 +47,370 @@ namespace allplacefinder Finder::Finder() { - + params = nullptr; } +bool Finder::create_lookup_dir(Params & params) +{ + if( params.osm_file.empty() ) + { + std::cerr << "you have to provide a path to osm file in --osm-file parameter" << std::endl; + return false; + } + + if( params.lookup_dir.empty() ) + { + std::cerr << "you have to provide a path to lookup dir in --lookup-dir parameter" << std::endl; + return false; + } + + if( !open_file(params.osm_file, osm_file_str, "I cannot open the osm file for reading") ) + return false; + + if( !dir_exists(params.lookup_dir ) ) + { + std::cerr << "There is no the lookup directory" << std::endl; + return false; + } + + this->params = ¶ms; + return create_lookup_dir(); +} + + +bool Finder::create_lookup_dir() +{ + pt::HTMLParser parser; + pt::Space space; + std::string file_path; + pt::wide_to_utf8(params->osm_file, file_path); + nodes = 0; + ways = 0; + + parser.set_item_parsed_listener(this); + pt::HTMLParser::Status status = parser.parse_xml_file(file_path, space, false); + + switch(status) + { + case pt::HTMLParser::cant_open_file: + std::cerr << "I cannot open the input file: " << file_path << std::endl; + break; + + case pt::HTMLParser::syntax_error: + std::cerr << "Syntax error: " << file_path << ":" << parser.get_last_parsed_line() << std::endl; + break; + + case pt::HTMLParser::ok: + //std::cout << space.serialize_to_json_str(true) << std::endl; + break; + } + + //std::cout << "nodes: " << nodes << std::endl; + //std::cout << "ways: " << ways << std::endl; + + return true; +} + + +bool Finder::item_parsed(const pt::HTMLParser::Item & item) +{ + std::string name_utf8; + pt::wide_to_utf8(item.name, name_utf8); + //std::cout << name_utf8 << std::endl; + + if( item.name == L"node" ) + { + nodes += 1; + parse_node(*item.space); + return false; + } + + if( item.name == L"way" ) + { + ways += 1; + parse_way(*item.space); + return false; + } + + return true; +} + + +void Finder::parse_node(pt::Space & node) +{ + pt::Space * attr = node.get_space(L"attr"); + pt::Space out; + + if( attr ) + { + pt::Space * id = attr->get_space(L"id"); + pt::Space * lat = attr->get_space(L"lat"); + pt::Space * lon = attr->get_space(L"lon"); + + if( id && lat && lon ) + { + long id_long = id->to_long(); + + if( id_long >= 0 ) + { + out.add(L"id", id_long); + out.add(L"lat", lat->to_wstr()); + out.add(L"lon", lon->to_wstr()); + + if( !save_file(L"node", id_long, out) ) + { + std::cout << "I cannot save node file with id: " << id_long << std::endl; + } + + //pt::Space * childs = node.get_space(L"childs"); + } + else + { + std::cout << "Skipping node with negative id" << std::endl; + } + } + else + { + std::cout << "Skipping node" << std::endl; + } + } + + //std::cout << out.serialize_to_json_str(true) << std::endl; +} + + + +void Finder::parse_way(pt::Space & way) +{ + pt::Space * attr = way.get_space(L"attr"); + pt::Space out; + + + if( attr ) + { + pt::Space * id = attr->get_space(L"id"); + + if( id ) + { + long id_long = id->to_long(); + + if( id_long >= 0 ) + { + out.add(L"id", id_long); + + pt::Space * childs = way.get_space(L"childs"); + + if( childs ) + { + if( parse_way_childs(out, *childs) ) + { + + + std::string str; + str = out.serialize_to_json_str(false); + std::cout << str << std::endl; + +// if( !save_file(L"way", id_long, out) ) +// { +// std::cout << "I cannot save way file with id: " << id_long << std::endl; +// } + + } + } + } + } + } + + //std::cout << out.serialize_to_json_str(true) << std::endl; +} + + +bool Finder::parse_way_childs(pt::Space & out, pt::Space & childs) +{ + size_t nd_len = 0; + bool has_name = false; + bool has_node = false; + + if( childs.is_table() ) + { + for(pt::Space * child : childs.value.value_table) + { + std::wstring * name = child->get_wstr(L"name"); + + if( name ) + { + if( *name == L"nd" ) + { + nd_len += 1; + } + + if( *name == L"tag" ) + { + pt::Space * attr = child->get_space(L"attr"); + + if( attr ) + { + std::wstring * k = attr->get_wstr(L"k"); + std::wstring * v = attr->get_wstr(L"v"); + + if( k && v && *k == L"name" ) + { + has_name = true; + out.add(L"name", *v); + } + } + } + } + } + + if( nd_len > 0 && has_name ) + { + size_t nd_mid = (nd_len / 2); + size_t index = 0; + + for(pt::Space * child : childs.value.value_table) + { + std::wstring * name = child->get_wstr(L"name"); + + if( name && *name == L"nd" ) + { + if( index == nd_mid ) + { + pt::Space * attr = child->get_space(L"attr"); + + if( attr ) + { + std::wstring * ref = attr->get_wstr(L"ref"); + + if( ref ) + { + out.add(L"node_id", *ref); + long node_id = pt::Tol(ref->c_str()); + has_node = get_node(node_id, out); + } + } + + break; + } + + index += 1; + } + } + } + } + + return nd_len > 0 && has_name && has_node; +} + + +bool Finder::get_node(long node_id, pt::Space & out) +{ + std::vector subdirs; + create_id_str(node_id, subdirs); + + std::wstring path; + path = params->lookup_dir; + path += L"/"; + path += L"node"; + + for(std::wstring & str : subdirs) + { + path += L"/"; + path += str; + } + + pt::SpaceParser parser; + pt::Space node; + + pt::SpaceParser::Status status = parser.parse_json_file(path, node); + + if( status == pt::SpaceParser::ok ) + { + std::wstring * lat = node.get_wstr(L"lat"); + std::wstring * lon = node.get_wstr(L"lon"); + + if( lat && lon ) + { + out.add(L"lat", *lat); + out.add(L"lon", *lon); + + return true; + } + } + + return false; +} + + + +bool Finder::save_file(const wchar_t * subdir, long id, pt::Space & space) +{ + std::vector subdirs; + //std::string p; + + create_id_str(id, subdirs); + +// for(size_t i = 0 ; i < path.size() ; ++i) +// { +// pt::wide_to_utf8(path[i], p); +// std::cout << p << " / "; +// } +// +// std::cout << std::endl; + + std::wstring path; + path = params->lookup_dir; + path += L"/"; + path += subdir; + + if( !create_dir_if_not_exists(path) ) + return false; + + for(size_t i = 0 ; i + 1 < subdirs.size() ; ++i) + { + path += L"/"; + path += subdirs[i]; + + if( !create_dir_if_not_exists(path) ) + return false; + } + + if( !subdirs.empty() ) + { + path += L"/"; + path += subdirs.back(); + + std::string path_utf8; + pt::wide_to_utf8(path, path_utf8); + + std::ofstream str(path_utf8, std::ios_base::out | std::ios_base::binary); + + if( str ) + { + text_stream.clear(); + space.serialize_to_json_stream(text_stream, true); + text_stream << '\n'; + save_stream(text_stream, str); + } + else + { + return false; + } + + str.close(); + + if( !str ) + return false; + + } + + return true; +} + + diff --git a/lib/finder.h b/lib/finder.h index 8897143..454905b 100644 --- a/lib/finder.h +++ b/lib/finder.h @@ -35,19 +35,41 @@ #ifndef headerfile_allplacefinder_lib_finder #define headerfile_allplacefinder_lib_finder +#include +#include "html/htmlparser.h" +#include "params.h" + namespace allplacefinder { -class Finder +class Finder : public pt::HTMLParser::ItemParsedListener { public: Finder(); + bool create_lookup_dir(Params & params); +private: + + std::ifstream osm_file_str; + std::ifstream lookup_dir_str; + Params * params; + pt::TextStream text_stream; + size_t nodes; + size_t ways; + + bool create_lookup_dir(); + bool item_parsed(const pt::HTMLParser::Item & item) override; + void parse_node(pt::Space & node); + void parse_way(pt::Space & way); + bool parse_way_childs(pt::Space & out, pt::Space & childs); + bool save_file(const wchar_t * subdir, long id, pt::Space & space); + bool get_node(long node_id, pt::Space & out); + }; } // namespace diff --git a/lib/misc.cpp b/lib/misc.cpp new file mode 100644 index 0000000..95f2023 --- /dev/null +++ b/lib/misc.cpp @@ -0,0 +1,187 @@ +/* + * This file is a part of AllPlaceFinder library + * and is distributed under the 2-Clause BSD licence. + * Author: Tomasz Sowa + */ + +/* + * Copyright (c) 2021, Tomasz Sowa + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +#include +#include "utf8/utf8.h" +#include "misc.h" +#include "convert/convert.h" + + +namespace allplacefinder +{ + +bool file_exists(const std::wstring & file_path) +{ + struct stat sb; + std::string file_path_utf8; + pt::wide_to_utf8(file_path, file_path_utf8); + + if( stat(file_path_utf8.c_str(), &sb) != 0 ) + return false; + + return (sb.st_mode & S_IFREG) != 0; +} + + +bool dir_exists(const std::wstring & file_path) +{ + struct stat sb; + std::string file_path_utf8; + pt::wide_to_utf8(file_path, file_path_utf8); + + if( stat(file_path_utf8.c_str(), &sb) != 0 ) + return false; + + return (sb.st_mode & S_IFDIR) != 0; +} + + + +bool open_file(const std::wstring & file_path, std::ifstream & str, const char * err_msg) +{ + std::string file_path_utf8; + pt::wide_to_utf8(file_path, file_path_utf8); + + str.clear(); + str.open(file_path_utf8.c_str(), std::ios_base::in | std::ios_base::binary); + + if( !str ) + { + std::cerr << err_msg << std::endl; + return false; + } + + return true; +} + + + +void create_id_str(long id, std::vector & path) +{ + wchar_t buffer[64]; + size_t len = sizeof(buffer) / sizeof(wchar_t); + size_t str_len = 0; + std::wstring d; + + path.clear(); + pt::Toa(id, buffer, len, 16, &str_len); + + if( str_len == 0 ) + return; + + size_t padding = 0; + + if( str_len < 16 ) + padding = 16 - str_len; + + d = L"00"; + + padding = padding & (~1); + + for(size_t i=0 ; i < padding ; i += 2) + { + path.push_back(d); + } + + d.clear(); + size_t i = 0; + + if( (str_len & 1) != 0 ) + { + d = L"0"; + d += buffer[0]; + i = 1; + path.push_back(d); + } + + for( ; i + 1 < str_len ; i += 2) + { + d = buffer[i]; + d += buffer[i+1]; + path.push_back(d); + } + +} + + +bool create_dir(const std::wstring & path) +{ + std::string path_utf8; + pt::wide_to_utf8(path, path_utf8); + + return mkdir(path_utf8.c_str(), 0755) == 0; +} + + +bool create_dir_if_not_exists(const std::wstring & path) +{ + if( !dir_exists(path) ) + { + if( !create_dir(path) ) + { + std::cout << "I cannot create directory: "; + print(path); + std::cout << std::endl; + return false; + } + } + + return true; +} + + +void print(const std::wstring & str) +{ + std::string str_utf8; + pt::wide_to_utf8(str, str_utf8); + std::cout << str_utf8; +} + + +void save_stream(const pt::TextStream & in, std::ofstream & out) +{ + pt::TextStream::const_iterator i = in.begin(); + + for( ; i != in.end() ; ++i) + { + out << *i; + } +} + + +} // namespace + + + diff --git a/lib/misc.h b/lib/misc.h new file mode 100644 index 0000000..4d4ddf5 --- /dev/null +++ b/lib/misc.h @@ -0,0 +1,62 @@ +/* + * This file is a part of AllPlaceFinder library + * and is distributed under the 2-Clause BSD licence. + * Author: Tomasz Sowa + */ + +/* + * Copyright (c) 2021, Tomasz Sowa + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef headerfile_allplacefinder_lib_misc +#define headerfile_allplacefinder_lib_misc + +#include +#include +#include +#include "textstream/textstream.h" + +namespace allplacefinder +{ + + bool file_exists(const std::wstring & file_path); + bool dir_exists(const std::wstring & file_path); + bool open_file(const std::wstring & file_path, std::ifstream & str, const char * err_msg); + + void create_id_str(long id, std::vector & path); + + bool create_dir(const std::wstring & path); + bool create_dir_if_not_exists(const std::wstring & path); + + void print(const std::wstring & str); + + void save_stream(const pt::TextStream & in, std::ofstream & out); + +} // namespace + +#endif + diff --git a/lib/params.cpp b/lib/params.cpp new file mode 100644 index 0000000..4cd0624 --- /dev/null +++ b/lib/params.cpp @@ -0,0 +1,63 @@ +/* + * This file is a part of AllPlaceFinder library + * and is distributed under the 2-Clause BSD licence. + * Author: Tomasz Sowa + */ + +/* + * Copyright (c) 2021, Tomasz Sowa + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "params.h" + + +namespace allplacefinder +{ + + +Params::Params() +{ + clear(); +} + + + +void Params::clear() +{ + create_lookup_dir = false; + + osm_file.clear(); + lookup_dir.clear(); +} + + + + + +} // namespace + + diff --git a/lib/params.h b/lib/params.h index 35d2753..fb74fe8 100644 --- a/lib/params.h +++ b/lib/params.h @@ -35,6 +35,8 @@ #ifndef headerfile_allplacefinder_lib_params #define headerfile_allplacefinder_lib_params +#include + namespace allplacefinder { @@ -43,7 +45,14 @@ class Params { public: + Params(); + void clear(); + + bool create_lookup_dir; + + std::wstring osm_file; + std::wstring lookup_dir; diff --git a/placefinder/Makefile b/placefinder/Makefile index efbedd4..a07516b 100644 --- a/placefinder/Makefile +++ b/placefinder/Makefile @@ -24,7 +24,7 @@ progname = placefinder all: $(progname) -$(progname): $(objfiles) +$(progname): $(objfiles) ../lib/allplacefinder.a ../../pikotools/src/pikotools.a $(CXX) -o $(progname) $(CXXFLAGS) $(LDFLAGS) $(objfiles) ../lib/allplacefinder.a ../../pikotools/src/pikotools.a diff --git a/placefinder/main.cpp b/placefinder/main.cpp index f01699d..a3f75b5 100644 --- a/placefinder/main.cpp +++ b/placefinder/main.cpp @@ -42,15 +42,62 @@ void print_syntax() { std::cout << "placefinder options:" << std::endl; std::cout << " -h print this help" << std::endl; + std::cout << " --osm-file file_name - a path to osm file name" << std::endl; + std::cout << " --lookup-dir dir_name - a path to lookup dir" << std::endl; + std::cout << " --create-lookup-dir - create a lookup dir, you have to provide a path in --lookup-dir parameter" << std::endl; +} + +bool parse_parametr(const pt::Space * input_par, std::wstring & param, const char * error_msg) +{ + if( input_par ) + { + if( input_par->table_size() != 1 ) + { + std::cout << error_msg << std::endl; + return false; + param = input_par->value.value_table.back()->to_wstr(); + } + } + + return true; } bool parse_parameters(const pt::Space & parameters, allplacefinder::Params & params) { + const pt::Space * osm_file = parameters.get_space(L"osm-file"); + const pt::Space * lookup_dir = parameters.get_space(L"lookup-dir"); + const pt::Space * create_lookup_dir = parameters.get_space(L"create-lookup-dir"); + bool ok = true; - return true; + if( osm_file ) + { + ok = ok && parse_parametr(osm_file, params.osm_file, "--osm-file parametr can be used only once"); + } + + if( lookup_dir ) + { + ok = ok && parse_parametr(lookup_dir, params.lookup_dir, "--lookup-dir parametr can be used only once"); + } + + params.create_lookup_dir = (create_lookup_dir != nullptr); + return ok; +} + + +int make_action(allplacefinder::Params & params) +{ + int status = 3; + + if( params.create_lookup_dir ) + { + allplacefinder::Finder finder; + status = finder.create_lookup_dir(params) ? 0 : 4; + } + + return status; } @@ -61,8 +108,8 @@ pt::Space options, arguments; pt::MainOptionsParser options_parser; allplacefinder::Params params; - arguments.add(L"osmfile", 1); - arguments.add(L"lookupdir", 1); + arguments.add(L"osm-file", 1); + arguments.add(L"lookup-dir", 1); pt::MainOptionsParser::Status status = options_parser.parse(argc, argv, options, arguments); @@ -80,16 +127,22 @@ allplacefinder::Params params; if( parse_parameters(options, params) ) { + // ------------------ + // for debug purposes + //params.create_lookup_dir = true; + //params.osm_file = L"/home/tomek/data/test.xml"; + //params.osm_file = L"/home/tomek/data/wroclaw_map.xml"; - //allplacefinder::Finder finder; + //params.lookup_dir = L"/home/tomek/data/lookup-dir"; + //params.lookup_dir = L"/data/lookup-dir"; + // ------------------ + return make_action(params); } - - //std::cout << options.serialize_to_json_str(true) << std::endl; -return 0; +return 2; }