From 620bb0cfb2a98a6d458b5774fc93ea0530c6d9da Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Mon, 4 Mar 2024 09:46:38 -0300 Subject: [PATCH 1/4] + Testing version for relations raw data support --- docs/Dev/debugging.md | 4 +- setup/db/underpass.sql | 5 - src/bootstrap/bootstrap.cc | 2 +- src/osm/osmchange.hh | 7 +- src/raw/queryraw.cc | 208 +++++++++++++++++++-- src/raw/queryraw.hh | 7 +- src/replicator/planetreplicator.cc | 3 + src/replicator/replication.cc | 1 - src/replicator/threads.cc | 51 +++-- src/testsuite/libunderpass.all/raw-test.cc | 40 ++++ src/underpass.cc | 11 +- src/underpassconfig.hh | 1 + src/utils/geoutil.cc | 12 +- src/utils/geoutil.hh | 10 +- 14 files changed, 303 insertions(+), 59 deletions(-) create mode 100644 src/testsuite/libunderpass.all/raw-test.cc diff --git a/docs/Dev/debugging.md b/docs/Dev/debugging.md index b109b5bd..d303546f 100644 --- a/docs/Dev/debugging.md +++ b/docs/Dev/debugging.md @@ -28,7 +28,7 @@ It's also possible to debug Underpass on MacOS. You should use `glibtool` instea `brew install gdb` -Note that gdb requires special privileges to access Mach ports. +Note that gdb requires special privileges to access Mac ports. You will need to codesign the binary. For instructions, see: https://sourceware.org/gdb/wiki/PermissionsDarwin Add the alias in your `~/.bashrc` file: @@ -38,7 +38,7 @@ Add the alias in your `~/.bashrc` file: ### With LLDB `lldb -- src/testsuite/libunderpass.all/yaml-test` -`lldb -- .libs/underpass --bootstrap ` +`lldb -- .libs/underpass --bootstrap` diff --git a/setup/db/underpass.sql b/setup/db/underpass.sql index 0542e21d..0641010f 100644 --- a/setup/db/underpass.sql +++ b/setup/db/underpass.sql @@ -111,11 +111,6 @@ CREATE TABLE IF NOT EXISTS public.way_refs ( node_id int8 ); -CREATE TABLE IF NOT EXISTS public.rel_refs ( - rel_id int8, - way_id int8 -); - CREATE UNIQUE INDEX nodes_id_idx ON public.nodes (osm_id DESC); CREATE UNIQUE INDEX ways_poly_id_idx ON public.ways_poly (osm_id DESC); CREATE UNIQUE INDEX ways_line_id_idx ON public.ways_line(osm_id DESC); diff --git a/src/bootstrap/bootstrap.cc b/src/bootstrap/bootstrap.cc index 2f81f22d..0004f58b 100644 --- a/src/bootstrap/bootstrap.cc +++ b/src/bootstrap/bootstrap.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // diff --git a/src/osm/osmchange.hh b/src/osm/osmchange.hh index d04defc8..18f186df 100644 --- a/src/osm/osmchange.hh +++ b/src/osm/osmchange.hh @@ -258,8 +258,11 @@ class OsmChangeFile std::map> userstats; ///< User statistics for this file - std::list> changes; ///< All the changes in this file - std::map nodecache; ///< Cache nodes across multiple changesets + std::list> changes; ///< All the changes in this file + + std::map nodecache; ///< Cache nodes across multiple changesets + + std::map> waycache; ///< Cache ways across multiple changesets /// Collect statistics for each user std::shared_ptr>> diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc index 9decd2ec..cd8db874 100644 --- a/src/raw/queryraw.cc +++ b/src/raw/queryraw.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // @@ -234,8 +234,70 @@ QueryRaw::applyChange(const OsmWay &way) const std::string QueryRaw::applyChange(const OsmRelation &relation) const { + std::string query = ""; + std::stringstream ss; + ss << std::setprecision(12) << boost::geometry::wkt(relation.multipolygon); + // TODO: support for both multipolygon & multilinestring + // ss << std::setprecision(12) << boost::geometry::wkt(relation.multilinestring); + std::string geostring = ss.str(); + + if (relation.action == osmobjects::create || relation.action == osmobjects::modify) { -} + query = "INSERT INTO relations as r (osm_id, tags, refs, geom, timestamp, version, \"user\", uid, changeset) VALUES("; + std::string format = "%d, %s, %s, %s, \'%s\', %d, \'%s\', %d, %d) \ + ON CONFLICT (osm_id) DO UPDATE SET tags = %s, refs = %s, geom = %s, timestamp = \'%s\', version = %d, \"user\" = \'%s\', uid = %d, changeset = %d WHERE r.version <= %d;"; + boost::format fmt(format); + + // osm_id + fmt % relation.id; + + //tags + auto tags = buildTagsQuery(relation.tags); + fmt % tags; + + // refs + std::string refs = ""; + for (auto it = std::begin(relation.members); it != std::end(relation.members); ++it) { + refs += std::to_string(it->ref) + ","; + } + refs.erase(refs.size() - 1); + refs = "ARRAY[" + refs + "]"; + fmt % refs; + + // geometry + std::string geometry; + geometry = "ST_GeomFromText(\'" + geostring + "\', 4326)"; + fmt % geometry; + + // timestamp + std::string timestamp = to_simple_string(boost::posix_time::microsec_clock::universal_time()); + fmt % timestamp; + // version + fmt % relation.version; + // user + fmt % dbconn->escapedString(relation.user); + // uid + fmt % relation.uid; + // changeset + fmt % relation.changeset; + + // ON CONFLICT + fmt % tags; + fmt % refs; + fmt % geometry; + fmt % timestamp; + fmt % relation.version; + fmt % dbconn->escapedString(relation.user); + fmt % relation.uid; + fmt % relation.changeset; + fmt % relation.version; + + query += fmt.str(); + } else if (relation.action == osmobjects::remove) { + query += "DELETE FROM relations where osm_id = " + std::to_string(relation.id) + ";"; + } + + return query;} std::vector arrayStrToVector(std::string &refs_str) { refs_str.erase(0, 1); @@ -249,10 +311,30 @@ std::vector arrayStrToVector(std::string &refs_str) { return refs; } -void QueryRaw::getNodeCache(std::shared_ptr osmchanges, const multipolygon_t &poly) +void +QueryRaw::getWaysByIds(std::string &relsForWayCacheIds, std::map> waycache) { +#ifdef TIMING_DEBUG + boost::timer::auto_cpu_timer timer("getWaysByIds(relsForWayCacheIds, waycache): took %w seconds\n"); +#endif + // Get all ways that have references to nodes + relsForWayCacheIds.erase(relsForWayCacheIds.size() - 1); + std::string waysQuery = "SELECT distinct(osm_id), ST_AsText(geom, 4326) from ways_poly wp where osm_id = any(ARRAY[" + relsForWayCacheIds + "])"; + auto ways_result = dbconn->query(waysQuery); + + // Fill vector of OsmWay objects + for (auto way_it = ways_result.begin(); way_it != ways_result.end(); ++way_it) { + auto way = std::make_shared(); + way->id = (*way_it)[0].as(); + boost::geometry::read_wkt((*way_it)[1].as(), way->polygon); + waycache.insert(std::pair(way->id, std::make_shared(*way))); + } +} + +// TODO: divide this function into multiple ones +void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const multipolygon_t &poly) { #ifdef TIMING_DEBUG - boost::timer::auto_cpu_timer timer("getNodeCache(osmchanges): took %w seconds\n"); + boost::timer::auto_cpu_timer timer("buildGeometries(osmchanges, poly): took %w seconds\n"); #endif std::string referencedNodeIds; std::string modifiedNodesIds; @@ -263,21 +345,30 @@ void QueryRaw::getNodeCache(std::shared_ptr osmchanges, const mul for (auto wit = std::begin(change->ways); wit != std::end(change->ways); ++wit) { OsmWay *way = wit->get(); if (way->action != osmobjects::remove) { - // Save referenced nodes for later use + // Save referenced nodes ids for later use for (auto rit = std::begin(way->refs); rit != std::end(way->refs); ++rit) { if (!osmchanges->nodecache.count(*rit)) { referencedNodeIds += std::to_string(*rit) + ","; } } + // Save ways for later use + if (way->linestring.size() == way->refs.size()) { + // Save only ways with a geometry that are inside the priority area + // these are mostly created ways + if (boost::geometry::within(way->linestring, poly)) { + osmchanges->waycache.insert(std::make_pair(way->id, std::make_shared(*way))); + } + } } else { removedWays.push_back(way->id); } } + // Save modified nodes for later use for (auto nit = std::begin(change->nodes); nit != std::end(change->nodes); ++nit) { OsmNode *node = nit->get(); if (node->action == osmobjects::modify) { - // Get only modified nodes inside priority area + // Get only modified nodes ids inside the priority area if (boost::geometry::within(node->point, poly)) { modifiedNodesIds += std::to_string(node->id) + ","; } @@ -298,6 +389,7 @@ void QueryRaw::getNodeCache(std::shared_ptr osmchanges, const mul referencedNodeIds += std::to_string(*rit) + ","; } } + // If the way is not marked as removed, mark it as modified if (std::find(removedWays.begin(), removedWays.end(), way->id) == removedWays.end()) { way->action = osmobjects::modify; change->ways.push_back(way); @@ -327,21 +419,107 @@ void QueryRaw::getNodeCache(std::shared_ptr osmchanges, const mul OsmChange *change = it->get(); for (auto wit = std::begin(change->ways); wit != std::end(change->ways); ++wit) { OsmWay *way = wit->get(); - way->linestring.clear(); - for (auto rit = way->refs.begin(); rit != way->refs.end(); ++rit) { - if (osmchanges->nodecache.count(*rit)) { - boost::geometry::append(way->linestring, osmchanges->nodecache.at(*rit)); + if (way->linestring.size() != way->refs.size()) { + way->linestring.clear(); + for (auto rit = way->refs.begin(); rit != way->refs.end(); ++rit) { + if (osmchanges->nodecache.count(*rit)) { + boost::geometry::append(way->linestring, osmchanges->nodecache.at(*rit)); + } + } + if (way->isClosed()) { + way->polygon = { {std::begin(way->linestring), std::end(way->linestring)} }; } } - if (way->isClosed()) { - way->polygon = { {std::begin(way->linestring), std::end(way->linestring)} }; + // Save way pointer for later use + if (boost::geometry::within(way->linestring, poly)) { + osmchanges->waycache.insert(std::make_pair(way->id, std::make_shared(*way))); } } } - // Build relation multipolyon geometries - // TODO - + // Filter out all relations that doesn't have at least 1 way in cache + // and are not type=multipolygon + std::string relsForWayCacheIds; + for (auto it = std::begin(osmchanges->changes); it != std::end(osmchanges->changes); it++) { + OsmChange *change = it->get(); + for (auto rel_it = std::begin(change->relations); rel_it != std::end(change->relations); ++rel_it) { + OsmRelation *relation = rel_it->get(); + if (relation->tags.count("type") && relation->tags.at("type") == "multipolygon") { + bool getWaysForRelation = false; + for (auto mit = relation->members.begin(); mit != relation->members.end(); ++mit) { + if (osmchanges->waycache.count(mit->ref)) { + getWaysForRelation = true; + break; + } + } + if (getWaysForRelation) { + relation->priority = true; + for (auto mit = relation->members.begin(); mit != relation->members.end(); ++mit) { + if (!osmchanges->waycache.count(mit->ref)) { + relsForWayCacheIds += std::to_string(mit->ref) + ","; + } + } + } else { + relation->priority = false; + } + } + } + } + // Get all missing ways geometries for relations + if (relsForWayCacheIds != "") { + getWaysByIds(relsForWayCacheIds, osmchanges->waycache); + } + + // Build relation MultiPolyon geometries + for (auto it = std::begin(osmchanges->changes); it != std::end(osmchanges->changes); it++) { + OsmChange *change = it->get(); + for (auto rel_it = std::begin(change->relations); rel_it != std::end(change->relations); ++rel_it) { + OsmRelation *relation = rel_it->get(); + if (relation->priority) { + bool first = true; + std::string multipolygon_str = "MULTIPOLYGON(("; + std::string innerGeoStr; + bool noWay = false; + for (auto mit = relation->members.begin(); mit != relation->members.end(); ++mit) { + if (!osmchanges->waycache.count(mit->ref)) { + noWay = true; + break; + } + auto way = osmchanges->waycache.at(mit->ref); + if (boost::geometry::num_points(way->linestring) == 0 || + boost::geometry::num_points(way->polygon) == 0 + ) { + noWay = true; + break; + } + std::stringstream ss; + ss << std::setprecision(12) << boost::geometry::wkt(way->polygon); + std::string geometry = ss.str(); + geometry.erase(0,8); + geometry.erase(geometry.size() - 1); + if (first && mit->role == "outer") { + multipolygon_str += geometry + ","; + } else { + if (mit->role == "inner") { + innerGeoStr += geometry + ","; + } else { + multipolygon_str += geometry + ","; + multipolygon_str += innerGeoStr; + innerGeoStr = ""; + } + } + } + if (innerGeoStr.size() > 0) { + multipolygon_str += innerGeoStr; + } + multipolygon_str.erase(multipolygon_str.size() - 1); + multipolygon_str += "))"; + if (!noWay) { + boost::geometry::read_wkt(multipolygon_str, relation->multipolygon); + } + } + } + } } diff --git a/src/raw/queryraw.hh b/src/raw/queryraw.hh index 8cabf4cf..57a52fba 100644 --- a/src/raw/queryraw.hh +++ b/src/raw/queryraw.hh @@ -66,13 +66,14 @@ class QueryRaw { std::string applyChange(const OsmWay &way) const; /// Build query for processed Relation std::string applyChange(const OsmRelation &relation) const; - /// Get nodes for filling Node cache - void getNodeCache(std::shared_ptr osmchanges, const multipolygon_t &poly); + /// Build all geometries for osmchanges + void buildGeometries(std::shared_ptr osmchanges, const multipolygon_t &poly); /// Get nodes for filling Node cache from ways refs void getNodeCacheFromWays(std::shared_ptr> ways, std::map &nodecache) const; // Get ways by refs std::list> getWaysByNodesRefs(std::string &nodeIds) const; - + // Get ways by ids (used for getting relations geometries) + void getWaysByIds(std::string &relsForWayCacheIds, std::map> waycache); // DB connection std::shared_ptr dbconn; // Get ways count diff --git a/src/replicator/planetreplicator.cc b/src/replicator/planetreplicator.cc index 15ac3fe6..c8dc306b 100644 --- a/src/replicator/planetreplicator.cc +++ b/src/replicator/planetreplicator.cc @@ -182,6 +182,9 @@ std::shared_ptr PlanetReplicator::findRemotePath(const underpassconfi fullurl = "https://" + config.planet_server + "/" + cached; remote->parse(fullurl); remote->updatePath(major, minor, index); + if (!config.silent) { + remote->dump(); + } return remote; }; diff --git a/src/replicator/replication.cc b/src/replicator/replication.cc index 4655fcf4..2caaffd1 100644 --- a/src/replicator/replication.cc +++ b/src/replicator/replication.cc @@ -673,7 +673,6 @@ RemoteURL::updatePath(int _major, int _minor, int _index) filespec = parts[0] + "/" + parts[1] + "/" + newpath + suffix; destdir = parts[0] + "/" + parts[1] + "/" + majorfmt.str() + "/" + minorfmt.str(); subpath = newpath; - dump(); } void diff --git a/src/replicator/threads.cc b/src/replicator/threads.cc index 9187ab98..803b01ea 100644 --- a/src/replicator/threads.cc +++ b/src/replicator/threads.cc @@ -181,6 +181,9 @@ startMonitorChangesets(std::shared_ptr &remote, if (last_task->status == reqfile_t::success || (last_task->status == reqfile_t::remoteNotFound && !caughtUpWithNow)) { remote->Increment(); + if (!config.silent) { + remote->dump(); + } } auto new_remote = std::make_shared(remote->getURL()); new_remote->destdir_base = remote->destdir_base; @@ -219,6 +222,9 @@ startMonitorChangesets(std::shared_ptr &remote, std::stoi(closest.url.substr(4, 3)), std::stoi(closest.url.substr(8, 3)) ); + if (!config.silent) { + remote->dump(); + } cores = 1; delay = std::chrono::seconds{45}; } @@ -309,6 +315,9 @@ startMonitorChanges(std::shared_ptr &remote, if (last_task->status == reqfile_t::success || (last_task->status == reqfile_t::remoteNotFound && !caughtUpWithNow)) { remote->Increment(); + if (!config.silent) { + remote->dump(); + } } auto new_remote = std::make_shared(remote->getURL()); new_remote->destdir_base = remote->destdir_base; @@ -353,6 +362,9 @@ startMonitorChanges(std::shared_ptr &remote, std::stoi(closest.url.substr(4, 3)), std::stoi(closest.url.substr(8, 3)) ); + if (!config.silent) { + remote->dump(); + } concurrentTasks = 1; delay = std::chrono::seconds{45}; } @@ -457,10 +469,13 @@ threadOsmChange(OsmChangeTask osmChangeTask) } } - // Finish filling node cache with nodes referenced in modified - // or created ways and also ways affected by modified nodes + // - Fill node cache with nodes referenced in modified + // or created ways and also ways affected by modified nodes + // - Add indirectly modified ways to osmchanges + // - Build ways geometries using nodecache + // - Build relation multipolyon geometries (TODO) if (!config->disable_raw) { - queryraw->getNodeCache(osmchanges, poly); + queryraw->buildGeometries(osmchanges, poly); } // Filter data by priority polygon @@ -526,23 +541,23 @@ threadOsmChange(OsmChangeTask osmChangeTask) } // Relations - // for (auto rit = std::begin(change->relations); rit != std::end(change->relations); ++rit) { - // osmobjects::OsmRelation *relation = rit->get(); + for (auto rit = std::begin(change->relations); rit != std::end(change->relations); ++rit) { + osmobjects::OsmRelation *relation = rit->get(); - // if (relation->action != osmobjects::remove && !relation->priority) { - // continue; - // } + if (relation->action != osmobjects::remove && !relation->priority) { + continue; + } - // // Remove deleted relations from validation table - // if (!config->disable_validation && relation->action == osmobjects::remove) { - // removed_relations->push_back(relation->id); - // } + // Remove deleted relations from validation table + if (!config->disable_validation && relation->action == osmobjects::remove) { + removed_relations->push_back(relation->id); + } - // // Update relations, ignore new ones outside priority area - // if (!config->disable_raw) { - // task.query += queryraw->applyChange(*relation); - // } - // } + // Update relations, ignore new ones outside priority area + if (!config->disable_raw) { + task.query += queryraw->applyChange(*relation); + } + } } } @@ -565,7 +580,7 @@ threadOsmChange(OsmChangeTask osmChangeTask) task.query += queryvalidate->updateValidation(validation_removals); task.query += queryvalidate->updateValidation(removed_nodes); task.query += queryvalidate->updateValidation(removed_ways); - // task.query += queryvalidate->updateValidation(removed_relations); + task.query += queryvalidate->updateValidation(removed_relations); } diff --git a/src/testsuite/libunderpass.all/raw-test.cc b/src/testsuite/libunderpass.all/raw-test.cc new file mode 100644 index 00000000..e09466dc --- /dev/null +++ b/src/testsuite/libunderpass.all/raw-test.cc @@ -0,0 +1,40 @@ +// +// Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +// +// This file is part of Underpass. +// +// Underpass is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Underpass is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Underpass. If not, see . +// + +#include +#include +#include "utils/log.hh" + +using namespace logger; +using namespace queryraw; + +TestState runtest; +class TestOsmChange : public osmchange::OsmChangeFile {}; + +int +main(int argc, char *argv[]) +{ + logger::LogFile &dbglogfile = logger::LogFile::getDefaultInstance(); + dbglogfile.setWriteDisk(true); + dbglogfile.setLogFilename("raw-test.log"); + dbglogfile.setVerbosity(3); + + + +} \ No newline at end of file diff --git a/src/underpass.cc b/src/underpass.cc index 723db754..63e25aa3 100644 --- a/src/underpass.cc +++ b/src/underpass.cc @@ -124,7 +124,8 @@ main(int argc, char *argv[]) ("disable-validation", "Disable validation") ("disable-raw", "Disable raw OSM data") ("norefs", "Disable refs (useful for non OSM data)") - ("bootstrap", "Bootstrap data tables"); + ("bootstrap", "Bootstrap data tables") + ("silent", "Silent"); // clang-format on opts::store(opts::command_line_parser(argc, argv).options(desc).positional(p).run(), vm); @@ -135,6 +136,7 @@ main(int argc, char *argv[]) return 1; } + // Data processing if (vm.count("norefs")) { config.norefs = true; } @@ -160,6 +162,10 @@ main(int argc, char *argv[]) config.destdir_base = vm["destdir_base"].as(); } + if (vm.count("silent")) { + config.silent = true; + } + // Concurrency if (vm.count("concurrency")) { const auto concurrency = vm["concurrency"].as(); @@ -307,6 +313,9 @@ main(int argc, char *argv[]) std::vector parts; boost::split(parts, vm["changeseturl"].as(), boost::is_any_of("/")); changeset->updatePath(stoi(parts[0]),stoi(parts[1]),stoi(parts[2])); + if (!config.silent) { + changeset->dump(); + } } if (!vm.count("osmchanges")) { multipolygon_t * oscboundary = &poly; diff --git a/src/underpassconfig.hh b/src/underpassconfig.hh index 85f7abf7..5fd49fe8 100644 --- a/src/underpassconfig.hh +++ b/src/underpassconfig.hh @@ -168,6 +168,7 @@ struct UnderpassConfig { bool disable_stats = false; bool disable_raw = false; bool norefs = false; + bool silent = false; /// /// \brief getPlanetServer returns either the command line supplied planet server diff --git a/src/utils/geoutil.cc b/src/utils/geoutil.cc index 63c07587..30b240fb 100644 --- a/src/utils/geoutil.cc +++ b/src/utils/geoutil.cc @@ -32,19 +32,19 @@ #include #include #include -#include +// #include #include #include #include #include -#include "boost/date_time/posix_time/posix_time.hpp" -#include -using namespace boost::posix_time; -using namespace boost::gregorian; +// #include "boost/date_time/posix_time/posix_time.hpp" +// #include +// using namespace boost::posix_time; +// using namespace boost::gregorian; #include "utils/geoutil.hh" -#include "osm/changeset.hh" +// #include "osm/changeset.hh" #include #include diff --git a/src/utils/geoutil.hh b/src/utils/geoutil.hh index b94fbbd8..21a08157 100644 --- a/src/utils/geoutil.hh +++ b/src/utils/geoutil.hh @@ -37,12 +37,12 @@ #include #include #include -#include +// #include -#include -#include "boost/date_time/posix_time/posix_time.hpp" -using namespace boost::posix_time; -using namespace boost::gregorian; +// #include +// #include "boost/date_time/posix_time/posix_time.hpp" +// using namespace boost::posix_time; +// using namespace boost::gregorian; #include #include From 84901c7ba9910569ce7269db66473810a7325fad Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Mon, 4 Mar 2024 19:40:40 -0300 Subject: [PATCH 2/4] + Tests for raw data processing --- Makefile.am | 2 +- src/osm/osmchange.cc | 3 + src/osm/osmobjects.hh | 2 +- src/raw/queryraw.cc | 14 +- src/raw/queryraw.hh | 2 +- src/replicator/replication.cc | 1 - src/replicator/threads.cc | 2 +- src/testsuite/libunderpass.all/Makefile.am | 6 + src/testsuite/libunderpass.all/raw-test.cc | 153 +++++++++++++++++- src/testsuite/libunderpass.all/val-test.cc | 5 +- src/testsuite/testdata/raw/raw-case-1.osc | 16 ++ src/testsuite/testdata/raw/raw-case-2.osc | 11 ++ src/testsuite/testdata/raw/raw-case-3.osc | 5 + .../testdata/validation/config/place.yaml | 6 + src/validate/validate.hh | 1 - 15 files changed, 212 insertions(+), 17 deletions(-) create mode 100644 src/testsuite/testdata/raw/raw-case-1.osc create mode 100644 src/testsuite/testdata/raw/raw-case-2.osc create mode 100644 src/testsuite/testdata/raw/raw-case-3.osc diff --git a/Makefile.am b/Makefile.am index 3b2677fb..9ab61234 100644 --- a/Makefile.am +++ b/Makefile.am @@ -153,7 +153,6 @@ endif install-data-hook: $(MKDIR_P) $(DESTDIR)/$(pkglibdir) $(MKDIR_P) /etc/underpass - cp -rvp $(srcdir)/setup/service/underpass.service /etc/systemd/system/ cp -rvp $(srcdir)/config/priority.geojson /etc/underpass/ cp -rvp $(srcdir)/config/replicator /etc/underpass/ cp -rvp $(srcdir)/config/validate /etc/underpass/ @@ -162,6 +161,7 @@ install-data-hook: $(MKDIR_P) $(DESTDIR)/$(docdir) cp -rvp $(srcdir)/docs/*.md $(DESTDIR)/$(docdir) cp -rvp $(srcdir)/setup/service $(DESTDIR)/$(pkglibdir) + cp -rvp $(srcdir)/setup/service/underpass.service /etc/systemd/system/ dist-hook: apidoc $(MKDIR_P) $(DESTDIR)/$(docdir) diff --git a/src/osm/osmchange.cc b/src/osm/osmchange.cc index 76ec8164..9598716d 100644 --- a/src/osm/osmchange.cc +++ b/src/osm/osmchange.cc @@ -137,6 +137,9 @@ OsmChangeFile::buildGeometriesFromNodeCache() { for (auto lit = std::begin(way->refs); lit != std::end(way->refs); ++lit) { boost::geometry::append(way->linestring, nodecache[*lit]); } + if (way->isClosed()) { + way->polygon = { {std::begin(way->linestring), std::end(way->linestring)} }; + } } } } diff --git a/src/osm/osmobjects.hh b/src/osm/osmobjects.hh index b090c3e6..2abd3578 100644 --- a/src/osm/osmobjects.hh +++ b/src/osm/osmobjects.hh @@ -170,7 +170,7 @@ class OsmWay : public OsmObject { /// is a linestring bool isClosed(void) { - return !linestring.empty() && boost::geometry::equals(linestring.back(), linestring.front()); + return refs.front() == refs.back(); }; /// Return the number of nodes in this way int numPoints(void) { return boost::geometry::num_points(linestring); }; diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc index cd8db874..89c7cdcb 100644 --- a/src/raw/queryraw.cc +++ b/src/raw/queryraw.cc @@ -312,13 +312,12 @@ std::vector arrayStrToVector(std::string &refs_str) { } void -QueryRaw::getWaysByIds(std::string &relsForWayCacheIds, std::map> waycache) { +QueryRaw::getWaysByIds(std::string &waysIds, std::map> &waycache) { #ifdef TIMING_DEBUG - boost::timer::auto_cpu_timer timer("getWaysByIds(relsForWayCacheIds, waycache): took %w seconds\n"); + boost::timer::auto_cpu_timer timer("getWaysByIds(waysIds, waycache): took %w seconds\n"); #endif // Get all ways that have references to nodes - relsForWayCacheIds.erase(relsForWayCacheIds.size() - 1); - std::string waysQuery = "SELECT distinct(osm_id), ST_AsText(geom, 4326) from ways_poly wp where osm_id = any(ARRAY[" + relsForWayCacheIds + "])"; + std::string waysQuery = "SELECT distinct(osm_id), ST_AsText(geom, 4326) from ways_poly wp where osm_id = any(ARRAY[" + waysIds + "])"; auto ways_result = dbconn->query(waysQuery); // Fill vector of OsmWay objects @@ -355,7 +354,7 @@ void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const if (way->linestring.size() == way->refs.size()) { // Save only ways with a geometry that are inside the priority area // these are mostly created ways - if (boost::geometry::within(way->linestring, poly)) { + if (poly.empty() || boost::geometry::within(way->linestring, poly)) { osmchanges->waycache.insert(std::make_pair(way->id, std::make_shared(*way))); } } @@ -369,7 +368,7 @@ void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const OsmNode *node = nit->get(); if (node->action == osmobjects::modify) { // Get only modified nodes ids inside the priority area - if (boost::geometry::within(node->point, poly)) { + if (poly.empty() || boost::geometry::within(node->point, poly)) { modifiedNodesIds += std::to_string(node->id) + ","; } } @@ -431,7 +430,7 @@ void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const } } // Save way pointer for later use - if (boost::geometry::within(way->linestring, poly)) { + if (poly.empty() || boost::geometry::within(way->linestring, poly)) { osmchanges->waycache.insert(std::make_pair(way->id, std::make_shared(*way))); } } @@ -467,6 +466,7 @@ void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const } // Get all missing ways geometries for relations if (relsForWayCacheIds != "") { + relsForWayCacheIds.erase(relsForWayCacheIds.size() - 1); getWaysByIds(relsForWayCacheIds, osmchanges->waycache); } diff --git a/src/raw/queryraw.hh b/src/raw/queryraw.hh index 57a52fba..153d1d3e 100644 --- a/src/raw/queryraw.hh +++ b/src/raw/queryraw.hh @@ -73,7 +73,7 @@ class QueryRaw { // Get ways by refs std::list> getWaysByNodesRefs(std::string &nodeIds) const; // Get ways by ids (used for getting relations geometries) - void getWaysByIds(std::string &relsForWayCacheIds, std::map> waycache); + void getWaysByIds(std::string &relsForWayCacheIds, std::map> &waycache); // DB connection std::shared_ptr dbconn; // Get ways count diff --git a/src/replicator/replication.cc b/src/replicator/replication.cc index 2caaffd1..0cdde75a 100644 --- a/src/replicator/replication.cc +++ b/src/replicator/replication.cc @@ -403,7 +403,6 @@ Planet::downloadFile(const std::string &url, const std::string &destdir_base) return file; } - RequestedFile Planet::readFile(std::string &filespec) { log_debug("Reading cached file: %1%", filespec); diff --git a/src/replicator/threads.cc b/src/replicator/threads.cc index 803b01ea..b8b1de4e 100644 --- a/src/replicator/threads.cc +++ b/src/replicator/threads.cc @@ -473,7 +473,7 @@ threadOsmChange(OsmChangeTask osmChangeTask) // or created ways and also ways affected by modified nodes // - Add indirectly modified ways to osmchanges // - Build ways geometries using nodecache - // - Build relation multipolyon geometries (TODO) + // - Build relation multipolyon geometries if (!config->disable_raw) { queryraw->buildGeometries(osmchanges, poly); } diff --git a/src/testsuite/libunderpass.all/Makefile.am b/src/testsuite/libunderpass.all/Makefile.am index f52ee79e..216e26dd 100644 --- a/src/testsuite/libunderpass.all/Makefile.am +++ b/src/testsuite/libunderpass.all/Makefile.am @@ -31,6 +31,7 @@ check_PROGRAMS = \ stats-test \ val-test \ val-unsquared-test \ + raw-test \ test-playground TOPSRC := $(shell cd $(top_srcdir) && pwd)/src @@ -82,6 +83,11 @@ val_unsquared_test_LDFLAGS = -L../.. val_unsquared_test_CPPFLAGS = -DDATADIR=\"$(TOPSRC)\" -I$(TOPSRC) val_unsquared_test_LDADD = -lpqxx -lunderpass $(BOOST_LIBS) ../../validate/defaultvalidation.lo ../../validate/geospatial.lo ../../validate/semantic.lo +raw_test_SOURCES = raw-test.cc +raw_test_LDFLAGS = -L../.. +raw_test_CPPFLAGS = -DDATADIR=\"$(TOPSRC)\" -I$(TOPSRC) +raw_test_LDADD = -lpqxx -lunderpass $(BOOST_LIBS) + # Test the replication classes #replication_test_SOURCES = replication-test.cc #replication_test_LDFLAGS = -L../.. diff --git a/src/testsuite/libunderpass.all/raw-test.cc b/src/testsuite/libunderpass.all/raw-test.cc index e09466dc..da81f664 100644 --- a/src/testsuite/libunderpass.all/raw-test.cc +++ b/src/testsuite/libunderpass.all/raw-test.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // @@ -20,13 +20,104 @@ #include #include #include "utils/log.hh" +#include "osm/osmobjects.hh" +#include "raw/queryraw.hh" +#include +#include "replicator/replication.hh" +#include +using namespace replication; using namespace logger; using namespace queryraw; TestState runtest; class TestOsmChange : public osmchange::OsmChangeFile {}; +class TestPlanet : public Planet { + public: + TestPlanet() = default; + + //! Clear the test DB and fill it with with initial test data + bool init_test_case(const std::string &dbconn) + { + source_tree_root = getenv("UNDERPASS_SOURCE_TREE_ROOT") + ? getenv("UNDERPASS_SOURCE_TREE_ROOT") + : "../"; + + const std::string test_replication_db_name{"underpass_test"}; + { + pqxx::connection conn{dbconn + " dbname=template1"}; + pqxx::nontransaction worker{conn}; + worker.exec0("DROP DATABASE IF EXISTS " + test_replication_db_name); + worker.exec0("CREATE DATABASE " + test_replication_db_name); + worker.commit(); + } + + { + pqxx::connection conn{dbconn + " dbname=" + test_replication_db_name}; + pqxx::nontransaction worker{conn}; + worker.exec0("CREATE EXTENSION postgis"); + worker.exec0("CREATE EXTENSION hstore"); + + // Create schema + const auto schema_path{source_tree_root + "setup/db/underpass.sql"}; + std::ifstream schema_definition(schema_path); + std::string sql((std::istreambuf_iterator(schema_definition)), + std::istreambuf_iterator()); + + assert(!sql.empty()); + worker.exec0(sql); + } + + return true; + }; + + std::string source_tree_root; +}; + +bool processFile(const std::string &filename, std::shared_ptr &db) { + auto queryraw = std::make_shared(db); + auto osmchanges = std::make_shared(); + std::string destdir_base = DATADIR; + multipolygon_t poly; + osmchanges->readChanges(destdir_base + "/testsuite/testdata/raw/" + filename); + queryraw->buildGeometries(osmchanges, poly); + std::string rawquery; + + for (auto it = std::begin(osmchanges->changes); it != std::end(osmchanges->changes); ++it) { + osmchange::OsmChange *change = it->get(); + // Nodes + for (auto nit = std::begin(change->nodes); nit != std::end(change->nodes); ++nit) { + osmobjects::OsmNode *node = nit->get(); + rawquery += queryraw->applyChange(*node); + } + // Ways + for (auto wit = std::begin(change->ways); wit != std::end(change->ways); ++wit) { + osmobjects::OsmWay *way = wit->get(); + rawquery += queryraw->applyChange(*way); + } + // Relations + for (auto rit = std::begin(change->relations); rit != std::end(change->relations); ++rit) { + osmobjects::OsmRelation *relation = rit->get(); + rawquery += queryraw->applyChange(*relation); + } + } + db->query(rawquery); +} + +const std::map expectedGeometries = { + {101874, "POLYGON((21.726001473 4.62042952837,21.726086573 4.62042742837,21.726084973 4.62036492836,21.725999873 4.62036702836,21.726001473 4.62042952837))"}, + {101875, "POLYGON((21.726001473 4.62042952837,21.726086573 4.62042742837,21.726084973 4.62036492836,21.725999873 4.62036702836,21.726001473 4.62042952837))"}, + {-101875, "POLYGON((21.72600148 4.62042953,21.726086573 4.62042742837,21.726084973 4.62036492836,21.725999873 4.62036702836,21.72600148 4.62042953))"} +}; + +std::string +getWKT(polygon_t polygon) { + std::stringstream ss; + ss << std::setprecision(12) << boost::geometry::wkt(polygon); + return ss.str(); +} + int main(int argc, char *argv[]) { @@ -35,6 +126,64 @@ main(int argc, char *argv[]) dbglogfile.setLogFilename("raw-test.log"); dbglogfile.setVerbosity(3); - + const std::string dbconn{getenv("UNDERPASS_TEST_DB_CONN") + ? getenv("UNDERPASS_TEST_DB_CONN") + : "user=underpass_test host=localhost password=underpass_test"}; + + TestPlanet test_planet; + test_planet.init_test_case(dbconn); + auto db = std::make_shared(); + + if (db->connect(dbconn + " dbname=underpass_test")) { + auto queryraw = std::make_shared(db); + std::map> waycache; + std::string waysIds; + + processFile("raw-case-1.osc", db); + processFile("raw-case-2.osc", db); + + for (auto const& x : expectedGeometries) { + waysIds += std::to_string(x.first) + ","; + } + waysIds.erase(waysIds.size() - 1); + + queryraw->getWaysByIds(waysIds, waycache); + + // 4 created Nodes, 1 created Way (same changeset) + if ( getWKT(waycache.at(101874)->polygon).compare(expectedGeometries.at(101874)) == 0) { + runtest.pass("4 created Nodes, 1 created Ways (same changeset)"); + } else { + runtest.fail("4 created Nodes, 1 created Ways (same changeset)"); + return 1; + } + + // 1 created Way, 4 existing Nodes (different changeset) + if ( getWKT(waycache.at(101875)->polygon).compare(expectedGeometries.at(101875)) == 0) { + runtest.pass("1 created Way, 4 existing Nodes (different changesets)"); + } else { + runtest.fail("1 created Way, 4 existing Nodes (different changesets)"); + return 1; + } + + // 1 modified node, indirectly modify other existing ways + processFile("raw-case-3.osc", db); + waycache.erase(101875); + queryraw->getWaysByIds(waysIds, waycache); + + if ( getWKT(waycache.at(101875)->polygon).compare(expectedGeometries.at(-101875)) == 0) { + runtest.pass("1 modified node, indirectly modify other existing ways (different changesets)"); + } else { + runtest.fail("1 modified node, indirectly modify other existing ways (different changesets)"); + return 1; + } + + // TODO: relations + // 1 created Relation using existing ways + // 1 modified node, indirectly modify other existing ways and relation + + + } + + } \ No newline at end of file diff --git a/src/testsuite/libunderpass.all/val-test.cc b/src/testsuite/libunderpass.all/val-test.cc index 301ea5ee..9c1c7207 100644 --- a/src/testsuite/libunderpass.all/val-test.cc +++ b/src/testsuite/libunderpass.all/val-test.cc @@ -144,9 +144,9 @@ test_semantic(std::shared_ptr &plugin) { node.addTag("building", "yes "); status = plugin->checkNode(node, "building"); if (status->hasStatus(badvalue)) { - runtest.pass("Validate::checkNode(single quotes) [semantic building]"); + runtest.pass("Validate::checkNode(extra space) [semantic building]"); } else { - runtest.fail("Validate::checkNode(single quotes) [semantic building]"); + runtest.fail("Validate::checkNode(extra space) [semantic building]"); return 1; } @@ -166,6 +166,7 @@ test_semantic(std::shared_ptr &plugin) { // Has valid tags, but it's incomplete node_place.addTag("place", "city"); status = plugin->checkNode(node_place, "place"); + if (!status->hasStatus(badvalue) && status->hasStatus(incomplete)) { runtest.pass("Validate::checkNode(incomplete but correct tagging) [semantic place]"); } else { diff --git a/src/testsuite/testdata/raw/raw-case-1.osc b/src/testsuite/testdata/raw/raw-case-1.osc new file mode 100644 index 00000000..d8adf181 --- /dev/null +++ b/src/testsuite/testdata/raw/raw-case-1.osc @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/testsuite/testdata/raw/raw-case-2.osc b/src/testsuite/testdata/raw/raw-case-2.osc new file mode 100644 index 00000000..5c3998fb --- /dev/null +++ b/src/testsuite/testdata/raw/raw-case-2.osc @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/src/testsuite/testdata/raw/raw-case-3.osc b/src/testsuite/testdata/raw/raw-case-3.osc new file mode 100644 index 00000000..f8eec93e --- /dev/null +++ b/src/testsuite/testdata/raw/raw-case-3.osc @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/src/testsuite/testdata/validation/config/place.yaml b/src/testsuite/testdata/validation/config/place.yaml index cd83ead7..1efc1f98 100644 --- a/src/testsuite/testdata/validation/config/place.yaml +++ b/src/testsuite/testdata/validation/config/place.yaml @@ -1,4 +1,10 @@ +config: + - badvalue: + - yes + - incomplete: + - yes + required_tags: - name diff --git a/src/validate/validate.hh b/src/validate/validate.hh index d3ed80f2..1a216941 100644 --- a/src/validate/validate.hh +++ b/src/validate/validate.hh @@ -200,7 +200,6 @@ class BOOST_SYMBOL_VISIBLE Validate { } virtual ~Validate(void){}; - // Validate(std::vector> &changes) {}; virtual std::shared_ptr checkNode(const osmobjects::OsmNode &node, const std::string &type) = 0; virtual std::shared_ptr checkWay(const osmobjects::OsmWay &way, const std::string &type) = 0; From d6f0c07d0ec6f6eef4944ee2a47cd716327e7ec0 Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Wed, 6 Mar 2024 09:55:55 -0300 Subject: [PATCH 3/4] Multi-polygon eometries for relations, tests for raw data, some updates on db and docs --- docs/Dev/debugging.md | 3 + setup/db/underpass.sql | 14 +- setup/install-macos-arm64.sh | 24 +-- src/bootstrap/bootstrap.cc | 10 +- src/osm/osmobjects.hh | 2 +- src/raw/queryraw.cc | 214 +++++++++++++++------ src/raw/queryraw.hh | 2 + src/replicator/threads.cc | 2 +- src/testsuite/libunderpass.all/raw-test.cc | 39 +++- src/testsuite/testdata/raw/raw-case-2.osc | 1 + src/testsuite/testdata/raw/raw-case-3.osc | 1 + src/testsuite/testdata/raw/raw-case-4.osc | 24 +++ src/testsuite/testdata/raw/raw-case-5.osc | 8 + 13 files changed, 254 insertions(+), 90 deletions(-) create mode 100644 src/testsuite/testdata/raw/raw-case-4.osc create mode 100644 src/testsuite/testdata/raw/raw-case-5.osc diff --git a/docs/Dev/debugging.md b/docs/Dev/debugging.md index d303546f..e70f9189 100644 --- a/docs/Dev/debugging.md +++ b/docs/Dev/debugging.md @@ -37,6 +37,9 @@ Add the alias in your `~/.bashrc` file: ### With LLDB +When running Underpass on a MacOS system with an Arm64 architecture, `gdb` might be not available. +You can use `lldb` instead. + `lldb -- src/testsuite/libunderpass.all/yaml-test` `lldb -- .libs/underpass --bootstrap` diff --git a/setup/db/underpass.sql b/setup/db/underpass.sql index 0641010f..3d30dfd4 100644 --- a/setup/db/underpass.sql +++ b/setup/db/underpass.sql @@ -99,24 +99,36 @@ CREATE TABLE IF NOT EXISTS public.relations ( changeset int8, geom public.geometry(Geometry,4326), tags JSONB, - refs int8[], + refs JSONB, timestamp timestamp with time zone, version int, "user" text, uid int8 ); +ALTER TABLE ONLY public.relations + ADD CONSTRAINT relations_pkey PRIMARY KEY (osm_id); + CREATE TABLE IF NOT EXISTS public.way_refs ( way_id int8, node_id int8 ); +CREATE TABLE IF NOT EXISTS public.rel_refs ( + rel_id int8, + way_id int8 +); + CREATE UNIQUE INDEX nodes_id_idx ON public.nodes (osm_id DESC); CREATE UNIQUE INDEX ways_poly_id_idx ON public.ways_poly (osm_id DESC); CREATE UNIQUE INDEX ways_line_id_idx ON public.ways_line(osm_id DESC); +CREATE UNIQUE INDEX relations_id_idx ON public.relations(osm_id DESC); CREATE INDEX way_refs_node_id_idx ON public.way_refs (node_id); CREATE INDEX way_refs_way_id_idx ON public.way_refs (way_id); +CREATE INDEX rel_refs_rel_id_idx ON public.rel_refs (rel_id); +CREATE INDEX rel_refs_way_id_idx ON public.rel_refs (way_id); + CREATE INDEX nodes_version_idx ON public.nodes (version); CREATE INDEX ways_poly_version_idx ON public.ways_poly (version); CREATE INDEX ways_line_version_idx ON public.ways_line (version); diff --git a/setup/install-macos-arm64.sh b/setup/install-macos-arm64.sh index d67d942e..2404f9c3 100644 --- a/setup/install-macos-arm64.sh +++ b/setup/install-macos-arm64.sh @@ -34,25 +34,15 @@ cd .. echo "Setting up build ..." ./autogen.sh mkdir build && cd build -../configure CXXFLAGS="-arch arm64 \ - -I/usr/local/include \ - -L/opt/homebrew/lib \ - -I/opt/homebrew/include \ - -I/opt/homebrew/Cellar/ \ - -L/usr/local/Cellar/ \ - -g -O2" CXX="g++" +../configure CXXFLAGS="-arch arm64 -g -O2" \ + LDFLAGS="-L/opt/homebrew/lib -L/usr/local/lib" \ + CPPFLAGS="-I/opt/homebrew/include -I/usr/local/include" \ + CXX="g++" \ + --enable-python=no \ + --with-boost=/opt/local/libexec/boost/1.76/lib + echo "Building ..." make -j$(nproc) && sudo make install -echo "Setting up libs ..." -ln -s /usr/local/lib/libboost_date_time.dylib .libs -ln -s /usr/local/lib/libboost_system.dylib .libs -ln -s /usr/local/lib/libboost_filesystem.dylib .libs -ln -s /usr/local/lib/libboost_log.dylib .libs -ln -s /usr/local/lib/libboost_program_options.dylib .libs -ln -s /usr/local/lib/libboost_iostreams.dylib .libs -ln -s /usr/local/lib/libboost_thread.dylib .libs -ln -s /usr/local/lib/libboost_serialization.dylib .libs -ln -s /usr/local/lib/libboost_regex.dylib .libs echo "Done! now you may want to initialize the database with the bootstrap script" diff --git a/src/bootstrap/bootstrap.cc b/src/bootstrap/bootstrap.cc index 0004f58b..61c46943 100644 --- a/src/bootstrap/bootstrap.cc +++ b/src/bootstrap/bootstrap.cc @@ -60,7 +60,7 @@ Bootstrap::allTasksQueries(std::shared_ptr> tasks) { void Bootstrap::start(const underpassconfig::UnderpassConfig &config) { - std::cout << "Connecting to the database ..." << std::endl; + std::cout << "Connecting to the database ... " << std::endl; db = std::make_shared(); if (!db->connect(config.underpass_db_url)) { std::cout << "Could not connect to Underpass DB, aborting bootstrapping thread!" << std::endl; @@ -348,10 +348,10 @@ Bootstrap::threadBootstrapRelationTask(RelationTask relationTask) if (i < relations->size()) { auto relation = relations->at(i); // relationval->push_back(validator->checkRelation(way, "building")); - // Fill the rel_members table - // for (auto ref = relation.refs.begin(); ref != relation.refs.end(); ++ref) { - // task.query += "INSERT INTO rel_refs (rel_id, way_id) VALUES (" + std::to_string(rel.id) + "," + std::to_string(*ref) + "); "; - // } + // Fill the rel_refs table + for (auto mit = relation.members.begin(); mit != relation.members.end(); ++mit) { + task.query += "INSERT INTO rel_refs (rel_id, way_id) VALUES (" + std::to_string(relation.id) + "," + std::to_string(mit->ref) + "); "; + } ++processed; } } diff --git a/src/osm/osmobjects.hh b/src/osm/osmobjects.hh index 2abd3578..2c3c6493 100644 --- a/src/osm/osmobjects.hh +++ b/src/osm/osmobjects.hh @@ -170,7 +170,7 @@ class OsmWay : public OsmObject { /// is a linestring bool isClosed(void) { - return refs.front() == refs.back(); + return (refs.size() > 3 && refs.front() == refs.back()); }; /// Return the number of nodes in this way int numPoints(void) { return boost::geometry::num_points(linestring); }; diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc index 89c7cdcb..77d91edf 100644 --- a/src/raw/queryraw.cc +++ b/src/raw/queryraw.cc @@ -86,6 +86,72 @@ QueryRaw::buildTagsQuery(std::map tags) const { } } +std::string +buildMembersQuery(std::list members) { + if (members.size() > 0) { + std::string membersStr = "jsonb_build_array("; + int count = 0; + for (auto mit = std::begin(members); mit != std::end(members); ++mit) { + membersStr += "jsonb_build_object("; + std::string member_format = "'%s', '%s',"; + boost::format member_fmt(member_format); + member_fmt % "role"; + member_fmt % mit->role; + membersStr += member_fmt.str(); + member_fmt % "type"; + member_fmt % mit->type; + membersStr += member_fmt.str(); + member_fmt % "ref"; + member_fmt % mit->ref; + membersStr += member_fmt.str(); + membersStr.erase(membersStr.size() - 1); + membersStr += "),"; + } + membersStr.erase(membersStr.size() - 1); + return membersStr += ")"; + } else { + return "null"; + } +} + +std::map parseJSONObjectStr(std::string input) { + std::map obj; + boost::property_tree::ptree pt; + try { + std::istringstream jsonStream(input); + boost::property_tree::read_json(jsonStream, pt); + } catch (const boost::property_tree::json_parser::json_parser_error& e) { + std::cerr << "Error parsing JSON: " << e.what() << std::endl; + return obj; + } + for (const auto& pair : pt) { + obj[pair.first] = pair.second.get_value(); + } + return obj; +} + +std::vector> parseJSONArrayStr(std::string input) { + std::vector> arr; + boost::property_tree::ptree pt; + try { + std::istringstream jsonStream(input); + boost::property_tree::read_json(jsonStream, pt); + } catch (const boost::property_tree::json_parser::json_parser_error& e) { + std::cerr << "Error parsing JSON: " << e.what() << std::endl; + return arr; + } + + for (const auto& item : pt) { + std::map obj; + for (const auto& pair : item.second) { + obj[pair.first] = pair.second.get_value(); + } + arr.push_back(obj); + } + + return arr; +} + std::string QueryRaw::applyChange(const OsmNode &node) const { @@ -251,17 +317,12 @@ QueryRaw::applyChange(const OsmRelation &relation) const // osm_id fmt % relation.id; - //tags + // tags auto tags = buildTagsQuery(relation.tags); fmt % tags; // refs - std::string refs = ""; - for (auto it = std::begin(relation.members); it != std::end(relation.members); ++it) { - refs += std::to_string(it->ref) + ","; - } - refs.erase(refs.size() - 1); - refs = "ARRAY[" + refs + "]"; + auto refs = buildMembersQuery(relation.members); fmt % refs; // geometry @@ -293,6 +354,11 @@ QueryRaw::applyChange(const OsmRelation &relation) const fmt % relation.version; query += fmt.str(); + + for (auto it = std::begin(relation.members); it != std::end(relation.members); ++it) { + query += "INSERT INTO rel_refs (rel_id, way_id) VALUES (" + std::to_string(relation.id) + "," + std::to_string(it->ref) + ");"; + } + } else if (relation.action == osmobjects::remove) { query += "DELETE FROM relations where osm_id = " + std::to_string(relation.id) + ";"; } @@ -311,6 +377,49 @@ std::vector arrayStrToVector(std::string &refs_str) { return refs; } +std::list> +QueryRaw::getRelationsByWaysRefs(std::string &wayIds) const +{ +#ifdef TIMING_DEBUG + boost::timer::auto_cpu_timer timer("getRelationsByWaysRefs(wayIds): took %w seconds\n"); +#endif + // Get all relations that have references to ways + std::list> rels; + + std::string relsQuery = "SELECT distinct(osm_id), refs, version, tags, uid, changeset from rel_refs join relations r on r.osm_id = rel_id where way_id = any(ARRAY[" + wayIds + "])"; + auto rels_result = dbconn->query(relsQuery); + + // Fill vector of OsmRelation objects + for (auto rel_it = rels_result.begin(); rel_it != rels_result.end(); ++rel_it) { + auto rel = std::make_shared(); + rel->id = (*rel_it)[0].as(); + std::string refs_str = (*rel_it)[1].as(); + auto members = parseJSONArrayStr(refs_str); + for (auto mit = members.begin(); mit != members.end(); ++mit) { + rel->addMember(std::stol(mit->at("ref")), osmobjects::osmtype_t::way, mit->at("role")); + } + rel->version = (*rel_it)[2].as(); + auto tags = (*rel_it)[3]; + if (!tags.is_null()) { + auto tags = parseJSONObjectStr((*rel_it)[3].as()); + for (auto const& [key, val] : tags) + { + rel->addTag(key, val); + } + } + auto uid = (*rel_it)[4]; + if (!uid.is_null()) { + rel->uid = (*rel_it)[4].as(); + } + auto changeset = (*rel_it)[5]; + if (!changeset.is_null()) { + rel->changeset = (*rel_it)[5].as(); + } + rels.push_back(rel); + } + return rels; +} + void QueryRaw::getWaysByIds(std::string &waysIds, std::map> &waycache) { #ifdef TIMING_DEBUG @@ -337,7 +446,9 @@ void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const #endif std::string referencedNodeIds; std::string modifiedNodesIds; + std::string modifiedWaysIds; std::vector removedWays; + std::vector removedRelations; for (auto it = std::begin(osmchanges->changes); it != std::end(osmchanges->changes); it++) { OsmChange *change = it->get(); @@ -351,7 +462,7 @@ void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const } } // Save ways for later use - if (way->linestring.size() == way->refs.size()) { + if (way->isClosed()) { // Save only ways with a geometry that are inside the priority area // these are mostly created ways if (poly.empty() || boost::geometry::within(way->linestring, poly)) { @@ -373,6 +484,11 @@ void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const } } } + + for (auto rel_it = std::begin(change->relations); rel_it != std::end(change->relations); ++rel_it) { + OsmRelation *relation = rel_it->get(); + removedRelations.push_back(relation->id); + } } // Add indirectly modified ways to osmchanges @@ -392,6 +508,24 @@ void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const if (std::find(removedWays.begin(), removedWays.end(), way->id) == removedWays.end()) { way->action = osmobjects::modify; change->ways.push_back(way); + modifiedWaysIds += std::to_string(way->id) + ","; + } + } + osmchanges->changes.push_back(change); + } + + // Add indirectly modified relations to osmchanges + if (modifiedWaysIds.size() > 1) { + modifiedWaysIds.erase(modifiedWaysIds.size() - 1); + auto modifiedRelations = getRelationsByWaysRefs(modifiedWaysIds); + auto change = std::make_shared(none); + for (auto rel_it = modifiedRelations.begin(); rel_it != modifiedRelations.end(); ++rel_it) { + auto relation = std::make_shared(*rel_it->get()); + // If the relation is not marked as removed, mark it as modified + if (std::find(removedRelations.begin(), removedRelations.end(), relation->id) == removedRelations.end()) { + relation->action = osmobjects::modify; + change->relations.push_back(relation); + } } osmchanges->changes.push_back(change); @@ -418,20 +552,23 @@ void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const OsmChange *change = it->get(); for (auto wit = std::begin(change->ways); wit != std::end(change->ways); ++wit) { OsmWay *way = wit->get(); - if (way->linestring.size() != way->refs.size()) { - way->linestring.clear(); - for (auto rit = way->refs.begin(); rit != way->refs.end(); ++rit) { - if (osmchanges->nodecache.count(*rit)) { - boost::geometry::append(way->linestring, osmchanges->nodecache.at(*rit)); - } - } - if (way->isClosed()) { - way->polygon = { {std::begin(way->linestring), std::end(way->linestring)} }; + way->linestring.clear(); + for (auto rit = way->refs.begin(); rit != way->refs.end(); ++rit) { + if (osmchanges->nodecache.count(*rit)) { + boost::geometry::append(way->linestring, osmchanges->nodecache.at(*rit)); } } + if (way->isClosed()) { + way->polygon = { {std::begin(way->linestring), std::end(way->linestring)} }; + } // Save way pointer for later use if (poly.empty() || boost::geometry::within(way->linestring, poly)) { - osmchanges->waycache.insert(std::make_pair(way->id, std::make_shared(*way))); + if (osmchanges->waycache.count(way->id)) { + osmchanges->waycache.at(way->id)->polygon = way->polygon; + } else { + osmchanges->waycache.insert(std::make_pair(way->id, std::make_shared(*way))); + } + } } } @@ -486,7 +623,8 @@ void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const break; } auto way = osmchanges->waycache.at(mit->ref); - if (boost::geometry::num_points(way->linestring) == 0 || + + if (boost::geometry::num_points(way->linestring) == 0 && boost::geometry::num_points(way->polygon) == 0 ) { noWay = true; @@ -555,44 +693,6 @@ QueryRaw::getNodeCacheFromWays(std::shared_ptr> ways, std::m } } -std::map parseJSONObjectStr(std::string input) { - std::map obj; - boost::property_tree::ptree pt; - try { - std::istringstream jsonStream(input); - boost::property_tree::read_json(jsonStream, pt); - } catch (const boost::property_tree::json_parser::json_parser_error& e) { - std::cerr << "Error parsing JSON: " << e.what() << std::endl; - return obj; - } - for (const auto& pair : pt) { - obj[pair.first] = pair.second.get_value(); - } - return obj; -} - -std::vector> parseJSONArrayStr(std::string input) { - std::vector> arr; - boost::property_tree::ptree pt; - try { - std::istringstream jsonStream(input); - boost::property_tree::read_json(jsonStream, pt); - } catch (const boost::property_tree::json_parser::json_parser_error& e) { - std::cerr << "Error parsing JSON: " << e.what() << std::endl; - return arr; - } - - for (const auto& item : pt) { - std::map obj; - for (const auto& pair : item.second) { - obj[pair.first] = pair.second.get_value(); - } - arr.push_back(obj); - } - - return arr; -} - std::list> QueryRaw::getWaysByNodesRefs(std::string &nodeIds) const { diff --git a/src/raw/queryraw.hh b/src/raw/queryraw.hh index 153d1d3e..8bee5c6c 100644 --- a/src/raw/queryraw.hh +++ b/src/raw/queryraw.hh @@ -74,6 +74,8 @@ class QueryRaw { std::list> getWaysByNodesRefs(std::string &nodeIds) const; // Get ways by ids (used for getting relations geometries) void getWaysByIds(std::string &relsForWayCacheIds, std::map> &waycache); + // Get relations by referenced ways + std::list> getRelationsByWaysRefs(std::string &wayIds) const; // DB connection std::shared_ptr dbconn; // Get ways count diff --git a/src/replicator/threads.cc b/src/replicator/threads.cc index b8b1de4e..3ccdc8b0 100644 --- a/src/replicator/threads.cc +++ b/src/replicator/threads.cc @@ -555,7 +555,7 @@ threadOsmChange(OsmChangeTask osmChangeTask) // Update relations, ignore new ones outside priority area if (!config->disable_raw) { - task.query += queryraw->applyChange(*relation); + // task.query += queryraw->applyChange(*relation); } } diff --git a/src/testsuite/libunderpass.all/raw-test.cc b/src/testsuite/libunderpass.all/raw-test.cc index da81f664..4260dca2 100644 --- a/src/testsuite/libunderpass.all/raw-test.cc +++ b/src/testsuite/libunderpass.all/raw-test.cc @@ -108,16 +108,26 @@ bool processFile(const std::string &filename, std::shared_ptr &db) { const std::map expectedGeometries = { {101874, "POLYGON((21.726001473 4.62042952837,21.726086573 4.62042742837,21.726084973 4.62036492836,21.725999873 4.62036702836,21.726001473 4.62042952837))"}, {101875, "POLYGON((21.726001473 4.62042952837,21.726086573 4.62042742837,21.726084973 4.62036492836,21.725999873 4.62036702836,21.726001473 4.62042952837))"}, - {-101875, "POLYGON((21.72600148 4.62042953,21.726086573 4.62042742837,21.726084973 4.62036492836,21.725999873 4.62036702836,21.72600148 4.62042953))"} + {101875-2, "POLYGON((21.72600148 4.62042953,21.726086573 4.62042742837,21.726084973 4.62036492836,21.725999873 4.62036702836,21.72600148 4.62042953))"}, + {211766, "MULTIPOLYGON(((21.72600148 4.62042953,21.726086573 4.62042742837,21.726084973 4.62036492836,21.725999873 4.62036702836,21.72600148 4.62042953),(21.7260170728 4.62041343508,21.7260713875 4.62041326798,21.7260708846 4.62037684165,21.7260165699 4.62038035061,21.7260170728 4.62041343508)))"}, + {211766-2, "MULTIPOLYGON(((21.72600148 4.62042953,21.726086573 4.62042742837,21.7260807753 4.62037032501,21.725999873 4.62036702836,21.72600148 4.62042953),(21.7260170728 4.62041343508,21.7260713875 4.62041326798,21.7260708846 4.62037684165,21.7260165699 4.62038035061,21.7260170728 4.62041343508)))"} }; std::string -getWKT(polygon_t polygon) { +getWKT(const polygon_t &polygon) { std::stringstream ss; ss << std::setprecision(12) << boost::geometry::wkt(polygon); return ss.str(); } +std::string +getWKTFromDB(const std::string &table, const long id, std::shared_ptr &db) { + auto result = db->query("SELECT ST_AsText(geom, 4326) from relations where osm_id=" + std::to_string(id)); + for (auto r_it = result.begin(); r_it != result.end(); ++r_it) { + return (*r_it)[0].as(); + } +} + int main(int argc, char *argv[]) { @@ -170,18 +180,31 @@ main(int argc, char *argv[]) waycache.erase(101875); queryraw->getWaysByIds(waysIds, waycache); - if ( getWKT(waycache.at(101875)->polygon).compare(expectedGeometries.at(-101875)) == 0) { - runtest.pass("1 modified node, indirectly modify other existing ways (different changesets)"); + if ( getWKT(waycache.at(101875)->polygon).compare(expectedGeometries.at(101875-2)) == 0) { + runtest.pass("1 modified Node, indirectly modify other existing Ways (different changesets)"); } else { - runtest.fail("1 modified node, indirectly modify other existing ways (different changesets)"); + runtest.fail("1 modified Node, indirectly modify other existing Ways (different changesets)"); return 1; } - // TODO: relations - // 1 created Relation using existing ways - // 1 modified node, indirectly modify other existing ways and relation + // 1 created Relation referencing 1 created Way and 1 existing Way + processFile("raw-case-4.osc", db); + if ( getWKTFromDB("relations", 211766, db).compare(expectedGeometries.at(211766)) == 0) { + runtest.pass("1 created Relation referencing 1 created Way and 1 existing Way (different changesets)"); + } else { + runtest.fail("1 created Relation referencing 1 created Way and 1 existing Way (different changesets)"); + return 1; + } + // 1 modified Node, indirectly modify other existing Ways and 1 Relation + processFile("raw-case-5.osc", db); + if ( getWKTFromDB("relations", 211766, db).compare(expectedGeometries.at(211766-2)) == 0) { + runtest.pass("1 modified Node, indirectly modify other existing Ways and 1 Relation (different changesets)"); + } else { + runtest.fail("1 modified Node, indirectly modify other existing Ways and 1 Relation (different changesets)"); + return 1; + } } diff --git a/src/testsuite/testdata/raw/raw-case-2.osc b/src/testsuite/testdata/raw/raw-case-2.osc index 5c3998fb..893ab9d0 100644 --- a/src/testsuite/testdata/raw/raw-case-2.osc +++ b/src/testsuite/testdata/raw/raw-case-2.osc @@ -1,3 +1,4 @@ + diff --git a/src/testsuite/testdata/raw/raw-case-3.osc b/src/testsuite/testdata/raw/raw-case-3.osc index f8eec93e..2d411bfb 100644 --- a/src/testsuite/testdata/raw/raw-case-3.osc +++ b/src/testsuite/testdata/raw/raw-case-3.osc @@ -1,3 +1,4 @@ + diff --git a/src/testsuite/testdata/raw/raw-case-4.osc b/src/testsuite/testdata/raw/raw-case-4.osc new file mode 100644 index 00000000..1c744f9e --- /dev/null +++ b/src/testsuite/testdata/raw/raw-case-4.osc @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/testsuite/testdata/raw/raw-case-5.osc b/src/testsuite/testdata/raw/raw-case-5.osc new file mode 100644 index 00000000..f6d041a6 --- /dev/null +++ b/src/testsuite/testdata/raw/raw-case-5.osc @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file From 17c7ab5d9e4fa64fe7d47cabe072d7a195296f70 Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Wed, 6 Mar 2024 10:17:23 -0300 Subject: [PATCH 4/4] Clean code: trailing spaces --- src/bootstrap/bootstrap.cc | 16 ++++++++-------- src/data/pq.cc | 2 +- src/osm/osmchange.cc | 2 +- src/raw/queryraw.cc | 10 ++++------ src/replicator/threads.cc | 4 ++-- src/stats/querystats.cc | 2 +- src/testsuite/libunderpass.all/geo-test.cc | 8 ++++---- .../libunderpass.all/planetreplicator-test.cc | 4 ++-- src/testsuite/libunderpass.all/under-test.cc | 4 ++-- src/testsuite/libunderpass.all/yaml-test.cc | 2 +- src/underpass.cc | 4 ++-- src/utils/geoutil.cc | 2 +- src/validate/geospatial.cc | 8 ++++---- src/validate/semantic.cc | 4 ++-- src/wrappers/python.cc | 2 +- 15 files changed, 36 insertions(+), 38 deletions(-) diff --git a/src/bootstrap/bootstrap.cc b/src/bootstrap/bootstrap.cc index 61c46943..1d84361f 100644 --- a/src/bootstrap/bootstrap.cc +++ b/src/bootstrap/bootstrap.cc @@ -58,7 +58,7 @@ Bootstrap::allTasksQueries(std::shared_ptr> tasks) { return queries; } -void +void Bootstrap::start(const underpassconfig::UnderpassConfig &config) { std::cout << "Connecting to the database ... " << std::endl; db = std::make_shared(); @@ -97,14 +97,14 @@ Bootstrap::start(const underpassconfig::UnderpassConfig &config) { } -void +void Bootstrap::processWays() { std::vector tables = { QueryRaw::polyTable, QueryRaw::lineTable }; - + for (auto table_it = tables.begin(); table_it != tables.end(); ++table_it) { std::cout << std::endl << "Processing ways ... "; long int total = queryraw->getCount(*table_it); @@ -138,7 +138,7 @@ Bootstrap::processWays() { std::ref(ways), }; std::cout << "\r" << "Processing " << *table_it << ": " << count << "/" << total << " (" << percentage << "%)"; - + boost::asio::post(pool, boost::bind(&Bootstrap::threadBootstrapWayTask, this, wayTask)); } @@ -157,9 +157,9 @@ Bootstrap::processWays() { } -void +void Bootstrap::processNodes() { - + std::cout << "Processing nodes ... "; long int total = queryraw->getCount("nodes"); long int count = 0; @@ -205,9 +205,9 @@ Bootstrap::processNodes() { } -void +void Bootstrap::processRelations() { - + std::cout << "Processing relations ... "; long int total = queryraw->getCount("relations"); long int count = 0; diff --git a/src/data/pq.cc b/src/data/pq.cc index 9dbe5435..ef7957f0 100644 --- a/src/data/pq.cc +++ b/src/data/pq.cc @@ -177,7 +177,7 @@ Pq::escapedString(const std::string &s) return sdb->esc(newstr); } -std::string +std::string Pq::escapedJSON(const std::string &s) { std::ostringstream o; for (auto c = s.cbegin(); c != s.cend(); c++) { diff --git a/src/osm/osmchange.cc b/src/osm/osmchange.cc index 9598716d..f6b7df24 100644 --- a/src/osm/osmchange.cc +++ b/src/osm/osmchange.cc @@ -425,7 +425,7 @@ OsmChange::dump(void) } } std::cerr << "Final timestamp: " << to_simple_string(final_entry) << std::endl; - + } void diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc index 77d91edf..5cb53ba2 100644 --- a/src/raw/queryraw.cc +++ b/src/raw/queryraw.cc @@ -225,7 +225,7 @@ QueryRaw::applyChange(const OsmWay &way) const ss << std::setprecision(12) << boost::geometry::wkt(way.linestring); } std::string geostring = ss.str(); - + if (way.refs.size() > 2 && (way.action == osmobjects::create || way.action == osmobjects::modify)) { if ((way.refs.front() != way.refs.back() && way.refs.size() == boost::geometry::num_points(way.linestring)) || @@ -306,7 +306,7 @@ QueryRaw::applyChange(const OsmRelation &relation) const // TODO: support for both multipolygon & multilinestring // ss << std::setprecision(12) << boost::geometry::wkt(relation.multilinestring); std::string geostring = ss.str(); - + if (relation.action == osmobjects::create || relation.action == osmobjects::modify) { query = "INSERT INTO relations as r (osm_id, tags, refs, geom, timestamp, version, \"user\", uid, changeset) VALUES("; @@ -568,7 +568,6 @@ void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const } else { osmchanges->waycache.insert(std::make_pair(way->id, std::make_shared(*way))); } - } } } @@ -657,8 +656,7 @@ void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const } } } - } - + } } void @@ -838,7 +836,7 @@ QueryRaw::getWaysFromDBWithoutRefs(long lastid, int pageSize, const std::string } else { waysQuery += ", tags FROM " + tableName + " order by osm_id desc limit " + std::to_string(pageSize) + ";"; } - + auto ways_result = dbconn->query(waysQuery); // Fill vector of OsmWay objects auto ways = std::make_shared>(); diff --git a/src/replicator/threads.cc b/src/replicator/threads.cc index 3ccdc8b0..9815e0bf 100644 --- a/src/replicator/threads.cc +++ b/src/replicator/threads.cc @@ -451,7 +451,7 @@ threadOsmChange(OsmChangeTask osmChangeTask) try { osmchanges->nodecache.clear(); - osmchanges->readXML(changes_xml); + osmchanges->readXML(changes_xml); if (osmchanges->changes.size() > 0) { task.timestamp = osmchanges->changes.back()->final_entry; log_debug("OsmChange final_entry: %1%", task.timestamp); @@ -496,7 +496,7 @@ threadOsmChange(OsmChangeTask osmChangeTask) auto removed_ways = std::make_shared>(); auto removed_relations = std::make_shared>(); auto validation_removals = std::make_shared>(); - + // Raw data and validation if (!config->disable_validation || !config->disable_raw) { for (auto it = std::begin(osmchanges->changes); it != std::end(osmchanges->changes); ++it) { diff --git a/src/stats/querystats.cc b/src/stats/querystats.cc index 30829923..437c53e0 100644 --- a/src/stats/querystats.cc +++ b/src/stats/querystats.cc @@ -271,7 +271,7 @@ QueryStats::applyChange(const changesets::ChangeSet &change) const } query += ", bbox=" + bbox.substr(2) + ")'));"; - + return query; } diff --git a/src/testsuite/libunderpass.all/geo-test.cc b/src/testsuite/libunderpass.all/geo-test.cc index fc63ccc9..b90f1f21 100644 --- a/src/testsuite/libunderpass.all/geo-test.cc +++ b/src/testsuite/libunderpass.all/geo-test.cc @@ -49,7 +49,7 @@ main(int argc, char* argv[]) dbglogfile.setWriteDisk(true); dbglogfile.setLogFilename("geo-test.log"); dbglogfile.setVerbosity(3); - + TestGU tgu; std::string test_data_dir(DATADIR); @@ -59,15 +59,15 @@ main(int argc, char* argv[]) } else { runtest.fail("Read file with bad relative path"); return 1; - } + } if (tgu.readFile("/etc/underpass/priority.geojson")) { runtest.pass("Read file with absolute path"); } else { runtest.fail("Read file with absolute path"); return 1; - } - + } + }; // local Variables: diff --git a/src/testsuite/libunderpass.all/planetreplicator-test.cc b/src/testsuite/libunderpass.all/planetreplicator-test.cc index 7372c587..8997eb87 100644 --- a/src/testsuite/libunderpass.all/planetreplicator-test.cc +++ b/src/testsuite/libunderpass.all/planetreplicator-test.cc @@ -55,7 +55,7 @@ void testPath(underpassconfig::UnderpassConfig config) { TestCO change; if (boost::filesystem::exists(osmchange->filespec)) { change.readChanges(osmchange->filespec); - } else { + } else { TestPlanet planet; auto data = planet.downloadFile(osmchange->getURL()).data; auto xml = planet.processData(osmchange->filespec, *data); @@ -128,7 +128,7 @@ main(int argc, char *argv[]) { } } } - + } // local Variables: diff --git a/src/testsuite/libunderpass.all/under-test.cc b/src/testsuite/libunderpass.all/under-test.cc index e063a17a..84ab26d2 100644 --- a/src/testsuite/libunderpass.all/under-test.cc +++ b/src/testsuite/libunderpass.all/under-test.cc @@ -46,9 +46,9 @@ int main(int argc, char* argv[]) { std::string basedir = DATADIR; - + Underpass under("underpass"); - + if (under.sdb->is_open()) { runtest.pass("Underpass::connect"); } else { diff --git a/src/testsuite/libunderpass.all/yaml-test.cc b/src/testsuite/libunderpass.all/yaml-test.cc index 4538daf4..8eab7fd2 100644 --- a/src/testsuite/libunderpass.all/yaml-test.cc +++ b/src/testsuite/libunderpass.all/yaml-test.cc @@ -46,7 +46,7 @@ main(int argc, char *argv[]) std::string filespec = DATADIR; filespec += "/testsuite/libunderpass.all/test.yaml"; yaml.read(filespec); - + if (yaml.get("tags").children.size() > 0) { runtest.pass("Yaml::get().children"); } else { diff --git a/src/underpass.cc b/src/underpass.cc index 63e25aa3..ee74c60a 100644 --- a/src/underpass.cc +++ b/src/underpass.cc @@ -182,7 +182,7 @@ main(int argc, char *argv[]) } else { config.concurrency = std::thread::hardware_concurrency(); } - + if (vm.count("timestamp") || vm.count("url")) { // Planet server if (vm.count("planet")) { @@ -335,7 +335,7 @@ main(int argc, char *argv[]) exit(0); } - + if (vm.count("bootstrap")){ std::thread bootstrapThread; std::cout << "Starting bootstrapping process ..." << std::endl; diff --git a/src/utils/geoutil.cc b/src/utils/geoutil.cc index 30b240fb..a6b3d526 100644 --- a/src/utils/geoutil.cc +++ b/src/utils/geoutil.cc @@ -60,7 +60,7 @@ GeoUtil::readFile(const std::string &filespec) if (!std::filesystem::exists(boundary_file)) { log_error("File not found: %1%", boundary_file); return false; - } + } log_debug("Opening geo data file: %1%", boundary_file); std::string foo = boundary_file.string(); GDALDataset *poDS = (GDALDataset *)GDALOpenEx(foo.c_str(), GDAL_OF_VECTOR, NULL, NULL, NULL); diff --git a/src/validate/geospatial.cc b/src/validate/geospatial.cc index 7a216679..81a2f992 100644 --- a/src/validate/geospatial.cc +++ b/src/validate/geospatial.cc @@ -56,7 +56,7 @@ Geospatial::checkWay(const osmobjects::OsmWay &way, const std::string &type, yam } auto config = tests.get("config"); - bool check_badgeom = config.get_value("badgeom") == "yes"; + bool check_badgeom = config.get_value("badgeom") == "yes"; // bool check_overlapping = config.get_value("overlapping") == "yes"; // bool check_duplicate = config.get_value("duplicate") == "yes"; @@ -96,7 +96,7 @@ Geospatial::checkWay(const osmobjects::OsmWay &way, const std::string &type, yam return status; } -bool +bool Geospatial::overlaps(const std::list> &allways, osmobjects::OsmWay &way) { #ifdef TIMING_DEBUG_X boost::timer::auto_cpu_timer timer("validate::overlaps: took %w seconds\n"); @@ -116,7 +116,7 @@ Geospatial::overlaps(const std::list> &allwa return false; } -bool +bool Geospatial::duplicate(const std::list> &allways, osmobjects::OsmWay &way) { #ifdef TIMING_DEBUG_X boost::timer::auto_cpu_timer timer("validate::duplicate: took %w seconds\n"); @@ -143,7 +143,7 @@ Geospatial::duplicate(const std::list> &allw return false; } -bool +bool Geospatial::unsquared( const linestring_t &way, double min_angle, diff --git a/src/validate/semantic.cc b/src/validate/semantic.cc index 91f8581c..12504249 100644 --- a/src/validate/semantic.cc +++ b/src/validate/semantic.cc @@ -183,7 +183,7 @@ Semantic::checkWay(const osmobjects::OsmWay &way, const std::string &type, yaml: yaml::Node required_tags; if (check_incomplete) { required_tags = tests.get("required_tags"); - } + } // List of valid tags to be validated yaml::Node tags; @@ -237,7 +237,7 @@ Semantic::checkRelation(const osmobjects::OsmRelation &relation, const std::stri yaml::Node required_tags; if (check_incomplete) { required_tags = tests.get("required_tags"); - } + } // List of valid tags to be validated yaml::Node tags; diff --git a/src/wrappers/python.cc b/src/wrappers/python.cc index 7bc666d3..eff00f57 100644 --- a/src/wrappers/python.cc +++ b/src/wrappers/python.cc @@ -108,7 +108,7 @@ std::string dumpJSON(ValidateStatus& self) { BOOST_PYTHON_MODULE(underpass) { - // + // // using namespace defaultvalidation; // class_("Validate") // // .def("checkTag", &DefaultValidation::checkTag)