From 35958c2ee5f865e4d9569d8dec9828e4eb7b9ded Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Mon, 23 Oct 2023 09:40:27 -0400 Subject: [PATCH] Fix for underpass command with no args. Add support for bootstrap with no refs --- config/replicator/planetreplicator.yaml | 8 +++-- src/bootstrap/bootstrap.cc | 26 +++++++++----- src/bootstrap/bootstrap.hh | 2 +- src/osm/osmobjects.hh | 7 +--- src/raw/queryraw.cc | 41 +++++++++++++++++++++- src/raw/queryraw.hh | 1 + src/underpass.cc | 46 ++++++++++++++----------- src/underpassconfig.hh | 1 + 8 files changed, 93 insertions(+), 39 deletions(-) diff --git a/config/replicator/planetreplicator.yaml b/config/replicator/planetreplicator.yaml index d9aa1617..a9ee4c65 100644 --- a/config/replicator/planetreplicator.yaml +++ b/config/replicator/planetreplicator.yaml @@ -22,6 +22,8 @@ minute: 2021-04-15 10:01:23 5000000: 2022-04-03 16:25:35 + 5500000: + 2023-03-22 21:57:39 changeset: 0: 2012-10-28 19:36:01 @@ -42,6 +44,6 @@ changeset: 4000000: 2020-06-28 23:26:01 4500000: - 2021-06-17 03:53 - 5000000: - 2022-05-30 17:50:02 + 2021-06-17 03:53:00 + 5500000: + 2023-05-19 06:23:54 diff --git a/src/bootstrap/bootstrap.cc b/src/bootstrap/bootstrap.cc index 09ded93a..1a8f6e3d 100644 --- a/src/bootstrap/bootstrap.cc +++ b/src/bootstrap/bootstrap.cc @@ -42,12 +42,14 @@ namespace bootstrap { void startProcessingWays(const underpassconfig::UnderpassConfig &config) { + std::cout << "Connecting to the database (" << config.underpass_db_url << ") ..." << std::endl; auto db = std::make_shared(); if (!db->connect(config.underpass_db_url)) { log_error("Could not connect to Underpass DB, aborting monitoring thread!"); return; } + std::cout << "Loading plugins ... " << std::endl; std::string plugins; if (boost::filesystem::exists("src/validate/.libs")) { plugins = "src/validate/.libs"; @@ -73,12 +75,15 @@ void startProcessingWays(const underpassconfig::UnderpassConfig &config) { }; for (auto table_it = tables.begin(); table_it != tables.end(); ++table_it) { + std::cout << "Counting geometries ... " << std::endl; int total = queryraw->getWaysCount(*table_it); + std::cout << "Total ways:" << total << std::endl; if (total > 0) { int count = 0; long lastid = 0; while (count < total) { int percentage = (count * 100) / total; + std::cout << "\r" << "Processing " << *table_it << ": " << count << "/" << total << " (" << percentage << "%)"; auto task = std::make_shared(); WayTask wayTask; wayTask.plugin = validator; @@ -87,11 +92,10 @@ void startProcessingWays(const underpassconfig::UnderpassConfig &config) { wayTask.task = task; wayTask.lastid = lastid; - processWays(wayTask, *table_it); + processWays(wayTask, *table_it, config); db->query(task->query); lastid = wayTask.lastid; count += wayTask.processed; - std::cout << "\r" << "Processing " << *table_it << ": " << count << "/" << total << " (" << percentage << "%)"; } } } @@ -100,7 +104,7 @@ void startProcessingWays(const underpassconfig::UnderpassConfig &config) { // This thread get started for every page of way void -processWays(WayTask &wayTask, const std::string &tableName) +processWays(WayTask &wayTask, const std::string &tableName, const underpassconfig::UnderpassConfig &config) { #ifdef TIMING_DEBUG boost::timer::auto_cpu_timer timer("bootstrap::processWays(wayTask): took %w seconds\n"); @@ -112,14 +116,18 @@ processWays(WayTask &wayTask, const std::string &tableName) auto queryraw = wayTask.queryraw; auto lastid = wayTask.lastid; - auto ways = queryraw->getWaysFromDB(lastid, tableName); + auto ways = std::make_shared>(); + if (!config.norefs) { + ways = queryraw->getWaysFromDB(lastid, tableName); + } else { + ways = queryraw->getWaysFromDBWithoutRefs(lastid, tableName); + } wayTask.processed = ways->size(); if (wayTask.processed > 0) { // Proccesing ways for (auto way = ways->begin(); way != ways->end(); ++way) { - // If it's closed polygon - if (way->refs.front() == way->refs.back()) { + if (way->isClosed()) { log_debug("Way Id: %1%", way->id); // Bad geometry @@ -135,8 +143,10 @@ processWays(WayTask &wayTask, const std::string &tableName) } // Fill the way_refs table - for (auto ref = way->refs.begin(); ref != way->refs.end(); ++ref) { - task->query += "INSERT INTO way_refs (way_id, node_id) VALUES (" + std::to_string(way->id) + "," + std::to_string(*ref) + "); "; + if (!config.norefs) { + for (auto ref = way->refs.begin(); ref != way->refs.end(); ++ref) { + task->query += "INSERT INTO way_refs (way_id, node_id) VALUES (" + std::to_string(way->id) + "," + std::to_string(*ref) + "); "; + } } } wayTask.lastid = ways->back().id; diff --git a/src/bootstrap/bootstrap.hh b/src/bootstrap/bootstrap.hh index 9e4aa966..efe52b10 100644 --- a/src/bootstrap/bootstrap.hh +++ b/src/bootstrap/bootstrap.hh @@ -47,6 +47,6 @@ struct WayTask { void startProcessingWays(const underpassconfig::UnderpassConfig &config); // This thread get started for every page of way -void processWays(WayTask &wayTask, const std::string &tableName); +void processWays(WayTask &wayTask, const std::string &tableName, const underpassconfig::UnderpassConfig &config); } \ No newline at end of file diff --git a/src/osm/osmobjects.hh b/src/osm/osmobjects.hh index 5f5db82a..e014cb80 100644 --- a/src/osm/osmobjects.hh +++ b/src/osm/osmobjects.hh @@ -169,12 +169,7 @@ class OsmWay : public OsmObject { /// is a linestring bool isClosed(void) { - if (refs.size() > 0) { - if (refs[0] == refs[refs.size() - 1]) { - return true; - } - } - return false; + return !linestring.empty() && boost::geometry::equals(linestring.back(), linestring.front()); }; /// Return the number of nodes in this way int numPoints(void) { return boost::geometry::num_points(linestring); }; diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc index dc2daa2b..20b79bb6 100644 --- a/src/raw/queryraw.cc +++ b/src/raw/queryraw.cc @@ -436,7 +436,7 @@ QueryRaw::getWaysFromDB(int lastid, const std::string &tableName) { waysQuery = "SELECT osm_id, refs, ST_AsText(geom, 4326)"; } waysQuery += ", version, tags FROM " + tableName + " where osm_id > " + std::to_string(lastid) + " order by osm_id asc limit 500;"; - + auto ways_result = dbconn->query(waysQuery); // Fill vector of OsmWay objects auto ways = std::make_shared>(); @@ -469,6 +469,45 @@ QueryRaw::getWaysFromDB(int lastid, const std::string &tableName) { return ways; } +std::shared_ptr> +QueryRaw::getWaysFromDBWithoutRefs(int lastid, const std::string &tableName) { + std::string waysQuery; + if (tableName == QueryRaw::polyTable) { + waysQuery = "SELECT osm_id, ST_AsText(ST_ExteriorRing(geom), 4326)"; + } else { + waysQuery = "SELECT osm_id, ST_AsText(geom, 4326)"; + } + waysQuery += ", version, tags FROM " + tableName + " where osm_id > " + std::to_string(lastid) + " order by osm_id asc limit 500;"; + + auto ways_result = dbconn->query(waysQuery); + // Fill vector of OsmWay objects + auto ways = std::make_shared>(); + for (auto way_it = ways_result.begin(); way_it != ways_result.end(); ++way_it) { + OsmWay way; + way.id = (*way_it)[0].as(); + + std::string poly = (*way_it)[1].as(); + boost::geometry::read_wkt(poly, way.linestring); + + if (tableName == QueryRaw::polyTable) { + way.polygon = { {std::begin(way.linestring), std::end(way.linestring)} }; + } + way.version = (*way_it)[2].as(); + auto tags = (*way_it)[3]; + if (!tags.is_null()) { + auto tags = parseTagsString((*way_it)[3].as()); + for (auto const& [key, val] : tags) + { + way.addTag(key, val); + } + } + ways->push_back(way); + + } + + return ways; +} + } // namespace queryraw // local Variables: diff --git a/src/raw/queryraw.hh b/src/raw/queryraw.hh index b47559f6..d835948e 100644 --- a/src/raw/queryraw.hh +++ b/src/raw/queryraw.hh @@ -77,6 +77,7 @@ class QueryRaw { int getWaysCount(const std::string &tableName); // Get ways by page std::shared_ptr> getWaysFromDB(int page, const std::string &tableName); + std::shared_ptr> getWaysFromDBWithoutRefs(int lastid, const std::string &tableName); }; diff --git a/src/underpass.cc b/src/underpass.cc index 2de1e3dd..07c7a9e3 100644 --- a/src/underpass.cc +++ b/src/underpass.cc @@ -37,7 +37,6 @@ #include #include -#include "underpassconfig.hh" #include "replicator/planetreplicator.hh" #include "boost/date_time/posix_time/posix_time.hpp" #include @@ -96,9 +95,9 @@ main(int argc, char *argv[]) opts::positional_options_description p; opts::variables_map vm; + opts::options_description desc("Allowed options"); try { - opts::options_description desc("Allowed options"); // clang-format off desc.add_options() ("help,h", "display help") @@ -124,33 +123,22 @@ main(int argc, char *argv[]) ("disable-stats", "Disable statistics") ("disable-validation", "Disable validation") ("disable-raw", "Disable raw OSM data") + ("norefs", "Disable refs (useful for non OSM data)") ("bootstrap", "Bootstrap data tables"); // clang-format on opts::store(opts::command_line_parser(argc, argv).options(desc).positional(p).run(), vm); opts::notify(vm); - if (vm.count("help")) { - - std::cout << "Usage: options_description [options]" << std::endl; - std::cout << desc << std::endl; - std::cout << "A few configuration options can be set through the " - "environment," - << std::endl; - std::cout << "this is the current status of the configuration " - "options with" - << std::endl; - std::cout << "the environment variable names and their current " - "values (possibly defaults)." - << std::endl; - std::cout << config.dbConfigHelp() << std::endl; - return 0; - } } catch (std::exception &e) { std::cout << e.what() << std::endl; return 1; } + if (vm.count("norefs")) { + config.norefs = true; + } + // Logging logger::LogFile &dbglogfile = logger::LogFile::getDefaultInstance(); if (vm.count("verbose")) { @@ -185,7 +173,7 @@ main(int argc, char *argv[]) config.concurrency = std::thread::hardware_concurrency(); } - if (!vm.count("bootstrap")) { + if (vm.count("timestamp") || vm.count("url")) { // Planet server if (vm.count("planet")) { config.planet_server = vm["planet"].as(); @@ -330,15 +318,33 @@ main(int argc, char *argv[]) exit(0); - } else { + } + + if (vm.count("bootstrap")){ std::thread bootstrapThread; + std::cout << "Starting bootstrapping proccess ..." << std::endl; bootstrapThread = std::thread(bootstrap::startProcessingWays, std::ref(config)); log_info("Waiting..."); if (bootstrapThread.joinable()) { bootstrapThread.join(); } + exit(0); } + std::cout << "Usage: options_description [options]" << std::endl; + std::cout << desc << std::endl; + std::cout << "A few configuration options can be set through the " + "environment," + << std::endl; + std::cout << "this is the current status of the configuration " + "options with" + << std::endl; + std::cout << "the environment variable names and their current " + "values (possibly defaults)." + << std::endl; + // std::cout << config.dbConfigHelp() << std::endl; + exit(0); + } // Local Variables: diff --git a/src/underpassconfig.hh b/src/underpassconfig.hh index e88f230f..6ddaa2a4 100644 --- a/src/underpassconfig.hh +++ b/src/underpassconfig.hh @@ -157,6 +157,7 @@ struct UnderpassConfig { bool disable_validation = false; bool disable_stats = false; bool disable_raw = false; + bool norefs = false; /// /// \brief getPlanetServer returns either the command line supplied planet server