Merge branch 'master' into feature/relationsProcessing

Makefile.am

@@ -29,6 +29,7 @@ CLEANFILES = \
 
 DISTCLEANFILES = docs/html docs/latex .lastmod
 MAINTAINERCLEANFILES = revno.h .lastmod
+ETCDIR = $(subst lib,etc,$(pkglibdir))
 
 SUBDIRS = \
 	src/validate
@@ -102,6 +103,7 @@ endif
 AM_CPPFLAGS = \
 	-DPKGLIBDIR=\"$(pkglibdir)\" \
 	-DSRCDIR=\"$(srcdir)\" \
+	-DETCDIR=\"$(ETCDIR)\" \
 	-DBOOST_LOCALE_HIDE_AUTO_PTR
 # Optionally timing can be turned on, which produces volumes of output which adds too much
 # clutter to the output. This should only be enabled when doing performance tuning.
@@ -119,7 +121,7 @@ EXTRA_DIST = \
 	dist/debian \
 	dist/redhat \
 	utils
-	
+
 
 DEJATOOL = libunderpass
 
@@ -151,17 +153,14 @@ apidoc:
 endif
 
 install-data-hook:
-	$(MKDIR_P) $(DESTDIR)/$(pkglibdir)
-	$(MKDIR_P) /etc/underpass
-	cp -rvp $(srcdir)/config/priority.geojson /etc/underpass/
-	cp -rvp $(srcdir)/config/replicator /etc/underpass/
-	cp -rvp $(srcdir)/config/validate /etc/underpass/
-	cp -rvp $(srcdir)/config/stats /etc/underpass/
-	cp -rvp $(srcdir)/config/default.yaml /etc/underpass/
-	$(MKDIR_P) $(DESTDIR)/$(docdir)
-	cp -rvp $(srcdir)/docs/*.md $(DESTDIR)/$(docdir)
+	$(MKDIR_P) $(DESTDIR)$(ETCDIR)
+	cp -rvp $(srcdir)/config/priority.geojson  $(DESTDIR)$(ETCDIR)/
+	cp -rvp $(srcdir)/config/replicator  $(DESTDIR)$(ETCDIR)/
+	cp -rvp $(srcdir)/config/stats  $(DESTDIR)$(ETCDIR)/
+	cp -rvp $(srcdir)/config/default.yaml  $(DESTDIR)$(ETCDIR)/
 	cp -rvp $(srcdir)/setup/service $(DESTDIR)/$(pkglibdir)
-	cp -rvp $(srcdir)/setup/service/underpass.service /etc/systemd/system/
+	$(MKDIR_P) $(DESTDIR)$(pkglibdir)/../system
+	cp -rvp $(srcdir)/setup/service/underpass.service  $(DESTDIR)$(pkglibdir)/../system/
 
 dist-hook: apidoc
 	$(MKDIR_P) $(DESTDIR)/$(docdir)
@@ -180,20 +179,20 @@ PCHFLAGS = \
         -I$(top_builddir) \
         -I$(top_srcdir)/src \
         -I$(top_srcdir)/src/stats \
-		-I$(top_srcdir)/src/validate \
+	-I$(top_srcdir)/src/validate \
         -I$(top_srcdir)/src/data
 
 PCHHEADERS = \
         $(top_srcdir)/src/replicator/planetreplicator.hh \
         $(top_srcdir)/src/stats/querystats.hh \
-		$(top_srcdir)/src/raw/queryraw.hh \
+	$(top_srcdir)/src/raw/queryraw.hh \
         $(top_srcdir)/src/replicator/replication.hh \
         $(top_srcdir)/src/osm/osmchange.hh \
         $(top_srcdir)/src/osm/changeset.hh \
         $(top_srcdir)/src/osm/osmobjects.hh \
         $(top_srcdir)/src/validate/validate.hh \
         $(top_srcdir)/src/utils/geoutil.hh \
-		$(top_srcdir)/src/utils/geo.hh \
+	$(top_srcdir)/src/utils/geo.hh \
         $(top_srcdir)/src/replicator/threads.hh
 
 PCHOTHER = \

configure.ac

@@ -163,9 +163,8 @@ fi
 dnl range-v3
 CPPFLAGS+=" $(pkg-config --cflags libpqxx)"
 CPPFLAGS+=" $(pkg-config --cflags gumbo)"
-CPPFLAGS+=" $(pkg-config --cflags gdal)"
 CPPFLAGS+=" $(pkg-config --cflags ompi)"
-CPPFLAGS+=" $(pkg-config --cflags python3)"
+CPPFLAGS+=" $(python3-config --cflags)"
 
 dnl This is a C++ project only
 AC_LANG_PUSH(C++)
@@ -183,9 +182,8 @@ LIBS+=" $(pkg-config --libs libpqxx)"
 LIBS+=" $(pkg-config --libs openssl)"
 LIBS+=" $(pkg-config --libs gumbo)"
 LIBS+=" $(pkg-config --libs zlib)"
-LIBS+=" $(pkg-config --libs gdal)"
 LIBS+=" $(pkg-config --libs ompi)"
-LIBS+=" $(pkg-config --libs python3)"
+LIBS+=" $(python3-config --libs)"
 if test x"${build_libxml}" = x"yes"; then
 dnl Debian Buster and Ubuntu Focal ship 2.6, Fedora ships 3,.0
 dnl The version in Focal is broken, so either build libxml++-3.0
@@ -209,16 +207,16 @@ LIBS+=" -lpthread -ldl"
 
 dnl 1.69 or newer
 AX_BOOST_BASE(1.69)
+AX_BOOST_IOSTREAMS
 AX_BOOST_SYSTEM
 AX_BOOST_SERIALIZATION
-AX_BOOST_LOG
-AX_BOOST_PROGRAM_OPTIONS
 AX_BOOST_DATE_TIME
 AX_BOOST_FILESYSTEM
-AX_BOOST_IOSTREAMS
 AX_BOOST_THREAD
 AX_BOOST_LOCALE
 AX_BOOST_TIMER
+AX_BOOST_LOG
+AX_BOOST_PROGRAM_OPTIONS
 dnl For some reason this test works on Fedora, and fails on Debian. For now
 dnl hardcode it.
 dnl AX_BOOST_REGEX
@@ -227,14 +225,17 @@ LIBS+=" -lboost_regex"
 dnl LIBS += "${BOOST_DATE_TIME} ${BOOST_SYSTEM} ${BOOST_FILESYSTEM} ${BOOST_LOG_LIB}"
 AC_LANG_POP(C++)
 
+CPPFLAGS+=" $(pkg-config --cflags gdal)"
+LIBS+=" $(pkg-config --libs gdal)"
+
 AM_PATH_PYTHON([3])
 
 AC_ARG_ENABLE([python],
   AS_HELP_STRING([--enable-python],[Enable Python binding]), [], [enable_python=yes])
 
 if test x"${enable_python}" = x"yes"; then
   LIBS+=" $(python3-config --libs)"
-  LIBS+=" $(pkg-config --libs python3-embed)"
+  LIBS+=" $(python3-config --embed)"
   CPPFLAGS+=" $(python3-config --cflags)"
   AX_BOOST_PYTHON
   AC_DEFINE(USE_PYTHON, [1], [Enable Python binding])
@@ -292,7 +293,7 @@ AC_OUTPUT
 if test x"${build_libxml}" = x"yes"; then
    echo "Using libxml for XML parsing"
 else
-   echo "Using RaqpidXML for XML parsing, which is used by boost::parse_tree"
+   echo "Using RapidXML for XML parsing, which is used by boost::parse_tree"
 fi
 # Local Variables:
 # c-basic-offset: 2

docs/get-started/Setup.md

@@ -17,15 +17,17 @@ postgres=# CREATE USER underpass WITH PASSWORD 'your_password';
 postgres=# CREATE DATABASE underpass;
 postgres=# GRANT ALL PRIVILEGES ON DATABASE "underpass" to underpass;
 postgres=# ALTER ROLE underpass SUPERUSER;
+postgres=# exit
 exit
-psql postgresql://underpass:your_password@localhost:5432/underpass < setup/underpass.sql
+psql postgresql://underpass:your_password@localhost:5432/underpass < setup/db/underpass.sql
 ```
 
 #### Requirements
 
 ```
 sudo apt install python3-pip -y
 sudo apt install python3.11-venv
+python3 -m venv ~/venv
 source ~/venv/bin/activate
 pip install fiona
 pip install shapely
@@ -36,7 +38,9 @@ apt install osm2pgsql
 
 Go to the `setup` directory and run the boostrap script:
 
+```
 cd utils
+chmod +x bootstrap.sh
 ./bootstrap.sh -r south-america -c uruguay
 ```
 

setup/bootstrap.sh

@@ -119,7 +119,7 @@ then
         then
             docker exec -w /code/build -t underpass underpass --bootstrap
         else
-            underpass --bootstrap
+            underpass --bootstrap  --server=$USER:$PASS@$HOST:$PORT/$DB
         fi
         echo "Done."
         echo " "

src/bootstrap/bootstrap.cc

@@ -271,7 +271,7 @@ Bootstrap::threadBootstrapWayTask(WayTask wayTask)
     auto wayval = std::make_shared<std::vector<std::shared_ptr<ValidateStatus>>>();
 
     // Proccesing ways
-    for (int i = taskIndex * page_size; i < (taskIndex + 1) * page_size; ++i) {
+    for (size_t i = taskIndex * page_size; i < (taskIndex + 1) * page_size; ++i) {
         if (i < ways->size()) {
             auto way = ways->at(i);
             wayval->push_back(validator->checkWay(way, "building"));
@@ -309,7 +309,7 @@ Bootstrap::threadBootstrapNodeTask(NodeTask nodeTask)
 
     // Proccesing nodes
     std::vector<std::string> node_tests = {"building", "natural", "place", "waterway"};
-    for (int i = taskIndex * page_size; i < (taskIndex + 1) * page_size; ++i) {
+    for (size_t i = taskIndex * page_size; i < (taskIndex + 1) * page_size; ++i) {
         if (i < nodes->size()) {
             auto node = nodes->at(i);
             for (auto test_it = std::begin(node_tests); test_it != std::end(node_tests); ++test_it) {
@@ -344,7 +344,7 @@ Bootstrap::threadBootstrapRelationTask(RelationTask relationTask)
     auto relationval = std::make_shared<std::vector<std::shared_ptr<ValidateStatus>>>();
 
     // Proccesing relations
-    for (int i = taskIndex * page_size; i < (taskIndex + 1) * page_size; ++i) {
+    for (size_t i = taskIndex * page_size; i < (taskIndex + 1) * page_size; ++i) {
         if (i < relations->size()) {
             auto relation = relations->at(i);
             // relationval->push_back(validator->checkRelation(way, "building"));
@@ -362,4 +362,4 @@ Bootstrap::threadBootstrapRelationTask(RelationTask relationTask)
 
 }
 
-}
+}

src/replicator/planetreplicator.cc

@@ -102,7 +102,8 @@ PlanetReplicator::PlanetReplicator(void) {};
 std::shared_ptr<RemoteURL> PlanetReplicator::findRemotePath(const underpassconfig::UnderpassConfig &config, ptime time) {
     yaml::Yaml yaml;
 
-    std::string rep_file = "/etc/underpass/replicator/planetreplicator.yaml";
+    std::string rep_file = ETCDIR;
+    rep_file += "/replicator/planetreplicator.yaml";
     yaml.read(rep_file);
     std::map<int, ptime> hashes;
 

src/stats/statsconfig.cc

@@ -51,7 +51,8 @@ namespace statsconfig {
 
     StatsConfig::StatsConfig() {
         if (path.empty()) {
-            path = "/etc/underpass/stats/statistics.yaml";
+            path = ETCDIR;
+            path += "/stats/statistics.yaml";
             if (!boost::filesystem::exists(path)) {
                 throw std::runtime_error("Statistics file not found: " + path);
             }

src/testsuite/libunderpass.all/Makefile.am

@@ -19,6 +19,8 @@
 
 AUTOMAKE_OPTIONS = dejagnu
 
+ETCDIR = $(subst lib,etc,$(pkglibdir))
+
 check_PROGRAMS = \
 	pq-test \
 	change-test \
@@ -55,6 +57,7 @@ AM_CXXFLAGS = \
         -DPKGLIBDIR=\"$(pkglibdir)\" \
         -DTOPSRCDIR=\"$(TOPSRC)\" \
         -DSRCDIR=\"$(srcdir)\" \
+	 -DETCDIR=\"$(ETCDIR)\" \
 	-DBOOST_LOCALE_HIDE_AUTO_PTR \
 	-Wno-deprecated-declarations
 

src/testsuite/libunderpass.all/geo-test.cc

@@ -61,7 +61,9 @@ main(int argc, char* argv[])
         return 1;
     }
 
-    if (tgu.readFile("/etc/underpass/priority.geojson")) {
+    std::string filespec = ETCDIR;
+    filespec += "/priority.geojson";
+    if (tgu.readFile(filespec)) {
         runtest.pass("Read file with absolute path");
     } else {
         runtest.fail("Read file with absolute path");

src/underpass.cc

@@ -85,7 +85,8 @@ main(int argc, char *argv[])
 {
 
     std::string datadir = "replication/";
-    std::string boundary = "/etc/underpass/priority.geojson";
+    std::string boundary = ETCDIR;
+    boundary += "/priority.geojson";
 
     UnderpassConfig config;
 

src/underpassconfig.hh

@@ -103,9 +103,11 @@ struct UnderpassConfig {
     UnderpassConfig()
     {
 
-        if (std::filesystem::exists("/etc/underpass/default.yaml")) {
+        std::string filespec = ETCDIR;
+        filespec += "default.yaml";
+        if (std::filesystem::exists(filespec)) {
             yaml::Yaml yaml;
-            yaml.read("/etc/underpass/default.yaml");
+            yaml.read(filespec);
             auto yamlConfig = yaml.get("config");
             if (yaml.contains_key("underpass_db_url")) {
                 underpass_db_url = yamlConfig.get_value("underpass_db_url");

src/validate/Makefile.am

@@ -19,6 +19,7 @@
 # AUTOMAKE_OPTIONS =  -Wno-portability dejagnu subdir-objects
 
 lib_LTLIBRARIES = libunderpass.la
+ETCDIR = $(subst lib,etc,$(pkglibdir))
 
 BOOST_LIBS = $(BOOST_DATE_TIME_LIB) \
 	$(BOOST_SYSTEM_LIB) \
@@ -47,6 +48,7 @@ AM_CPPFLAGS = \
 	-I$(top_srcdir)/src \
 	-DPKGLIBDIR=\"$(pkglibdir)\" \
 	-DSRCDIR=\"$(srcdir)\" \
+	-DETCDIR=\"$(ETCDIR)\" \
 	-DBOOST_LOCALE_HIDE_AUTO_PTR
 
 install-data-hook:

src/validate/semantic.cc

@@ -137,7 +137,7 @@ Semantic::checkNode(const osmobjects::OsmNode &node, const std::string &type, ya
     auto required_tags = tests.get("required_tags");
 
     std::string key;
-    int tagexists = 0;
+    size_t tagexists = 0;
     status->center = node.point;
 
     if (node.tags.count(type)) {
@@ -196,7 +196,7 @@ Semantic::checkWay(const osmobjects::OsmWay &way, const std::string &type, yaml:
         return status;
     }
 
-    int tagexists = 0;
+    size_t tagexists = 0;
     if (way.tags.count(type)) {
         for (auto vit = std::begin(way.tags); vit != std::end(way.tags); ++vit) {
             if (check_badvalue) {
@@ -250,7 +250,7 @@ Semantic::checkRelation(const osmobjects::OsmRelation &relation, const std::stri
         return status;
     }
 
-    int tagexists = 0;
+    size_t tagexists = 0;
     if (relation.tags.count(type)) {
         for (auto vit = std::begin(relation.tags); vit != std::end(relation.tags); ++vit) {
             if (check_badvalue) {

src/validate/validate.hh

@@ -179,7 +179,8 @@ class ValidateStatus {
 class BOOST_SYMBOL_VISIBLE Validate {
   public:
     Validate(void) {
-        std::string path = "/etc/underpass/validate";
+        std::string path = ETCDIR;
+        path += "/validate";
         loadConfig(path);
     }