diff --git a/docs/osmstats.md b/docs/osmstats.md index 9d9219e4..d349dd2a 100644 --- a/docs/osmstats.md +++ b/docs/osmstats.md @@ -57,7 +57,7 @@ buildings_modified | This value is updated by counting the existing buildings mo pois_added | This value is updated by counting the POIs added by the user in the change file pois_modified | This value is updated by counting the existing POIs modified by the user in the change file editor | The editor used, and comes from the changeset -user_id | The user ID, comes from the changeset +uid | The user ID, comes from the changeset created_at | The timestamp this changeset was created, comes from the changeset closed_at | The timestamp this changeset was closed, comes from the changeset verified | Whether this data has been validated @@ -128,7 +128,7 @@ level | The badge level Keyword | Description --------|------------ -user_id | The OSM user ID +uid | The OSM user ID badge_id | The badge ID updated_at | The timestamp of the user receiving this badge diff --git a/docs/statistics.md b/docs/statistics.md index e8a0a4e7..a7a2d59b 100644 --- a/docs/statistics.md +++ b/docs/statistics.md @@ -211,7 +211,7 @@ backend and the frontend, without having modify the database schema. An example query to count the total number of buildings added by the user **4321** for a Tasking Manager project **1234** would be this: > SELECT SUM(CAST(added::hstore->'building' AS DOUBLE precision)) FROM -changesets WHERE 'hotosm-project-1234' = ANY(hashtags) AND user_id=4321; +changesets WHERE 'hotosm-project-1234' = ANY(hashtags) AND uid=4321; The source is the satellite imagery used for remote mapping. @@ -221,7 +221,7 @@ Keyword | Description --------|------------ id | The ID of this changeset editor | The editor used for this changeset -user_id | The OSM User ID of the mapper +uid | The OSM User ID of the mapper created_at | The timestamp when this changes was uploaded closed_at | The timestamp when this uploaded change completed processing updated_at | The timestamp when this last had data updated diff --git a/python/dbapi/api/raw.py b/python/dbapi/api/raw.py index 421b62aa..37900385 100644 --- a/python/dbapi/api/raw.py +++ b/python/dbapi/api/raw.py @@ -19,116 +19,321 @@ from .db import UnderpassDB -RESULTS_PER_PAGE = 25 +RESULTS_PER_PAGE = 500 +RESULTS_PER_PAGE_LIST = 100 -class Raw: - def __init__(self, db): - self.underpassDB = db +def tagsQueryFilter(tagsQuery, table): + query = "" + tags = tagsQuery.split(",") + keyValue = tags[0].split("=") - def getPolygons( - self, + if len(keyValue) == 2: + query += "{0}.tags->>'{1}' ~* '^{2}'".format(table, keyValue[0], keyValue[1]) + else: + query += "{0}.tags->>'{1}' IS NOT NULL".format(table, keyValue[0]) + + for tag in tags[1:]: + keyValue = tag.split("=") + if len(keyValue) == 2: + query += "OR {0}.tags->>'{1}' ~* '^{2}'".format(table, keyValue[0], keyValue[1]) + else: + query += "OR {0}.tags->>'{1}' IS NOT NULL".format(table, keyValue[0]) + return query + +def getGeoType(table): + if table == "ways_poly": + return "Polygon" + elif table == "ways_line": + return "LineString" + return "Node" + +def geoFeaturesQuery( area = None, - key = None, - value = None, + tags = None, hashtag = None, - responseType = "json", - page = None - ): + dateTo = None, + dateFrom = None, + page = 0, + table = None): + + geoType = getGeoType(table) query = "with t_ways AS ( \ - SELECT raw_poly.osm_id as id, raw_poly.timestamp, geometry, tags, status FROM raw_poly \ - LEFT JOIN validation ON validation.osm_id = raw_poly.osm_id \ + SELECT '" + geoType + "' as type, " + table + ".osm_id as id, " + table + ".timestamp, geom as geometry, tags, status, hashtags, editor, created_at FROM " + table + " \ + LEFT JOIN validation ON validation.osm_id = " + table + ".osm_id \ + LEFT JOIN changesets c ON c.id = " + table + ".changeset \ WHERE \ - {0} {1} {2} {3} \ + {0} {1} {2} {3} {4} \ ), \ t_features AS ( \ SELECT jsonb_build_object( 'type', 'Feature', 'id', id, 'properties', to_jsonb(t_ways) \ - 'geometry' , 'geometry', ST_AsGeoJSON(geometry)::jsonb ) AS feature FROM t_ways \ ) SELECT jsonb_build_object( 'type', 'FeatureCollection', 'features', jsonb_agg(t_features.feature) ) \ as result FROM t_features;".format( - "ST_Intersects(\"geometry\", ST_GeomFromText('POLYGON(({0}))', 4326) )".format(area) if area else "1=1 ", - "and raw_poly.tags ? '{0}'".format(key) if key and not value else "", - "and raw_poly.tags->'{0}' ~* '^{1}'".format(key, value) if key and value else "", - "ORDER BY raw_poly.timestamp DESC LIMIT " + str(RESULTS_PER_PAGE) + " OFFSET {0}".format(page * RESULTS_PER_PAGE) if page else "", + "ST_Intersects(\"geom\", ST_GeomFromText('POLYGON(({0}))', 4326) )".format(area) if area else "1=1 ", + "AND (" + tagsQueryFilter(tags, table) + ")" if tags else "", + "AND " + table + ".changeset IN (SELECT c.id FROM changesets c where jsonb_path_exists(to_jsonb(hashtags), '$[*] ? (@ like_regex \"^{0}\")') GROUP BY C.id)".format(hashtag) if hashtag else "", + "AND created at >= {0} AND created_at <= {1}".format(dateFrom, dateTo) if dateFrom and dateTo else "", + "LIMIT " + str(RESULTS_PER_PAGE), ) - return self.underpassDB.run(query, responseType, True) + return query + +def listFeaturesQuery( + area = None, + tags = None, + hashtag = None, + page = 0, + dateFrom = None, + dateTo = None, + table = None): + + geoType = getGeoType(table) + if table == "nodes": + osmType = "node" + else: + osmType = "way" + + query = "with t_ways AS ( \ + SELECT '" + osmType + "' as type, '" + geoType + "' as geotype, " + table + ".osm_id as id, ST_X(ST_Centroid(geom)) as lat, ST_Y(ST_Centroid(geom)) as lon, " + table + ".timestamp, tags, status, created_at FROM " + table + " \ + LEFT JOIN validation ON validation.osm_id = " + table + ".osm_id \ + LEFT JOIN changesets c ON c.id = " + table + ".changeset \ + WHERE \ + {0} {1} {2} {3} {4} \ + ), t_features AS ( \ + SELECT to_jsonb(t_ways) as feature from t_ways \ + ) SELECT jsonb_agg(t_features.feature) as result FROM t_features;".format( + "ST_Intersects(\"geom\", ST_GeomFromText('POLYGON(({0}))', 4326) )".format(area) if area else "1=1 ", + "AND (" + tagsQueryFilter(tags, table) + ")" if tags else "", + "AND " + table + ".changeset IN (SELECT c.id FROM changesets c where jsonb_path_exists(to_jsonb(hashtags), '$[*] ? (@ like_regex \"^{0}\")') GROUP BY C.id)".format(hashtag) if hashtag else "", + "AND created_at >= '{0}' AND created_at <= '{1}'".format(dateFrom, dateTo) if (dateFrom and dateTo) else "", + "AND created_at IS NOT NULL ORDER BY created_at DESC LIMIT " + str(RESULTS_PER_PAGE_LIST) + (" OFFSET {0}".format(page * RESULTS_PER_PAGE_LIST) if page else ""), + ) + return query + +class Raw: + def __init__(self, db): + self.underpassDB = db + + def getPolygons( + self, + area = None, + tags = None, + hashtag = None, + responseType = "json", + dateFrom = None, + dateTo = None, + page = None + ): + return self.underpassDB.run(geoFeaturesQuery( + area, + tags, + hashtag, + page, + dateFrom, + dateTo, + "ways_poly" + ), responseType, True) + + def getLines( + self, + area = None, + tags = None, + hashtag = None, + responseType = "json", + dateFrom = None, + dateTo = None, + page = None + ): + return self.underpassDB.run(geoFeaturesQuery( + area, + tags, + hashtag, + page, + dateFrom, + dateTo, + "ways_line" + ), responseType, True) def getNodes( self, area = None, - key = None, - value = None, + tags = None, hashtag = None, - responseType = "json" + responseType = "json", + dateFrom = None, + dateTo = None, + page = None ): - query = "with t_nodes AS ( \ - SELECT raw_node.osm_id as id, geometry, tags, status FROM raw_node \ - LEFT JOIN validation ON validation.osm_id = raw_node.osm_id \ - WHERE \ - ST_Intersects(\"geometry\", \ - ST_GeomFromText('POLYGON(({0}))', 4326) \ - ) {1} {2} \ - ), \ - t_features AS ( \ - SELECT jsonb_build_object( 'type', 'Feature', 'id', id, 'properties', to_jsonb(t_nodes) \ - - 'geometry' - 'osm_id' , 'geometry', ST_AsGeoJSON(geometry)::jsonb ) AS feature FROM t_nodes \ - ) SELECT jsonb_build_object( 'type', 'FeatureCollection', 'features', jsonb_agg(t_features.feature) ) \ - as result FROM t_features;".format( + return self.underpassDB.run(geoFeaturesQuery( area, - "and raw_node.tags ? '{0}'".format(key) if key and not value else "", - "and raw_node.tags->'{0}' ~* '^{1}'".format(key, value) if key and value else "", - ) - return self.underpassDB.run(query, responseType, True) + tags, + hashtag, + page, + dateFrom, + dateTo, + "nodes" + ), responseType, True) + + + def getAll( + self, + area = None, + tags = None, + hashtag = None, + responseType = "json", + dateFrom = None, + dateTo = None, + page = None + ): + + polygons = self.getPolygons( + area, + tags, + hashtag, + responseType, + dateFrom, + dateTo, + page) + + lines = self.getLines( + area, + tags, + hashtag, + responseType, + dateFrom, + dateTo, + page) + + nodes = self.getNodes( + area, + tags, + hashtag, + responseType, + dateFrom, + dateTo, + page) + + result = {'type': 'FeatureCollection', 'features': []} + + if 'features' in polygons and polygons['features']: + result['features'] = result['features'] + polygons['features'] + + if 'features' in lines and lines['features']: + result['features'] = result['features'] + lines['features'] + + elif 'features' in nodes and nodes['features']: + result['features'] = result['features'] + nodes['features'] + + return result def getPolygonsList( self, area = None, - key = None, - value = None, + tags = None, hashtag = None, responseType = "json", + dateFrom = None, + dateTo = None, page = None ): - if page == 0: - page = 1 + return self.underpassDB.run(listFeaturesQuery( + area, + tags, + hashtag, + page, + dateFrom, + dateTo, + "ways_poly" + ), responseType, True) + + def getLinesList( + self, + area = None, + tags = None, + hashtag = None, + responseType = "json", + dateFrom = None, + dateTo = None, + page = None + ): + return self.underpassDB.run(listFeaturesQuery( + area, + tags, + hashtag, + page, + dateFrom, + dateTo, + "ways_line" + ), responseType, True) - query = "with t_ways AS ( \ - SELECT raw_poly.osm_id as id, ST_X(ST_Centroid(geometry)) as lat, ST_Y(ST_Centroid(geometry)) as lon, raw_poly.timestamp, tags, status FROM raw_poly \ - LEFT JOIN validation ON validation.osm_id = raw_poly.osm_id \ - WHERE \ - {0} {1} {2} {3} \ - ), t_features AS ( \ - SELECT to_jsonb(t_ways) as feature from t_ways \ - ) SELECT jsonb_agg(t_features.feature) as result FROM t_features;".format( - "ST_Intersects(\"geometry\", ST_GeomFromText('POLYGON(({0}))', 4326) )".format(area) if area else "1=1 ", - "and raw_poly.tags ? '{0}'".format(key) if key and not value else "", - "and raw_poly.tags->'{0}' ~* '^{1}'".format(key, value) if key and value else "", - "ORDER BY raw_poly.timestamp DESC LIMIT " + str(RESULTS_PER_PAGE) + " OFFSET {0}".format(page * RESULTS_PER_PAGE) if page else "", - ) - return self.underpassDB.run(query, responseType, True) def getNodesList( self, area = None, - key = None, - value = None, + tags = None, hashtag = None, responseType = "json", + dateFrom = None, + dateTo = None, page = None ): - if page == 0: - page = 1 + return self.underpassDB.run(listFeaturesQuery( + area, + tags, + hashtag, + page, + dateFrom, + dateTo, + "nodes" + ), responseType, True) - query = "with t_nodes AS ( \ - SELECT raw_node.osm_id as id, ST_X(ST_Centroid(geometry)) as lat, ST_Y(ST_Centroid(geometry)) as lon, tags, status FROM raw_node \ - LEFT JOIN validation ON validation.osm_id = raw_node.osm_id \ - WHERE {0} {1} {2} {3} \ - ), \ - t_features AS ( \ - SELECT to_jsonb(t_nodes) AS feature FROM t_nodes \ - ) SELECT jsonb_agg(t_features.feature) as result FROM t_features;".format( - "ST_Intersects(\"geometry\", ST_GeomFromText('POLYGON(({0}))', 4326) )".format(area) if area else "1=1 ", - "and raw_node.tags ? '{0}'".format(key) if key and not value else "", - "and raw_node.tags->'{0}' ~* '^{1}'".format(key, value) if key and value else "", - "ORDER BY raw_node.timestamp DESC LIMIT " + str(RESULTS_PER_PAGE) + " OFFSET {0}".format(page * RESULTS_PER_PAGE) if page else "", - ) - return self.underpassDB.run(query, responseType, True) \ No newline at end of file + + def getAllList( + self, + area = None, + tags = None, + hashtag = None, + responseType = "json", + dateFrom = None, + dateTo = None, + page = None + ): + + polygons = self.getPolygonsList( + area, + tags, + hashtag, + responseType, + dateFrom, + dateTo, + page) + + lines = self.getLinesList( + area, + tags, + hashtag, + responseType, + dateFrom, + dateTo, + page) + + nodes = self.getNodesList( + area, + tags, + hashtag, + responseType, + dateFrom, + dateTo, + page) + + result = [] + + if polygons: + result = result + polygons + + if lines: + result = result + lines + + if nodes: + result = result + nodes + + return result \ No newline at end of file diff --git a/python/dbapi/api/report.py b/python/dbapi/api/report.py index 97b39648..36654370 100644 --- a/python/dbapi/api/report.py +++ b/python/dbapi/api/report.py @@ -42,7 +42,7 @@ def getDataQualityGeo( st_y(location) as lon \ from changesets \ INNER JOIN validation \ - ON validation.change_id = changesets.id \ + ON validation.changeset = changesets.id \ where validation.status = 'badgeom' \ {0} {1} {2} {3} \ order by osm_id \ @@ -91,16 +91,16 @@ def getDataQualityTag( {0} {1} {2} {3} \ ), \ t1 AS ( \ - SELECT change_id, source, osm_id, type, \ + SELECT changeset, source, osm_id, type, \ unnest(values) as unnest_values \ from validation, t2 \ - where change_id = t2.id \ + where changeset = t2.id \ ) \ select \ 'https://osm.org/' || t1.type || '/' || t1.osm_id as link, \ t1.unnest_values as tag, t1.source \ FROM t1, t2 \ - where t1.change_id = t2.id \ + where t1.changeset = t2.id \ limit {4} offset {5}".format( "and closed_at >= '{0}'".format(fromDate) if fromDate else "", "and closed_at <= '{0}'".format(toDate) if toDate else "", @@ -127,15 +127,15 @@ def getDataQualityTagStats( {0} {1} {2} {3} \ ), \ t1 AS ( \ - SELECT change_id, source, \ + SELECT changeset, source, \ unnest(values) as unnest_values \ from validation, t2 \ - where change_id = t2.id \ + where changeset = t2.id \ ) \ SELECT \ t1.unnest_values as tag, t1.source, count(t1.unnest_values) \ FROM t1, t2 \ - where t1.change_id = t2.id \ + where t1.changeset = t2.id \ group by t1.unnest_values, t1.source \ order by count desc \ limit {4} offset {5}".format( diff --git a/python/restapi/main.py b/python/restapi/main.py index a7544013..0e757e5a 100644 --- a/python/restapi/main.py +++ b/python/restapi/main.py @@ -51,6 +51,7 @@ origins = [ "http://localhost", "http://localhost:5000", + "http://localhost:3000", "http://127.0.0.1", "http://127.0.0.1:5000" ] @@ -62,7 +63,7 @@ allow_origins=origins, allow_credentials=True, allow_methods=["*"], - allow_headers=["*"], + allow_headers=["*"] ) db = UnderpassDB(config.UNDERPASS_DB) @@ -156,8 +157,10 @@ def osmchangeValidate(request: OsmchangeValidateRequest): def getPolygons(request: RawRequest): results = rawer.getPolygons( area = request.area or None, - key = request.key or "", - value = request.value or "", + tags = request.tags or "", + hashtag = request.hashtag or "", + dateFrom = request.dateFrom or "", + dateTo = request.dateTo or "", page = request.page ) return results @@ -166,8 +169,35 @@ def getPolygons(request: RawRequest): def getNodes(request: RawRequest): results = rawer.getNodes( area = request.area, - key = request.key or "", - value = request.value or "" + tags = request.tags or "", + hashtag = request.hashtag or "", + dateFrom = request.dateFrom or "", + dateTo = request.dateTo or "", + page = request.page + ) + return results + +@app.post("/raw/lines") +def getLines(request: RawRequest): + results = rawer.getLines( + area = request.area, + tags = request.tags or "", + hashtag = request.hashtag or "", + dateFrom = request.dateFrom or "", + dateTo = request.dateTo or "", + page = request.page + ) + return results + +@app.post("/raw/all") +def getLines(request: RawRequest): + results = rawer.getAll( + area = request.area, + tags = request.tags or "", + hashtag = request.hashtag or "", + dateFrom = request.dateFrom or "", + dateTo = request.dateTo or "", + page = request.page ) return results @@ -175,8 +205,10 @@ def getNodes(request: RawRequest): def getPolygonsList(request: RawRequest): results = rawer.getPolygonsList( area = request.area or None, - key = request.key or "", - value = request.value or "", + tags = request.tags or "", + hashtag = request.hashtag or "", + dateFrom = request.dateFrom or "", + dateTo = request.dateTo or "", page = request.page ) return results @@ -185,8 +217,22 @@ def getPolygonsList(request: RawRequest): def getNodesList(request: RawRequest): results = rawer.getNodesList( area = request.area or None, - key = request.key or "", - value = request.value or "", + tags = request.tags or "", + hashtag = request.hashtag or "", + dateFrom = request.dateFrom or "", + dateTo = request.dateTo or "", page = request.page ) - return results \ No newline at end of file + return results + +@app.post("/raw/allList") +def getAllList(request: RawRequest): + results = rawer.getAllList( + area = request.area or None, + tags = request.tags or "", + hashtag = request.hashtag or "", + dateFrom = request.dateFrom or "", + dateTo = request.dateTo or "", + page = request.page, + ) + return results diff --git a/python/restapi/models.py b/python/restapi/models.py index 4047c1c9..6cb3e648 100644 --- a/python/restapi/models.py +++ b/python/restapi/models.py @@ -13,6 +13,8 @@ class OsmchangeValidateRequest(BaseModel): class RawRequest(BaseModel): area: str = None - key: str = None - value: str = None + tags: str = None + hashtag: str = None + dateFrom: str = None + dateTo: str = None page: int = None diff --git a/setup/underpass.sql b/setup/underpass.sql index b3a8a6bf..145f21c2 100644 --- a/setup/underpass.sql +++ b/setup/underpass.sql @@ -23,7 +23,7 @@ SET default_tablespace = ''; CREATE TABLE IF NOT EXISTS public.changesets ( id int8 NOT NULL, editor text, - user_id integer NOT NULL, + uid integer NOT NULL, created_at timestamptz, closed_at timestamptz, updated_at timestamptz, @@ -46,8 +46,8 @@ CREATE TYPE public.status AS ENUM ('notags', 'complete', 'incomplete', 'badvalue CREATE TABLE IF NOT EXISTS public.validation ( osm_id int8, - user_id int8, - change_id int8, + uid int8, + changeset int8, type public.objtype, status public.status, values text[], @@ -59,23 +59,39 @@ CREATE TABLE IF NOT EXISTS public.validation ( ALTER TABLE ONLY public.validation ADD CONSTRAINT validation_pkey PRIMARY KEY (osm_id, status, source); -CREATE TABLE IF NOT EXISTS public.raw_poly ( +CREATE TABLE IF NOT EXISTS public.ways_poly ( osm_id int8, - change_id int8, - geometry public.geometry(Polygon,4326), - tags public.hstore, + changeset int8, + geom public.geometry(Polygon,4326), + tags JSONB, refs int8[], timestamp timestamp with time zone, - version int + version int, + "user" text, + uid int8 ); -CREATE TABLE IF NOT EXISTS public.raw_node ( +CREATE TABLE IF NOT EXISTS public.ways_line ( osm_id int8, - change_id int8, - geometry public.geometry(Point,4326), - tags public.hstore, + changeset int8, + geom public.geometry(LineString,4326), + tags JSONB, + refs int8[], timestamp timestamp with time zone, - version int + version int, + "user" text, + uid int8 +); + +CREATE TABLE IF NOT EXISTS public.nodes ( + osm_id int8, + changeset int8, + geom public.geometry(Point,4326), + tags JSONB, + timestamp timestamp with time zone, + version int, + "user" text, + uid int8 ); CREATE TABLE IF NOT EXISTS public.way_refs ( @@ -83,7 +99,16 @@ CREATE TABLE IF NOT EXISTS public.way_refs ( node_id int8 ); -CREATE UNIQUE INDEX IF NOT EXISTS raw_osm_id_idx ON public.raw_node (osm_id); -CREATE UNIQUE INDEX IF NOT EXISTS raw_poly_osm_id_idx ON public.raw_poly (osm_id); -CREATE INDEX IF NOT EXISTS way_refs_nodes_idx ON public.way_refs (node_id); -CREATE INDEX IF NOT EXISTS way_refs_ways_idx ON public.way_refs (way_id); +CREATE UNIQUE INDEX nodes_id_idx ON public.nodes (osm_id); +CREATE UNIQUE INDEX ways_poly_id_idx ON public.ways_poly (osm_id); +CREATE UNIQUE INDEX ways_line_id_idx ON public.ways_line(osm_id); +CREATE INDEX way_refs_node_id_idx ON public.way_refs (node_id); +CREATE INDEX way_refs_way_id_idx ON public.way_refs (way_id); + +CREATE INDEX nodes_version_idx ON public.nodes (version); +CREATE INDEX ways_poly_version_idx ON public.ways_poly (version); +CREATE INDEX ways_line_version_idx ON public.ways_line (version); + +CREATE INDEX nodes_timestamp_idx ON public.nodes(timestamp DESC); +CREATE INDEX ways_poly_timestamp_idx ON public.ways_poly(timestamp DESC); +CREATE INDEX ways_line_timestamp_idx ON public.ways_line(timestamp DESC); diff --git a/src/bootstrap/bootstrap.cc b/src/bootstrap/bootstrap.cc index ea070d1d..bbed47d9 100644 --- a/src/bootstrap/bootstrap.cc +++ b/src/bootstrap/bootstrap.cc @@ -67,26 +67,32 @@ void startProcessingWays(const underpassconfig::UnderpassConfig &config) { auto queryvalidate = std::make_shared(db); auto queryraw = std::make_shared(db); + std::vector tables = { + QueryRaw::polyTable, + QueryRaw::lineTable + }; - int total = queryraw->getWaysCount(); - - if (total > 0) { - int count = 0; - long lastid = 0; - while (count < total) { - int percentage = (count * 100) / total; - auto task = std::make_shared(); - WayTask wayTask; - wayTask.plugin = validator; - wayTask.queryvalidate = queryvalidate; - wayTask.queryraw = queryraw; - wayTask.task = task; - wayTask.lastid = lastid; - processWays(wayTask); - db->query(task->query); - lastid = wayTask.lastid; - count += wayTask.processed; - std::cout << "\r" << "Processing : " << count << "/" << total << " (" << percentage << "%)"; + for (auto table_it = tables.begin(); table_it != tables.end(); ++table_it) { + int total = queryraw->getWaysCount(*table_it); + if (total > 0) { + int count = 0; + long lastid = 0; + while (count < total) { + int percentage = (count * 100) / total; + auto task = std::make_shared(); + WayTask wayTask; + wayTask.plugin = validator; + wayTask.queryvalidate = queryvalidate; + wayTask.queryraw = queryraw; + wayTask.task = task; + wayTask.lastid = lastid; + + processWays(wayTask, *table_it); + db->query(task->query); + lastid = wayTask.lastid; + count += wayTask.processed; + std::cout << "\r" << "Processing " << *table_it << ": " << count << "/" << total << " (" << percentage << "%)"; + } } } @@ -94,7 +100,7 @@ void startProcessingWays(const underpassconfig::UnderpassConfig &config) { // This thread get started for every page of way void -processWays(WayTask &wayTask) +processWays(WayTask &wayTask, const std::string &tableName) { #ifdef TIMING_DEBUG boost::timer::auto_cpu_timer timer("bootstrap::processWays(wayTask): took %w seconds\n"); @@ -106,23 +112,26 @@ processWays(WayTask &wayTask) auto queryraw = wayTask.queryraw; auto lastid = wayTask.lastid; - auto ways = queryraw->getWaysFromDB(lastid); + auto ways = queryraw->getWaysFromDB(lastid, tableName); wayTask.processed = ways->size(); - - // Proccesing ways - for (auto way = ways->begin(); way != ways->end(); ++way) { - if (way->refs.front() == way->refs.back()) { - log_debug("Way Id: %1%", way->id); - - // Bad geometry - if (way->containsKey("building") && (boost::geometry::num_points(way->linestring) - 1 < 4 || - plugin->unsquared(way->linestring)) - ) { - auto status = ValidateStatus(*way); - status.timestamp = boost::posix_time::microsec_clock::universal_time(); - status.source = "building"; - boost::geometry::centroid(way->linestring, status.center); - task->query += queryvalidate->applyChange(status, badgeom); + if (wayTask.processed > 0) { + // Proccesing ways + for (auto way = ways->begin(); way != ways->end(); ++way) { + + // If it's closed polygon + if (way->refs.front() == way->refs.back()) { + log_debug("Way Id: %1%", way->id); + + // Bad geometry + if (way->containsKey("building") && (boost::geometry::num_points(way->linestring) - 1 < 4 || + plugin->unsquared(way->linestring)) + ) { + auto status = ValidateStatus(*way); + status.timestamp = boost::posix_time::microsec_clock::universal_time(); + status.source = "building"; + boost::geometry::centroid(way->linestring, status.center); + task->query += queryvalidate->applyChange(status, badgeom); + } } // Fill the way_refs table @@ -130,8 +139,8 @@ processWays(WayTask &wayTask) task->query += "INSERT INTO way_refs (way_id, node_id) VALUES (" + std::to_string(way->id) + "," + std::to_string(*ref) + "); "; } } + wayTask.lastid = ways->back().id; } - wayTask.lastid = ways->back().id; } diff --git a/src/bootstrap/bootstrap.hh b/src/bootstrap/bootstrap.hh index daf66839..9e4aa966 100644 --- a/src/bootstrap/bootstrap.hh +++ b/src/bootstrap/bootstrap.hh @@ -47,6 +47,6 @@ struct WayTask { void startProcessingWays(const underpassconfig::UnderpassConfig &config); // This thread get started for every page of way -void processWays(WayTask &wayTask); +void processWays(WayTask &wayTask, const std::string &tableName); } \ No newline at end of file diff --git a/src/data/pq.cc b/src/data/pq.cc index 72b5f7e2..678bcd8e 100644 --- a/src/data/pq.cc +++ b/src/data/pq.cc @@ -167,6 +167,8 @@ Pq::escapedString(std::string text) newstr += "'"; } else if (text[i] == '\"') { newstr += """; + } else if (text[i] == '\'') { + newstr += """; } else if (text[i] == ')') { newstr += ")"; } else if (text[i] == '(') { @@ -178,7 +180,7 @@ Pq::escapedString(std::string text) } i++; } - return sdb->esc(boost::locale::conv::to_utf(newstr, "Latin1")); + return sdb->esc(newstr); } } // namespace pq diff --git a/src/osm/changeset.cc b/src/osm/changeset.cc index 4c0e5aa2..8a80738d 100644 --- a/src/osm/changeset.cc +++ b/src/osm/changeset.cc @@ -84,8 +84,8 @@ using namespace boost::gregorian; #include "utils/log.hh" using namespace logger; -/// \namespace changeset -namespace changeset { +/// \namespace changesets +namespace changesets { bool ChangeSetFile::readChanges(const std::vector &buffer) @@ -305,7 +305,7 @@ ChangeSetFile::readXML(std::istream &xml) for (auto value: pt.get_child("osm")) { if (value.first == "changeset") { - changeset::ChangeSet change; + changesets::ChangeSet change; // Process the tags. These don't exist for every element for (auto tag: value.second) { if (tag.first == "tag") { @@ -349,7 +349,7 @@ ChangeSetFile::on_start_element(const Glib::ustring &name, { // log_debug("Element %1%", name); if (name == "changeset") { - auto change = std::make_shared(attributes); + auto change = std::make_shared(attributes); changes.push_back(change); if (change->closed_at != not_a_date_time && (last_closed_at == not_a_date_time || change->closed_at > last_closed_at)) { last_closed_at = change->closed_at; @@ -436,4 +436,4 @@ ChangeSetFile::on_start_element(const Glib::ustring &name, } #endif // EOF LIBXML -} // namespace changeset +} // namespace changesets diff --git a/src/osm/changeset.hh b/src/osm/changeset.hh index 03eec492..d43d8829 100644 --- a/src/osm/changeset.hh +++ b/src/osm/changeset.hh @@ -61,8 +61,8 @@ namespace geoutil { class GeoUtil; }; -/// \namespace changeset -namespace changeset { +/// \namespace changesets +namespace changesets { /// \file changeset.hh @@ -179,7 +179,7 @@ class ChangeSetFile ptime last_closed_at = not_a_date_time; }; -} // namespace changeset +} // namespace changesets #endif // EOF __CHANGESET_HH__ diff --git a/src/osm/osmchange.cc b/src/osm/osmchange.cc index 73793b11..d5a9705c 100644 --- a/src/osm/osmchange.cc +++ b/src/osm/osmchange.cc @@ -360,7 +360,7 @@ OsmChangeFile::on_start_element(const Glib::ustring &name, } else if (attr_pair.name == "user") { change->obj->user = attr_pair.value; } else if (attr_pair.name == "changeset") { - change->obj->change_id = std::stol(attr_pair.value); + change->obj->changeset = std::stol(attr_pair.value); } else if (attr_pair.name == "lat") { auto lat = reinterpret_cast(change->obj.get()); lat->setLatitude(std::stod(attr_pair.value)); @@ -519,14 +519,14 @@ OsmChangeFile::collectStats(const multipolygon_t &poly) node->tags.find("created_at") != node->tags.end()) { continue; } - ostats = (*mstats)[node->change_id]; + ostats = (*mstats)[node->changeset]; if (ostats.get() == 0) { ostats = std::make_shared(); - ostats->change_id = node->change_id; - ostats->user_id = node->uid; + ostats->changeset = node->changeset; + ostats->uid = node->uid; ostats->username = node->user; ostats->closed_at = node->timestamp; - (*mstats)[node->change_id] = ostats; + (*mstats)[node->changeset] = ostats; } auto hits = scanTags(node->tags, osmchange::node); for (auto hit = std::begin(*hits); hit != std::end(*hits); ++hit) { @@ -557,14 +557,14 @@ OsmChangeFile::collectStats(const multipolygon_t &poly) if (way->tags.size() == 1 && way->tags.find("created_at") != way->tags.end()) { continue; } - ostats = (*mstats)[way->change_id]; + ostats = (*mstats)[way->changeset]; if (ostats.get() == 0) { ostats = std::make_shared(); - ostats->change_id = way->change_id; - ostats->user_id = way->uid; + ostats->changeset = way->changeset; + ostats->uid = way->uid; ostats->username = way->user; ostats->closed_at = way->timestamp; - (*mstats)[way->change_id] = ostats; + (*mstats)[way->changeset] = ostats; } auto hits = scanTags(way->tags, osmchange::way); @@ -597,7 +597,7 @@ OsmChangeFile::collectStats(const multipolygon_t &poly) } double length = boost::geometry::length(globe, boost::geometry::strategy::distance::haversine(6371.0)); - // log_debug("LENGTH: %1% %2%", std::to_string(length), way->change_id); + // log_debug("LENGTH: %1% %2%", std::to_string(length), way->changeset); ostats->added[tag] += length; } } @@ -613,14 +613,14 @@ OsmChangeFile::collectStats(const multipolygon_t &poly) if (relation->tags.size() == 0) { continue; } - ostats = (*mstats)[relation->change_id]; + ostats = (*mstats)[relation->changeset]; if (ostats.get() == 0) { ostats = std::make_shared(); - ostats->change_id = relation->change_id; - ostats->user_id = relation->uid; + ostats->changeset = relation->changeset; + ostats->uid = relation->uid; ostats->username = relation->user; ostats->closed_at = relation->timestamp; - (*mstats)[relation->change_id] = ostats; + (*mstats)[relation->changeset] = ostats; } auto hits = scanTags(relation->tags, osmchange::relation); for (auto hit = std::begin(*hits); hit != std::end(*hits); ++hit) { @@ -663,8 +663,8 @@ OsmChangeFile::scanTags(std::map tags, osmchange::osmt void ChangeStats::dump(void) { - std::cerr << "Dumping ChangeStats for: \t " << change_id << std::endl; - std::cerr << "\tUser ID: \t\t " << user_id << std::endl; + std::cerr << "Dumping ChangeStats for: \t " << changeset << std::endl; + std::cerr << "\tUser ID: \t\t " << uid << std::endl; std::cerr << "\tUser Name: \t\t " << username << std::endl; std::cerr << "\tAdded features: " << added.size() << std::endl; for (auto it = std::begin(added); it != std::end(added); ++it) { @@ -732,7 +732,7 @@ OsmChangeFile::validateWays(const multipolygon_t &poly, std::shared_ptrrefs.front() == way->refs.back()) { status->timestamp = boost::posix_time::microsec_clock::universal_time(); - status->user_id = way->uid; + status->uid = way->uid; // Overlapping if (plugin->overlaps(change->ways, *way)) { diff --git a/src/osm/osmchange.hh b/src/osm/osmchange.hh index 0f67a52c..e66c4984 100644 --- a/src/osm/osmchange.hh +++ b/src/osm/osmchange.hh @@ -68,8 +68,8 @@ typedef enum { empty, node, way, relation, member } osmtype_t; /// which later gets added to the database statistics. class ChangeStats { public: - long change_id = 0; ///< The ID of this change - long user_id = 0; ///< The User ID + long changeset = 0; ///< The ID of this change + long uid = 0; ///< The User ID std::string username; ///< The User Name ptime created_at; ///< The starting timestamp ptime closed_at; ///< The finished timestamp diff --git a/src/osm/osmobjects.cc b/src/osm/osmobjects.cc index a14bcc7b..9f9c0ae3 100644 --- a/src/osm/osmobjects.cc +++ b/src/osm/osmobjects.cc @@ -79,8 +79,8 @@ OsmObject::dump(void) const } else { std::cerr << "\tNot in Priority area" << std::endl; } - if (change_id > 0) { - std::cerr << "\tChange ID: " << std::to_string(change_id) << std::endl; + if (changeset > 0) { + std::cerr << "\tChange ID: " << std::to_string(changeset) << std::endl; } if (tags.size() > 0) { std::cerr << "\tTags: " << tags.size() << std::endl; diff --git a/src/osm/osmobjects.hh b/src/osm/osmobjects.hh index 92ac88e6..5f5db82a 100644 --- a/src/osm/osmobjects.hh +++ b/src/osm/osmobjects.hh @@ -72,7 +72,7 @@ class OsmObject { void setAction(action_t act) { action = act; }; void setUID(long val) { uid = val; }; - void setChangeID(long val) { change_id = val; }; + void setChangeID(long val) { changeset = val; }; action_t action = none; ///< the action that contains this object osmtype_t type = empty; ///< The type of this object, node, way, or relation @@ -81,7 +81,7 @@ class OsmObject { ptime timestamp; ///< The timestamp of this object's creation or modification long uid = 0; ///< The User ID of the mapper of this object std::string user; ///< The User name of the mapper of this object - long change_id = 0; ///< The changeset ID this object is contained in + long changeset = 0; ///< The changeset ID this object is contained in std::map tags; ///< OSM metadata tags bool priority = false; ///< Whether it's in the priority area diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc index d4b1cc10..dc2daa2b 100644 --- a/src/raw/queryraw.cc +++ b/src/raw/queryraw.cc @@ -24,6 +24,8 @@ /// includes querying existing data in the database, as well as /// updating the database. +// TODO: add support for relations/multipolygon + // This is generated by autoconf #ifdef HAVE_CONFIG_H #include "unconfig.h" @@ -63,10 +65,10 @@ QueryRaw::applyChange(const OsmNode &node) const { std::string query; if (node.action == osmobjects::create || node.action == osmobjects::modify) { - query = "INSERT INTO raw_node as r (osm_id, geometry, tags, timestamp, version) VALUES("; - std::string format = "%d, ST_GeomFromText(\'%s\', 4326), %s, \'%s\', %d \ - ) ON CONFLICT (osm_id) DO UPDATE SET geometry = ST_GeomFromText(\'%s\', \ - 4326), tags = %s, timestamp = \'%s\', version = %d WHERE r.version < %d;"; + query = "INSERT INTO nodes as r (osm_id, geom, tags, timestamp, version, \"user\", uid, changeset) VALUES("; + std::string format = "%d, ST_GeomFromText(\'%s\', 4326), %s, \'%s\', %d, \'%s\', %d, %d \ + ) ON CONFLICT (osm_id) DO UPDATE SET geom = ST_GeomFromText(\'%s\', \ + 4326), tags = %s, timestamp = \'%s\', version = %d, \"user\" = \'%s\', uid = %d, changeset = %d WHERE r.version < %d;"; boost::format fmt(format); // osm_id @@ -82,14 +84,14 @@ QueryRaw::applyChange(const OsmNode &node) const std::string tags = ""; if (node.tags.size() > 0) { for (auto it = std::begin(node.tags); it != std::end(node.tags); ++it) { - std::string tag_format = "\"%s\" => \"%s\","; + std::string tag_format = "\"%s\" : \"%s\","; boost::format tag_fmt(tag_format); tag_fmt % dbconn->escapedString(it->first); tag_fmt % dbconn->escapedString(it->second); tags += tag_fmt.str(); } tags.erase(tags.size() - 1); - tags = "'" + tags + "'"; + tags = "'{" + tags + "}'"; } else { tags = "null"; } @@ -99,36 +101,59 @@ QueryRaw::applyChange(const OsmNode &node) const fmt % timestamp; // version fmt % node.version; + // user + fmt % dbconn->escapedString(node.user); + // uid + fmt % node.uid; + // changeset + fmt % node.changeset; // ON CONFLICT fmt % geometry; fmt % tags; fmt % timestamp; fmt % node.version; + fmt % dbconn->escapedString(node.user); + fmt % node.uid; + fmt % node.changeset; fmt % node.version; query += fmt.str(); } else if (node.action == osmobjects::remove) { - query = "DELETE from raw_node where osm_id = " + std::to_string(node.id) + ";"; + query = "DELETE from nodes where osm_id = " + std::to_string(node.id) + ";"; } return query; } + +const std::string QueryRaw::polyTable = "ways_poly"; +const std::string QueryRaw::lineTable = "ways_line"; + std::string QueryRaw::applyChange(const OsmWay &way) const { std::string query = ""; - - if (way.refs.size() > 3 && (way.refs.front() == way.refs.back()) + const std::string* tableName; + + std::stringstream ss; + if (way.refs.size() > 3 && (way.refs.front() == way.refs.back())) { + tableName = &QueryRaw::polyTable; + ss << std::setprecision(12) << boost::geometry::wkt(way.polygon); + } else { + tableName = &QueryRaw::lineTable; + ss << std::setprecision(12) << boost::geometry::wkt(way.linestring); + } + std::string geostring = ss.str(); + + if (way.refs.size() > 2 && (way.action == osmobjects::create || way.action == osmobjects::modify)) { - if (way.refs.size() == boost::geometry::num_points(way.linestring)) { - query = "INSERT INTO raw_poly as r (osm_id, tags, refs, geometry, timestamp, version) VALUES("; - std::string format = "%d, %s, %s, %s, \'%s\', %d) \ - ON CONFLICT (osm_id) DO UPDATE SET tags = %s, refs = %s, geometry = %s, timestamp = \'%s\', version = %d WHERE r.version <= %d;"; + query = "INSERT INTO " + *tableName + " as r (osm_id, tags, refs, geom, timestamp, version, \"user\", uid, changeset) VALUES("; + std::string format = "%d, %s, %s, %s, \'%s\', %d, \'%s\', %d, %d) \ + ON CONFLICT (osm_id) DO UPDATE SET tags = %s, refs = %s, geom = %s, timestamp = \'%s\', version = %d, \"user\" = \'%s\', uid = %d, changeset = %d WHERE r.version <= %d;"; boost::format fmt(format); // osm_id @@ -138,14 +163,14 @@ QueryRaw::applyChange(const OsmWay &way) const std::string tags = ""; if (way.tags.size() > 0) { for (auto it = std::begin(way.tags); it != std::end(way.tags); ++it) { - std::string tag_format = "\"%s\" => \"%s\","; + std::string tag_format = "\"%s\" : \"%s\","; boost::format tag_fmt(tag_format); tag_fmt % dbconn->escapedString(it->first); tag_fmt % dbconn->escapedString(it->second); tags += tag_fmt.str(); } tags.erase(tags.size() - 1); - tags = "'" + tags + "'"; + tags = "'{" + tags + "}'"; } else { tags = "null"; } @@ -163,9 +188,6 @@ QueryRaw::applyChange(const OsmWay &way) const // geometry std::string geometry; - std::stringstream ss; - ss << std::setprecision(12) << boost::geometry::wkt(way.polygon); - std::string geostring = ss.str(); geometry = "ST_GeomFromText(\'" + geostring + "\', 4326)"; fmt % geometry; @@ -174,6 +196,12 @@ QueryRaw::applyChange(const OsmWay &way) const fmt % timestamp; // version fmt % way.version; + // user + fmt % dbconn->escapedString(way.user); + // uid + fmt % way.uid; + // changeset + fmt % way.changeset; // ON CONFLICT fmt % tags; @@ -181,6 +209,9 @@ QueryRaw::applyChange(const OsmWay &way) const fmt % geometry; fmt % timestamp; fmt % way.version; + fmt % dbconn->escapedString(way.user); + fmt % way.uid; + fmt % way.changeset; fmt % way.version; query += fmt.str(); @@ -192,7 +223,8 @@ QueryRaw::applyChange(const OsmWay &way) const } } else if (way.action == osmobjects::remove) { query += "DELETE FROM way_refs WHERE way_id=" + std::to_string(way.id) + ";"; - query += "DELETE FROM raw_poly where osm_id = " + std::to_string(way.id) + ";"; + query += "DELETE FROM " + QueryRaw::polyTable + " where osm_id = " + std::to_string(way.id) + ";"; + query += "DELETE FROM " + QueryRaw::lineTable + " where osm_id = " + std::to_string(way.id) + ";"; } return query; @@ -273,7 +305,7 @@ void QueryRaw::getNodeCache(std::shared_ptr osmchanges, const mul if (referencedNodeIds.size() > 1) { referencedNodeIds.erase(referencedNodeIds.size() - 1); // Get Nodes from DB - std::string nodesQuery = "SELECT osm_id, st_x(geometry) as lat, st_y(geometry) as lon FROM raw_node where osm_id in (" + referencedNodeIds + ");"; + std::string nodesQuery = "SELECT osm_id, st_x(geom) as lat, st_y(geom) as lon FROM nodes where osm_id in (" + referencedNodeIds + ");"; auto result = dbconn->query(nodesQuery); // Fill nodecache for (auto node_it = result.begin(); node_it != result.end(); ++node_it) { @@ -296,7 +328,9 @@ void QueryRaw::getNodeCache(std::shared_ptr osmchanges, const mul boost::geometry::append(way->linestring, osmchanges->nodecache.at(*rit)); } } - way->polygon = { {std::begin(way->linestring), std::end(way->linestring)} }; + if (way->isClosed()) { + way->polygon = { {std::begin(way->linestring), std::end(way->linestring)} }; + } } } @@ -321,7 +355,7 @@ QueryRaw::getNodeCacheFromWays(std::shared_ptr> ways, std::m nodeIds.erase(nodeIds.size() - 1); // Get Nodes from DB - std::string nodesQuery = "SELECT osm_id, st_x(geometry) as lat, st_y(geometry) as lon FROM raw_node where osm_id in (" + nodeIds + ") and st_x(geometry) is not null and st_y(geometry) is not null;"; + std::string nodesQuery = "SELECT osm_id, st_x(geom) as lat, st_y(geom) as lon FROM nodes where osm_id in (" + nodeIds + ") and st_x(geom) is not null and st_y(geom) is not null;"; auto result = dbconn->query(nodesQuery); // Fill nodecache for (auto node_it = result.begin(); node_it != result.end(); ++node_it) { @@ -340,7 +374,7 @@ std::map parseTagsString(const std::string& input) { std::string token; while (std::getline(ss, token, ',')) { // Find the position of the arrow - size_t arrowPos = token.find("=>"); + size_t arrowPos = token.find(":"); if (arrowPos != std::string::npos) { std::string key = token.substr(1, arrowPos - 1); std::string value = token.substr(arrowPos + 2); @@ -361,7 +395,8 @@ QueryRaw::getWaysByNodesRefs(std::string &nodeIds) const // Get all ways that have references to nodes std::list> ways; - std::string waysQuery = "SELECT distinct(osm_id), refs, version, tags from way_refs join raw_poly rp on rp.osm_id = way_id where node_id = any(ARRAY[" + nodeIds + "]);"; + std::string waysQuery = "SELECT distinct(osm_id), refs, version, tags from way_refs join ways_poly wp on wp.osm_id = way_id where node_id = any(ARRAY[" + nodeIds + "])"; + waysQuery += " UNION SELECT distinct(osm_id), refs, version, tags from way_refs join ways_line wl on wl.osm_id = way_id where node_id = any(ARRAY[" + nodeIds + "]);"; auto ways_result = dbconn->query(waysQuery); // Fill vector of OsmWay objects @@ -386,17 +421,23 @@ QueryRaw::getWaysByNodesRefs(std::string &nodeIds) const return ways; } -int QueryRaw::getWaysCount() { - std::string query = "select count(osm_id) from raw_poly"; +int QueryRaw::getWaysCount(const std::string &tableName) { + std::string query = "select count(osm_id) from " + tableName; auto result = dbconn->query(query); return result[0][0].as(); } std::shared_ptr> -QueryRaw::getWaysFromDB(int lastid) { - std::string waysQuery = "SELECT osm_id, refs, ST_AsText(ST_ExteriorRing(geometry), 4326), version, tags FROM raw_poly where osm_id > " + std::to_string(lastid) + " order by osm_id asc limit 500;"; +QueryRaw::getWaysFromDB(int lastid, const std::string &tableName) { + std::string waysQuery; + if (tableName == QueryRaw::polyTable) { + waysQuery = "SELECT osm_id, refs, ST_AsText(ST_ExteriorRing(geom), 4326)"; + } else { + waysQuery = "SELECT osm_id, refs, ST_AsText(geom, 4326)"; + } + waysQuery += ", version, tags FROM " + tableName + " where osm_id > " + std::to_string(lastid) + " order by osm_id asc limit 500;"; + auto ways_result = dbconn->query(waysQuery); - // Fill vector of OsmWay objects auto ways = std::make_shared>(); for (auto way_it = ways_result.begin(); way_it != ways_result.end(); ++way_it) { @@ -405,41 +446,29 @@ QueryRaw::getWaysFromDB(int lastid) { std::string refs_str = (*way_it)[1].as(); if (refs_str.size() > 1) { way.refs = arrayStrToVector(refs_str); - } - std::string poly = (*way_it)[2].as(); - boost::geometry::read_wkt(poly, way.linestring); - way.polygon = { {std::begin(way.linestring), std::end(way.linestring)} }; - way.version = (*way_it)[3].as(); - auto tags = (*way_it)[4]; - if (!tags.is_null()) { - auto tags = parseTagsString((*way_it)[4].as()); - for (auto const& [key, val] : tags) - { - way.addTag(key, val); + + std::string poly = (*way_it)[2].as(); + boost::geometry::read_wkt(poly, way.linestring); + + if (tableName == QueryRaw::polyTable) { + way.polygon = { {std::begin(way.linestring), std::end(way.linestring)} }; + } + way.version = (*way_it)[3].as(); + auto tags = (*way_it)[4]; + if (!tags.is_null()) { + auto tags = parseTagsString((*way_it)[4].as()); + for (auto const& [key, val] : tags) + { + way.addTag(key, val); + } } + ways->push_back(way); } - ways->push_back(way); } return ways; } -std::shared_ptr -QueryRaw::getWayById(long id) { - std::string waysQuery = "SELECT osm_id, refs, version FROM raw_poly where osm_id=" + std::to_string(id) + ";"; - auto result = dbconn->query(waysQuery); - - // Fill vector of OsmWay objects - OsmWay way; - way.id = result[0][0].as(); - way.version = result[0][2].as(); - std::string refs_str = result[0][1].as(); - if (refs_str.size() > 1) { - way.refs = arrayStrToVector(refs_str); - } - return std::make_shared(way); -} - } // namespace queryraw // local Variables: diff --git a/src/raw/queryraw.hh b/src/raw/queryraw.hh index 7ba79eae..b47559f6 100644 --- a/src/raw/queryraw.hh +++ b/src/raw/queryraw.hh @@ -56,6 +56,9 @@ class QueryRaw { ~QueryRaw(void){}; QueryRaw(std::shared_ptr db); + static const std::string polyTable; + static const std::string lineTable; + /// Build query for processed Node std::string applyChange(const OsmNode &node) const; /// Build query for processed Way @@ -71,11 +74,9 @@ class QueryRaw { // DB connection std::shared_ptr dbconn; // Get ways count - int getWaysCount(); + int getWaysCount(const std::string &tableName); // Get ways by page - std::shared_ptr> getWaysFromDB(int page); - // Get single way by id - std::shared_ptr getWayById(long id); + std::shared_ptr> getWaysFromDB(int page, const std::string &tableName); }; diff --git a/src/replicator/planetreplicator.cc b/src/replicator/planetreplicator.cc index 5430cf52..94731950 100644 --- a/src/replicator/planetreplicator.cc +++ b/src/replicator/planetreplicator.cc @@ -75,7 +75,7 @@ using namespace underpassconfig; using namespace logger; // Forward declarations -namespace changeset { +namespace changesets { class ChangeSet; }; diff --git a/src/replicator/planetreplicator.hh b/src/replicator/planetreplicator.hh index 544ab41f..c6deff59 100644 --- a/src/replicator/planetreplicator.hh +++ b/src/replicator/planetreplicator.hh @@ -53,7 +53,7 @@ class PlanetReplicator : public replication::Planet { private: std::vector default_minutes; std::vector default_changesets; - std::shared_ptr changes; ///< All the changes in the file + std::shared_ptr changes; ///< All the changes in the file std::shared_ptr> hashes; ///< Existing hashtags }; diff --git a/src/replicator/replication.cc b/src/replicator/replication.cc index afac0afe..6f1174ca 100644 --- a/src/replicator/replication.cc +++ b/src/replicator/replication.cc @@ -217,7 +217,7 @@ StateFile::isValid() const bool Replication::readChanges(const std::string &file) { - changeset::ChangeSetFile changeset; + changesets::ChangeSetFile changeset; std::ifstream stream; stream.open(file, std::ifstream::in); changeset.readXML(stream); diff --git a/src/replicator/threads.cc b/src/replicator/threads.cc index 90c6987f..f0790845 100644 --- a/src/replicator/threads.cc +++ b/src/replicator/threads.cc @@ -374,7 +374,7 @@ threadChangeSet(std::shared_ptr &remote, task.status = file.status; if (file.status == reqfile_t::success) { - auto changeset = std::make_unique(); + auto changeset = std::make_unique(); log_debug("Processing ChangeSet: %1%", remote->filespec); auto xml = planet->processData(remote->filespec, *file.data); std::istream& input(xml); diff --git a/src/stats/querystats.cc b/src/stats/querystats.cc index 20a23e80..21011723 100644 --- a/src/stats/querystats.cc +++ b/src/stats/querystats.cc @@ -105,7 +105,7 @@ QueryStats::applyChange(const osmchange::ChangeStats &change) const // Some of the data field in the changset come from a different file, // which may not be downloaded yet. ptime now = boost::posix_time::microsec_clock::universal_time(); - std::string aquery = "INSERT INTO changesets (id, user_id, closed_at, updated_at, "; + std::string aquery = "INSERT INTO changesets (id, uid, closed_at, updated_at, "; if (change.added.size() > 0) { aquery += "added, "; @@ -116,8 +116,8 @@ QueryStats::applyChange(const osmchange::ChangeStats &change) const aquery.erase(aquery.size() - 2); aquery += ")"; - aquery += " VALUES(" + std::to_string(change.change_id) + ", "; - aquery += std::to_string(change.user_id) + ", "; + aquery += " VALUES(" + std::to_string(change.changeset) + ", "; + aquery += std::to_string(change.uid) + ", "; aquery += "\'" + to_simple_string(change.closed_at) + "\', "; aquery += "\'" + to_simple_string(now) + "\', "; @@ -152,13 +152,13 @@ QueryStats::applyChange(const osmchange::ChangeStats &change) const } std::string -QueryStats::applyChange(const changeset::ChangeSet &change) const +QueryStats::applyChange(const changesets::ChangeSet &change) const { #ifdef TIMING_DEBUG_X boost::timer::auto_cpu_timer timer("applyChange(changeset): took %w seconds\n"); #endif - std::string query = "INSERT INTO changesets (id, editor, user_id, created_at, closed_at, updated_at"; + std::string query = "INSERT INTO changesets (id, editor, uid, created_at, closed_at, updated_at"; if (change.hashtags.size() > 0) { query += ", hashtags "; diff --git a/src/stats/querystats.hh b/src/stats/querystats.hh index c52ce370..2ab7c4f0 100644 --- a/src/stats/querystats.hh +++ b/src/stats/querystats.hh @@ -51,7 +51,7 @@ using namespace boost::gregorian; using namespace pq; // Forward declarations -namespace changeset { +namespace changesets { class ChangeSet; }; namespace osmchange { @@ -74,7 +74,7 @@ class QueryStats { ~QueryStats(void){}; QueryStats(std::shared_ptr db); /// Build query for processed ChangeSet - std::string applyChange(const changeset::ChangeSet &change) const; + std::string applyChange(const changesets::ChangeSet &change) const; /// Build query for processed OsmChange std::string applyChange(const osmchange::ChangeStats &change) const; // Database connection, used for escape strings diff --git a/src/testsuite/libunderpass.all/areafilter-test.cc b/src/testsuite/libunderpass.all/areafilter-test.cc index efd24497..da34a20a 100644 --- a/src/testsuite/libunderpass.all/areafilter-test.cc +++ b/src/testsuite/libunderpass.all/areafilter-test.cc @@ -28,7 +28,7 @@ TestState runtest; using namespace logger; -class TestChangeset : public changeset::ChangeSetFile {}; +class TestChangeset : public changesets::ChangeSetFile {}; class TestOsmChange : public osmchange::OsmChangeFile {}; int @@ -92,7 +92,7 @@ main(int argc, char *argv[]) osmchangeFile += "/testsuite/testdata/areafilter-test.osm"; TestChangeset changeset; TestOsmChange osmchange; - changeset::ChangeSet *testChangeset; + changesets::ChangeSet *testChangeset; // ChangeSet - Whole world changeset.readChanges(changesetFile); diff --git a/src/testsuite/libunderpass.all/change-test.cc b/src/testsuite/libunderpass.all/change-test.cc index edc28a0d..f3a319ff 100644 --- a/src/testsuite/libunderpass.all/change-test.cc +++ b/src/testsuite/libunderpass.all/change-test.cc @@ -39,7 +39,7 @@ namespace opts = boost::program_options; using namespace logger; -using namespace changeset; +using namespace changesets; using namespace boost::posix_time; using namespace boost::gregorian; @@ -67,7 +67,7 @@ using namespace boost::gregorian; TestState runtest; -class TestCS : public changeset::ChangeSetFile { +class TestCS : public changesets::ChangeSetFile { }; class TestCO : public osmchange::OsmChangeFile { @@ -110,7 +110,7 @@ main(int argc, char *argv[]) if (vm.count("changefile")) { std::string file = vm["changefile"].as(); std::cout << "Importing change file " << file << std::endl; - auto changeset = std::make_shared(); + auto changeset = std::make_shared(); changeset->readChanges(file); changeset->dump(); exit(0); @@ -174,13 +174,13 @@ main(int argc, char *argv[]) std::map changeset_ids_found; for (const auto &change: testco.changes) { for (const auto &node: change->nodes) { - changeset_ids_found.insert(std::pair(node->change_id, node->change_id)); + changeset_ids_found.insert(std::pair(node->changeset, node->changeset)); } for (const auto &way: change->ways) { - changeset_ids_found.insert(std::pair(way->change_id, way->change_id)); + changeset_ids_found.insert(std::pair(way->changeset, way->changeset)); } for (const auto &relation: change->relations) { - changeset_ids_found.insert(std::pair(relation->change_id, relation->change_id)); + changeset_ids_found.insert(std::pair(relation->changeset, relation->changeset)); } } bool all_changesets_tracked = true; @@ -198,14 +198,14 @@ main(int argc, char *argv[]) auto tf = testco.changes.front(); auto tnf = tf->nodes.front(); - if (tnf->change_id == 99069702 && tnf->id == 5776216755) { + if (tnf->changeset == 99069702 && tnf->id == 5776216755) { runtest.pass("ChangeSetFile::readChanges(first change, first node)"); } else { runtest.fail("ChangeSetFile::readChanges(first change, first node)"); } auto twf = tf->ways.front(); // twf->dump(); - if (twf->change_id == 99069879L && twf->id == 474556695L && + if (twf->changeset == 99069879L && twf->id == 474556695L && twf->uid == 1041828L) { runtest.pass("ChangeSetFile::readChanges(first change, first way)"); } else { @@ -213,14 +213,14 @@ main(int argc, char *argv[]) } auto tnb = tf->nodes.back(); // tnb->dump(); - if (tnb->change_id == 94802322L && tnb->id == 289112823L) { + if (tnb->changeset == 94802322L && tnb->id == 289112823L) { runtest.pass("ChangeSetFile::readChanges(first change, last node)"); } else { runtest.fail("ChangeSetFile::readChanges(first change, last node)"); } auto twb = tf->ways.back(); // twb->dump(); - if (twb->change_id == 99063443L && twb->id == 67365141L && + if (twb->changeset == 99063443L && twb->id == 67365141L && twb->uid == 1137406L) { runtest.pass("ChangeSetFile::readChanges(first change, last way)"); } else { diff --git a/src/testsuite/libunderpass.all/hashtags-test.cc b/src/testsuite/libunderpass.all/hashtags-test.cc index 15545818..e117826d 100644 --- a/src/testsuite/libunderpass.all/hashtags-test.cc +++ b/src/testsuite/libunderpass.all/hashtags-test.cc @@ -28,7 +28,7 @@ TestState runtest; using namespace logger; -class TestChangeset : public changeset::ChangeSetFile {}; +class TestChangeset : public changesets::ChangeSetFile {}; int main(int argc, char *argv[]) @@ -38,7 +38,7 @@ main(int argc, char *argv[]) std::string changesetFile(DATADIR); changesetFile += "/testsuite/testdata/hashtags-test.osc"; TestChangeset changeset; - changeset::ChangeSet *change; + changesets::ChangeSet *change; changeset.readChanges(changesetFile); multipolygon_t polyEmpty; changeset.areaFilter(polyEmpty); diff --git a/src/testsuite/libunderpass.all/stats-test.cc b/src/testsuite/libunderpass.all/stats-test.cc index 164e7a8b..b119670c 100644 --- a/src/testsuite/libunderpass.all/stats-test.cc +++ b/src/testsuite/libunderpass.all/stats-test.cc @@ -65,7 +65,7 @@ class TestStats { std::string jsonstr = ""; for (auto it = std::begin(*stats); it != std::end(*stats); ++it) { auto changestats = it->second; - jsonstr += "{\"changeset_id\":" + std::to_string(changestats->change_id) + ", "; + jsonstr += "{\"changeset_id\":" + std::to_string(changestats->changeset) + ", "; jsonstr += "\"filespec\": \"" + filespec + "\" "; if (changestats->added.size() > 0) { @@ -233,9 +233,9 @@ class TestStats { for (auto it = std::begin(*stats); it != std::end(*stats); ++it) { auto changestats = it->second; - if (changestats->change_id == validation.at("change_id")) { + if (changestats->changeset == validation.at("changeset")) { if (this->verbose) { - std::cout << "change_id: " << changestats->change_id << std::endl; + std::cout << "changeset: " << changestats->changeset << std::endl; } // TODO: get values to test from YAML validation file testStat(changestats, validation, "highway"); diff --git a/src/testsuite/libunderpass.all/val-test.cc b/src/testsuite/libunderpass.all/val-test.cc index aa4afcce..36bf7fa2 100644 --- a/src/testsuite/libunderpass.all/val-test.cc +++ b/src/testsuite/libunderpass.all/val-test.cc @@ -159,7 +159,7 @@ test_semantic_building(std::shared_ptr &plugin) { osmobjects::OsmNode node; node.id = 11111; - node.change_id = 22222; + node.changeset = 22222; // Node with no tags status = plugin->checkPOI(node, "building"); @@ -171,7 +171,7 @@ test_semantic_building(std::shared_ptr &plugin) { osmobjects::OsmNode node_place; node_place.id = 11111; - node_place.change_id = 22222; + node_place.changeset = 22222; // Has valid tags, but it's incomplete node_place.addTag("place", "city"); diff --git a/src/testsuite/testdata/stats/107235440.yaml b/src/testsuite/testdata/stats/107235440.yaml index ae6245de..81fbdd36 100644 --- a/src/testsuite/testdata/stats/107235440.yaml +++ b/src/testsuite/testdata/stats/107235440.yaml @@ -1,4 +1,4 @@ -- change_id: +- changeset: - 107235440 - modified_highway: - 8 diff --git a/src/testsuite/testdata/stats/highway.yaml b/src/testsuite/testdata/stats/highway.yaml index 92c44f9a..f6ce0077 100644 --- a/src/testsuite/testdata/stats/highway.yaml +++ b/src/testsuite/testdata/stats/highway.yaml @@ -1,4 +1,4 @@ -- change_id: +- changeset: - 1 - added_highway: - 2 diff --git a/src/testsuite/testdata/stats/test_stats.yaml b/src/testsuite/testdata/stats/test_stats.yaml index 676ced58..6623d303 100644 --- a/src/testsuite/testdata/stats/test_stats.yaml +++ b/src/testsuite/testdata/stats/test_stats.yaml @@ -1,4 +1,4 @@ -- change_id: +- changeset: - 1 - modified_highway: - 2 diff --git a/src/testsuite/testdata/stats/test_statsconfig2.yaml b/src/testsuite/testdata/stats/test_statsconfig2.yaml index a83e3e45..75f146b4 100644 --- a/src/testsuite/testdata/stats/test_statsconfig2.yaml +++ b/src/testsuite/testdata/stats/test_statsconfig2.yaml @@ -1,4 +1,4 @@ -- change_id: +- changeset: - 1 - added_building: - 1 diff --git a/src/testsuite/testdata/stats/test_statsconfig3.yaml b/src/testsuite/testdata/stats/test_statsconfig3.yaml index fdca3065..d0374c56 100644 --- a/src/testsuite/testdata/stats/test_statsconfig3.yaml +++ b/src/testsuite/testdata/stats/test_statsconfig3.yaml @@ -1,4 +1,4 @@ -- change_id: +- changeset: - 1 - added_building: - 2 diff --git a/src/testsuite/testdata/test_stats.yaml b/src/testsuite/testdata/test_stats.yaml index 7da36cd5..b150dc05 100644 --- a/src/testsuite/testdata/test_stats.yaml +++ b/src/testsuite/testdata/test_stats.yaml @@ -1,4 +1,4 @@ -- change_id: +- changeset: - 1 - modified_highway: - 2 diff --git a/src/underpass.cc b/src/underpass.cc index a3351af9..2de1e3dd 100644 --- a/src/underpass.cc +++ b/src/underpass.cc @@ -77,7 +77,7 @@ using namespace underpassconfig; using namespace logger; // Forward declarations -namespace changeset { +namespace changesets { class ChangeSet; }; diff --git a/src/validate/hotosm.cc b/src/validate/hotosm.cc index f1b58989..37639837 100644 --- a/src/validate/hotosm.cc +++ b/src/validate/hotosm.cc @@ -110,7 +110,7 @@ Hotosm::checkPOI(const osmobjects::OsmNode &node, const std::string &type) { auto status = std::make_shared(node); status->timestamp = boost::posix_time::microsec_clock::universal_time(); - status->user_id = node.uid; + status->uid = node.uid; if (yamls.size() == 0) { log_error("No config files!"); @@ -176,7 +176,7 @@ Hotosm::checkWay(const osmobjects::OsmWay &way, const std::string &type) auto status = std::make_shared(way); status->timestamp = boost::posix_time::microsec_clock::universal_time(); - status->user_id = way.uid; + status->uid = way.uid; if (yamls.size() == 0) { log_error("No config files!"); diff --git a/src/validate/queryvalidate.cc b/src/validate/queryvalidate.cc index b90a4fa3..dcbe8933 100644 --- a/src/validate/queryvalidate.cc +++ b/src/validate/queryvalidate.cc @@ -129,17 +129,17 @@ QueryValidate::applyChange(const ValidateStatus &validation, const valerror_t &s std::string query; if (validation.values.size() > 0) { - query = "INSERT INTO validation as v (osm_id, change_id, user_id, type, status, values, timestamp, location, source, version) VALUES("; + query = "INSERT INTO validation as v (osm_id, changeset, uid, type, status, values, timestamp, location, source, version) VALUES("; format = "%d, %d, %g, \'%s\', \'%s\', ARRAY[%s], \'%s\', ST_GeomFromText(\'%s\', 4326), \'%s\', %s) "; } else { - query = "INSERT INTO validation as v (osm_id, change_id, user_id, type, status, timestamp, location, source, version) VALUES("; + query = "INSERT INTO validation as v (osm_id, changeset, uid, type, status, timestamp, location, source, version) VALUES("; format = "%d, %d, %g, \'%s\', \'%s\', \'%s\', ST_GeomFromText(\'%s\', 4326), \'%s\', %s) "; } format += "ON CONFLICT (osm_id, status, source) DO UPDATE SET version = %d, timestamp = \'%s\' WHERE v.version < %d;"; boost::format fmt(format); fmt % validation.osm_id; - fmt % validation.change_id; - fmt % validation.user_id; + fmt % validation.changeset; + fmt % validation.uid; fmt % objtypes[validation.objtype]; std::string valtmp; fmt % status_list[status]; diff --git a/src/validate/validate.hh b/src/validate/validate.hh index ea0555e2..5ec3788a 100644 --- a/src/validate/validate.hh +++ b/src/validate/validate.hh @@ -104,16 +104,16 @@ class ValidateStatus { ValidateStatus(void){}; ValidateStatus(const osmobjects::OsmNode &node) { osm_id = node.id; - user_id = node.uid; - change_id = node.change_id; + uid = node.uid; + changeset = node.changeset; version = node.version; objtype = osmobjects::node; timestamp = node.timestamp; } ValidateStatus(const osmobjects::OsmWay &way) { osm_id = way.id; - user_id = way.uid; - change_id = way.change_id; + uid = way.uid; + changeset = way.changeset; objtype = osmobjects::way; version = way.version; timestamp = way.timestamp; @@ -130,8 +130,8 @@ class ValidateStatus { void dump(void) const { std::cerr << "Dumping Validation Statistics" << std::endl; std::cerr << "\tOSM ID: " << osm_id << std::endl; - std::cerr << "\tUser ID: " << user_id << std::endl; - std::cerr << "\tChange ID: " << change_id << std::endl; + std::cerr << "\tUser ID: " << uid << std::endl; + std::cerr << "\tChangeset: " << changeset << std::endl; std::map results; results[notags] = "No tags"; @@ -156,8 +156,8 @@ class ValidateStatus { std::unordered_set status; osmobjects::osmtype_t objtype; long osm_id = 0; ///< The OSM ID of the feature - long user_id = 0; ///< The user ID of the mapper creating/modifying this feature - long change_id = 0; ///< The changeset ID + long uid = 0; ///< The user ID of the mapper creating/modifying this feature + long changeset = 0; ///< The changeset ID long version = 0; ///< The object version ptime timestamp; ///< The timestamp when this validation was performed point_t center; ///< The centroid of the building polygon diff --git a/src/wrappers/python.cc b/src/wrappers/python.cc index 12ff30ea..83ac89ad 100644 --- a/src/wrappers/python.cc +++ b/src/wrappers/python.cc @@ -69,8 +69,8 @@ std::string dumpJSON(ValidateStatus& self) { std::string output = ""; output += "{\n"; output += "\t\"osm_id\":" + std::to_string(self.osm_id) + ",\n"; - output += "\t\"user_id\":" + std::to_string(self.user_id) + ",\n"; - output += "\t\"change_id\":" + std::to_string(self.change_id) + ",\n"; + output += "\t\"uid\":" + std::to_string(self.uid) + ",\n"; + output += "\t\"changeset\":" + std::to_string(self.changeset) + ",\n"; if (self.status.size() > 0) { output += "\t\"results\": ["; diff --git a/utils/bootstrap.sh b/utils/bootstrap.sh index 043e2581..4881ec99 100644 --- a/utils/bootstrap.sh +++ b/utils/bootstrap.sh @@ -23,9 +23,10 @@ # database with OSM data for a country # ----- -localfiles='false' +localfiles=false +use_docker=false -while getopts r:c:h::u:p:d:l flag +while getopts r:c:h::u:p:d:l:k flag do case "${flag}" in r) region=${OPTARG};; @@ -34,7 +35,8 @@ do u) user=${OPTARG};; p) port=${OPTARG};; d) database=${OPTARG};; - l) localfiles=${OPTARG};; + l) localfiles=true;; + k) use_docker=true;; esac done @@ -51,6 +53,7 @@ else USER=underpass fi + if [ -n "${REGION}" ] && [ -n "${COUNTRY}" ] then @@ -61,11 +64,16 @@ then echo Port: $PORT echo Database: $DB - if [ -z "${localfiles}" ] + if "$localfiles"; then echo "Use local files?: yes" fi + if "$use_docker"; + then + echo "Use Docker?: yes" + fi + echo " " echo "*** WARNING: THIS ACTION WILL OVERWRITE DATA IN THE CURRENT DATABASE ***" echo " " @@ -82,10 +90,10 @@ then fi echo "Cleaning database ..." - PGPASSWORD=$PASS psql --host $HOST --user $USER --port $PORT $DB -c 'DROP TABLE IF EXISTS raw_poly; DROP TABLE IF EXISTS raw_node; DROP TABLE IF EXISTS way_refs; DROP TABLE IF EXISTS validation; DROP TABLE IF EXISTS changesets;' + PGPASSWORD=$PASS psql --host $HOST --user $USER --port $PORT $DB -c 'DROP TABLE IF EXISTS ways_poly; DROP TABLE IF EXISTS ways_line; DROP TABLE IF EXISTS nodes; DROP TABLE IF EXISTS way_refs; DROP TABLE IF EXISTS validation; DROP TABLE IF EXISTS changesets;' PGPASSWORD=$PASS psql --host $HOST --user $USER --port $PORT $DB --file '../setup/underpass.sql' - if [ -z "${localfiles}" ] + if "$localfiles"; then echo "(Using local files)" else @@ -99,11 +107,22 @@ then PGPASSWORD=$PASS psql --host $HOST --user $USER --port $PORT $DB < raw-underpass.sql echo "Configuring Underpass ..." - python3 poly2geojson.py $COUNTRY.poly && \ - docker cp $COUNTRY.geojson underpass:/usr/local/lib/underpass/config/ - docker cp $COUNTRY.geojson underpass:/code/config + python3 poly2geojson.py $COUNTRY.poly + if "$use_docker"; + then + docker cp $COUNTRY.geojson underpass:/usr/local/lib/underpass/config/priority.geojson + docker cp $COUNTRY.geojson underpass:/code/config/priority.geojson + else + cp $COUNTRY.geojson /usr/local/lib/underpass/config/priority.geojson + cp $COUNTRY.geojson ../config/priority.geojson + fi echo "Bootstrapping database ..." - docker exec -w /code/build -t underpass ./underpass --bootstrap + if "$use_docker"; + then + docker exec -w /code/build -t underpass ./underpass --bootstrap + else + cd ../build && ./underpass --bootstrap + fi echo "Done." echo " " fi @@ -124,5 +143,6 @@ else echo " -u user (Database user)" echo " -p port (Database port)" echo " -d database (Database name)" - echo " -l (Use local files instead of download them)" + echo " -l yes (Use local files instead of download them)" + echo " -k yes (Use Docker Underpass installation)" fi diff --git a/utils/clean-osmchanges.sql b/utils/clean-osmchanges.sql index 430fc65a..78b7b97c 100644 --- a/utils/clean-osmchanges.sql +++ b/utils/clean-osmchanges.sql @@ -6,7 +6,7 @@ We run this query every 24 hours for cleaning the database when running the replicator process with areaFilter disabled for osmchanges (--osmnoboundary). */ -DELETE FROM validation v where v.change_id in ( +DELETE FROM validation v where v.changeset in ( SELECT id from changesets WHERE bbox is NULL AND closed_at < NOW() - INTERVAL '24 HOURS' diff --git a/utils/raw-underpass.lua b/utils/raw-underpass.lua index 578eb589..348f4a6c 100644 --- a/utils/raw-underpass.lua +++ b/utils/raw-underpass.lua @@ -1,98 +1,169 @@ -- Copyright (c) 2020, 2021, 2023 Humanitarian OpenStreetMap Team --- --- This file is part of Underpass. --- --- Underpass is free software: you can redistribute it and/or modify --- it under the terms of the GNU General Public License as published by --- the Free Software Foundation, either version 3 of the License, or --- (at your option) any later version. --- --- Underpass is distributed in the hope that it will be useful, --- but WITHOUT ANY WARRANTY; without even the implied warranty of --- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the --- GNU General Public License for more details. --- --- You should have received a copy of the GNU General Public License --- along with Underpass. If not, see . +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. + +-- This program is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU Affero General Public License for more details. + +-- You should have received a copy of the GNU Affero General Public License +-- along with this program. If not, see . + +-- Humanitarian OpenStreetmap Team +-- 1100 13th Street NW Suite 800 Washington, D.C. 20005 +-- -- This is lua script for osm2pgsql in order to create and process custom schema to store incoming osm data efficiently --- osm2pgsql -H postgis -U underpass -p underpass -P 5432 -d underpass --extra-attributes --output=flex --style ./raw-underpass.lua pokhara_all.osm.pbf +-- osm2pgsql --create -H localhost -U admin -P 5432 -d postgres -W --extra-attributes --output=flex --style ./raw.lua nepal-latest-internal.osm.pbf + -- Set projection to 4326 local srid = 4326 local tables = {} -tables.raw_node = osm2pgsql.define_table{ - name="raw_node", +tables.nodes = osm2pgsql.define_table{ + name="nodes", -- This will generate a derived nodes table which stores all the nodes feature with their point geometry - ids = {type='node',id_column = 'osm_id'}, + ids = {type='node',id_column = 'osm_id' }, columns = { + { column = 'uid', type = 'int' }, + { column = 'user', type = 'text' }, { column = 'version', type = 'int' }, + { column = 'changeset', type = 'int' }, { column = 'timestamp', sql_type = 'timestamp' }, - { column = 'tags', sql_type = 'public.hstore' }, - { column = 'geometry', type = 'point', projection = srid }, + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'point', projection = srid }, } } -tables.raw_poly = osm2pgsql.define_table{ - name="raw_poly", - -- This will generate a derived polygon table which stores all the ways feature without their geometry - ids = {type='way',id_column = 'osm_id'}, +tables.ways_line = osm2pgsql.define_table{ + name="ways_line", + -- This will generate a derived ways line table which stores all the ways feature with linestring geometry + ids = {type='way',id_column = 'osm_id' }, columns = { + { column = 'uid', type = 'int' }, + { column = 'user', type = 'text' }, { column = 'version', type = 'int' }, + { column = 'changeset', type = 'int' }, { column = 'timestamp', sql_type = 'timestamp' }, - { column = 'tags', sql_type = 'public.hstore' }, - { column = 'refs', type= 'text', sql_type = 'int8[]'}, - { column = 'geometry', type = 'polygon', projection = srid } + { column = 'tags', type = 'jsonb' }, + { column = 'refs', type= 'text', sql_type = 'bigint[]'}, + { column = 'geom', type = 'linestring', projection = srid }, + { column = 'country', sql_type= 'int[]', create_only = true }, } + } ---tables.raw_line = osm2pgsql.define_table{ --- name="raw_line", - -- This will generate a derived nodes table which stores all the nodes feature with their point geometry --- ids = {type='way',id_column = 'osm_id'}, --- columns = { --- { column = 'version', type = 'int' }, --- { column = 'timestamp', sql_type = 'timestamp' }, --- { column = 'tags', sql_type = 'public.hstore' }, --- { column = 'refs', type= 'text', sql_type = 'int8[]'}, --- } ---} - -function tags_to_hstore(tags) - local hstore = '' - for k,v in pairs(tags) do - hstore = hstore .. string.format('%s=>%s,', string.format('%q', k), string.format('%q', v)) - end - return hstore:sub(1, -2) - end +tables.ways_poly = osm2pgsql.define_table{ + name="ways_poly", + -- This will generate a derived ways poly table which stores all the ways feature with polygon geometry + ids = {type='way',id_column = 'osm_id' }, + columns = { + { column = 'uid', type = 'int' }, + { column = 'user', type = 'text' }, + { column = 'version', type = 'int' }, + { column = 'changeset', type = 'int' }, + { column = 'timestamp', sql_type = 'timestamp' }, + { column = 'tags', type = 'jsonb' }, + { column = 'refs', type= 'text', sql_type = 'bigint[]'}, + { column = 'geom', type = 'polygon', projection = srid }, + { column = 'grid', type = 'int', create_only = true }, + { column = 'country', sql_type= 'int[]', create_only = true }, + } + +} + +tables.rels = osm2pgsql.define_table{ + name="relations", + -- This will generate a derived realtion table which stores all the relation feature to query without storing meta data parts and members + + ids = {type='relation', id_column = 'osm_id' }, + columns = { + + { column = 'uid', type = 'int' }, + { column = 'user', type = 'text' }, + { column = 'version', type = 'int' }, + { column = 'changeset', type = 'int' }, + { column = 'timestamp', sql_type = 'timestamp' }, + { column = 'tags', type = 'jsonb' }, + { column = 'refs', type = 'jsonb'}, + { column = 'geom', type = 'geometry', projection = srid }, + { column = 'country',sql_type= 'int[]', create_only = true }, + } +} function osm2pgsql.process_node(object) - tables.raw_node:add_row({ + + tables.nodes:add_row({ + uid = object.uid, + user = object.user, version = object.version, - tags = tags_to_hstore(object.tags), + changeset = object.changeset, timestamp = os.date('!%Y-%m-%dT%H:%M:%SZ', object.timestamp), - geometry = { create = 'point' }, + tags = object.tags, + geom = { create = 'point' } }) end -function osm2pgsql.process_way(object) +function osm2pgsql.process_way(object) + if object.is_closed and #object.nodes>3 then - tables.raw_poly:add_row({ + tables.ways_poly:add_row({ + uid = object.uid, + user = object.user, + version = object.version, + changeset = object.changeset, + timestamp = os.date('!%Y-%m-%dT%H:%M:%SZ', object.timestamp), + tags = object.tags, + refs = '{' .. table.concat(object.nodes, ',') .. '}', + geom = { create = 'area' }, + + }) + else + tables.ways_line:add_row({ + uid = object.uid, + user = object.user, version = object.version, - tags = tags_to_hstore(object.tags), + changeset = object.changeset, timestamp = os.date('!%Y-%m-%dT%H:%M:%SZ', object.timestamp), + tags = object.tags, refs = '{' .. table.concat(object.nodes, ',') .. '}', - geometry = { create = 'area' }, + geom = { create = 'line' }, + }) - --else - -- tables.raw_line:add_row({ - -- version = object.version, - -- tags = tags_to_hstore(object.tags), - -- timestamp = os.date('!%Y-%m-%dT%H:%M:%SZ', object.timestamp), - -- refs = '{' .. table.concat(object.nodes, ',') .. '}', - -- }) end -end \ No newline at end of file +end + +function osm2pgsql.process_relation(object) + if object.tags.type == 'multipolygon' or object.tags.type == 'boundary' then + tables.rels:add_row({ + uid = object.uid, + user = object.user, + version = object.version, + changeset = object.changeset, + timestamp = os.date('!%Y-%m-%dT%H:%M:%SZ', object.timestamp), + tags = object.tags, + geom = { create = 'area' }, + refs = object.members + + }) + else + tables.rels:add_row({ + uid = object.uid, + user = object.user, + version = object.version, + changeset = object.changeset, + timestamp = os.date('!%Y-%m-%dT%H:%M:%SZ', object.timestamp), + tags = object.tags, + geom = { create = 'line' }, + refs = object.members + + }) + end +end + diff --git a/utils/raw-underpass.sql b/utils/raw-underpass.sql index d69ba8d7..6639ad24 100644 --- a/utils/raw-underpass.sql +++ b/utils/raw-underpass.sql @@ -1,10 +1,14 @@ -CREATE UNIQUE INDEX raw_osm_id_idx ON public.raw_node (osm_id); -CREATE UNIQUE INDEX raw_poly_osm_id_idx ON public.raw_poly (osm_id); +CREATE UNIQUE INDEX nodes_id_idx ON public.nodes (osm_id); +CREATE UNIQUE INDEX ways_poly_id_idx ON public.ways_poly (osm_id); +CREATE UNIQUE INDEX ways_line_id_idx ON public.ways_line(osm_id); + CREATE INDEX way_refs_node_id_idx ON public.way_refs (node_id); CREATE INDEX way_refs_way_id_idx ON public.way_refs (way_id); -CREATE INDEX node_version_idx ON public.raw_node (version); -CREATE INDEX way_version_idx ON public.raw_poly (version); +CREATE INDEX nodes_version_idx ON public.nodes (version); +CREATE INDEX ways_poly_version_idx ON public.ways_poly (version); +CREATE INDEX ways_line_version_idx ON public.ways_line (version); -CREATE INDEX node_timestamp_idx ON public.raw_node(timestamp DESC); -CREATE INDEX way_timestamp_idx ON public.raw_poly(timestamp DESC); +CREATE INDEX nodes_timestamp_idx ON public.nodes(timestamp DESC); +CREATE INDEX ways_poly_timestamp_idx ON public.ways_poly(timestamp DESC); +CREATE INDEX ways_line_timestamp_idx ON public.ways_line(timestamp DESC); diff --git a/utils/raw_with_ref.lua b/utils/raw_with_ref.lua index d7985604..05527cb7 100644 --- a/utils/raw_with_ref.lua +++ b/utils/raw_with_ref.lua @@ -1,20 +1,21 @@ -- Copyright (c) 2020, 2021, 2023 Humanitarian OpenStreetMap Team --- --- This file is part of Underpass. --- --- Underpass is free software: you can redistribute it and/or modify --- it under the terms of the GNU General Public License as published by --- the Free Software Foundation, either version 3 of the License, or --- (at your option) any later version. --- --- Underpass is distributed in the hope that it will be useful, --- but WITHOUT ANY WARRANTY; without even the implied warranty of --- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the --- GNU General Public License for more details. --- --- You should have received a copy of the GNU General Public License --- along with Underpass. If not, see . +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. + +-- This program is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU Affero General Public License for more details. + +-- You should have received a copy of the GNU Affero General Public License +-- along with this program. If not, see . + +-- Humanitarian OpenStreetmap Team +-- 1100 13th Street NW Suite 800 Washington, D.C. 20005 +-- -- This is lua script for osm2pgsql in order to create and process custom schema to store incoming osm data efficiently -- osm2pgsql --create -H localhost -U admin -P 5432 -d postgres -W --extra-attributes --output=flex --style ./raw.lua nepal-latest-internal.osm.pbf @@ -37,7 +38,7 @@ tables.nodes = osm2pgsql.define_table{ { column = 'timestamp', sql_type = 'timestamp' }, { column = 'tags', type = 'jsonb' }, { column = 'geom', type = 'point', projection = srid }, - { column = 'country', type= 'int', create_only = true }, + { column = 'country', sql_type= 'int[]', create_only = true }, } @@ -77,7 +78,7 @@ tables.ways_poly = osm2pgsql.define_table{ { column = 'refs', type= 'text', sql_type = 'bigint[]'}, { column = 'geom', type = 'polygon', projection = srid }, { column = 'grid', type = 'int', create_only = true }, - { column = 'country', type= 'int', create_only = true }, + { column = 'country', sql_type= 'int[]', create_only = true }, } } @@ -99,17 +100,14 @@ tables.rels = osm2pgsql.define_table{ { column = 'geom', type = 'geometry', projection = srid }, { column = 'country',sql_type= 'int[]', create_only = true }, - } } -- Returns true if there are no tags left. function clean_tags(tags) tags.odbl = nil - tags.created_by = nil - tags.source = nil + -- tags.created_by = nil tags['source:ref'] = nil - return next(tags) == nil end