Skip to content
This repository has been archived by the owner on Oct 5, 2024. It is now read-only.

Commit

Permalink
Refactor, + nodes/relations support for bootstrapping
Browse files Browse the repository at this point in the history
  • Loading branch information
@emi420
emi420 committed Feb 28, 2024
1 parent 0393150 commit ef79e72
Show file tree
Hide file tree
Showing 12 changed files with 343 additions and 79 deletions.
4 changes: 2 additions & 2 deletions config/validate/building.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ config:
- badgeom_maxangle:
- 91
- badvalue:
- yes
- no
- incomplete:
- yes
- no


4 changes: 2 additions & 2 deletions setup/db/underpass.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ CREATE TABLE IF NOT EXISTS public.relations (
changeset int8,
geom public.geometry(Geometry,4326),
tags JSONB,
members int8[],
refs int8[],
timestamp timestamp with time zone,
version int,
"user" text,
Expand All @@ -111,7 +111,7 @@ CREATE TABLE IF NOT EXISTS public.way_refs (
node_id int8
);

CREATE TABLE IF NOT EXISTS public.rel_members (
CREATE TABLE IF NOT EXISTS public.rel_refs (
rel_id int8,
way_id int8
);
Expand Down
123 changes: 103 additions & 20 deletions src/bootstrap/bootstrap.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ Bootstrap::start(const underpassconfig::UnderpassConfig &config) {

processWays();
processNodes();
// processRels();
processRelations();

}

Expand All @@ -106,22 +106,20 @@ Bootstrap::processWays() {
};

for (auto table_it = tables.begin(); table_it != tables.end(); ++table_it) {
std::cout << std::endl << "Counting geometries ... " << std::endl;
std::cout << std::endl << "Processing ways ... ";
long int total = queryraw->getCount(*table_it);
long int count = 0;
int num_chunks = total / page_size;

std::cout << "Total: " << total << std::endl;
std::cout << "Threads: " << concurrency << std::endl;
std::cout << "Page size: " << page_size << std::endl;
long lastid = 0;

int concurrentTasks = concurrency;
int taskIndex = 0;
int percentage = 0;

for (int chunkIndex = 1; chunkIndex <= (num_chunks/concurrentTasks); chunkIndex++) {
for (int chunkIndex = 0; chunkIndex <= (num_chunks/concurrentTasks); chunkIndex++) {

int percentage = (count * 100) / total;
percentage = (count * 100) / total;

auto ways = std::make_shared<std::vector<OsmWay>>();
if (!norefs) {
Expand Down Expand Up @@ -152,6 +150,8 @@ Bootstrap::processWays() {
count += it->processed;
}
}
percentage = (count * 100) / total;
std::cout << "\r" << "Processing " << *table_it << ": " << count << "/" << total << " (" << percentage << "%)";
}
std::cout << std::endl;

Expand All @@ -160,22 +160,20 @@ Bootstrap::processWays() {
void
Bootstrap::processNodes() {

std::cout << std::endl << "Counting nodes ... " << std::endl;
std::cout << "Processing nodes ... ";
long int total = queryraw->getCount("nodes");
long int count = 0;
int num_chunks = total / page_size;

std::cout << "Total: " << total << std::endl;
std::cout << "Threads: " << concurrency << std::endl;
std::cout << "Page size: " << page_size << std::endl;
long lastid = 0;

int concurrentTasks = concurrency;
int taskIndex = 0;
int percentage = 0;

for (int chunkIndex = 1; chunkIndex <= (num_chunks/concurrentTasks); chunkIndex++) {
for (int chunkIndex = 0; chunkIndex <= (num_chunks/concurrentTasks); chunkIndex++) {

int percentage = (count * 100) / total;
percentage = (count * 100) / total;

auto nodes = std::make_shared<std::vector<OsmNode>>();
nodes = queryraw->getNodesFromDB(lastid, concurrency * page_size);
Expand All @@ -190,7 +188,6 @@ Bootstrap::processNodes() {
std::ref(nodes),
};
std::cout << "\r" << "Processing nodes: " << count << "/" << total << " (" << percentage << "%)";

boost::asio::post(pool, boost::bind(&Bootstrap::threadBootstrapNodeTask, this, nodeTask));
}

Expand All @@ -202,6 +199,57 @@ Bootstrap::processNodes() {
count += it->processed;
}
}
percentage = (count * 100) / total;
std::cout << "\r" << "Processing nodes: " << count << "/" << total << " (" << percentage << "%)";
std::cout << std::endl;

}

void
Bootstrap::processRelations() {

std::cout << "Processing relations ... ";
long int total = queryraw->getCount("relations");
long int count = 0;
int num_chunks = total / page_size;

long lastid = 0;

int concurrentTasks = concurrency;
int taskIndex = 0;
int percentage = 0;

for (int chunkIndex = 0; chunkIndex <= (num_chunks/concurrentTasks); chunkIndex++) {

percentage = (count * 100) / total;

auto relations = std::make_shared<std::vector<OsmRelation>>();
relations = queryraw->getRelationsFromDB(lastid, concurrency * page_size);

auto tasks = std::make_shared<std::vector<BootstrapTask>>(concurrentTasks);
boost::asio::thread_pool pool(concurrentTasks);
for (int taskIndex = 0; taskIndex < concurrentTasks; taskIndex++) {
auto taskRelations = std::make_shared<std::vector<OsmRelation>>();
RelationTask relationTask {
taskIndex,
std::ref(tasks),
std::ref(relations),
};
std::cout << "\r" << "Processing relations: " << count << "/" << total << " (" << percentage << "%)";
boost::asio::post(pool, boost::bind(&Bootstrap::threadBootstrapRelationTask, this, relationTask));
}

pool.join();

db->query(allTasksQueries(tasks));
lastid = relations->back().id;
for (auto it = tasks->begin(); it != tasks->end(); ++it) {
count += it->processed;
}
}
percentage = (count * 100) / total;
std::cout << "\r" << "Processing relations: " << count << "/" << total << " (" << percentage << "%)";

std::cout << std::endl;

}
Expand All @@ -223,9 +271,9 @@ Bootstrap::threadBootstrapWayTask(WayTask wayTask)
auto wayval = std::make_shared<std::vector<std::shared_ptr<ValidateStatus>>>();

// Proccesing ways
for (int i = 0; i < page_size; ++i) {
if (i * taskIndex < ways->size()) {
auto way = ways->at(i * (taskIndex + 1));
for (int i = taskIndex * page_size; i < (taskIndex + 1) * page_size; ++i) {
if (i < ways->size()) {
auto way = ways->at(i);
wayval->push_back(validator->checkWay(way, "building"));
// Fill the way_refs table
if (!norefs) {
Expand Down Expand Up @@ -261,9 +309,9 @@ Bootstrap::threadBootstrapNodeTask(NodeTask nodeTask)

// Proccesing nodes
std::vector<std::string> node_tests = {"building", "natural", "place", "waterway"};
for (int i = 0; i < page_size; ++i) {
if (i * taskIndex < nodes->size()) {
auto node = nodes->at(i * (taskIndex + 1));
for (int i = taskIndex * page_size; i < (taskIndex + 1) * page_size; ++i) {
if (i < nodes->size()) {
auto node = nodes->at(i);
for (auto test_it = std::begin(node_tests); test_it != std::end(node_tests); ++test_it) {
if (node.containsKey(*test_it)) {
nodeval->push_back(validator->checkNode(node, *test_it));
Expand All @@ -279,4 +327,39 @@ Bootstrap::threadBootstrapNodeTask(NodeTask nodeTask)

}

// This thread get started for every page of relation
void
Bootstrap::threadBootstrapRelationTask(RelationTask relationTask)
{
#ifdef TIMING_DEBUG
boost::timer::auto_cpu_timer timer("bootstrap::threadBootstrapRelationTask(relationTask): took %w seconds\n");
#endif
auto taskIndex = relationTask.taskIndex;
auto tasks = relationTask.tasks;
auto relations = relationTask.relations;

BootstrapTask task;
int processed = 0;

auto relationval = std::make_shared<std::vector<std::shared_ptr<ValidateStatus>>>();

// Proccesing relations
for (int i = taskIndex * page_size; i < (taskIndex + 1) * page_size; ++i) {
if (i < relations->size()) {
auto relation = relations->at(i);
// relationval->push_back(validator->checkRelation(way, "building"));
// Fill the rel_members table
// for (auto ref = relation.refs.begin(); ref != relation.refs.end(); ++ref) {
// task.query += "INSERT INTO rel_refs (rel_id, way_id) VALUES (" + std::to_string(rel.id) + "," + std::to_string(*ref) + "); ";
// }
++processed;
}
}
// queryvalidate->relations(relval, task.query);
task.processed = processed;
const std::lock_guard<std::mutex> lock(tasks_change_mutex);
(*tasks)[taskIndex] = task;

}

}
7 changes: 7 additions & 0 deletions src/bootstrap/bootstrap.hh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,11 @@ struct NodeTask {
std::shared_ptr<std::vector<OsmNode>> nodes;
};

struct RelationTask {
int taskIndex;
std::shared_ptr<std::vector<BootstrapTask>> tasks;
std::shared_ptr<std::vector<OsmRelation>> relations;
};

class Bootstrap {
public:
Expand All @@ -61,10 +66,12 @@ class Bootstrap {
void start(const underpassconfig::UnderpassConfig &config);
void processWays();
void processNodes();
void processRelations();

// This thread get started for every page of way
void threadBootstrapWayTask(WayTask wayTask);
void threadBootstrapNodeTask(NodeTask nodeTask);
void threadBootstrapRelationTask(RelationTask relationTask);
std::string allTasksQueries(std::shared_ptr<std::vector<BootstrapTask>> tasks);

std::shared_ptr<Validate> validator;
Expand Down
8 changes: 5 additions & 3 deletions src/osm/osmobjects.hh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ typedef boost::geometry::model::d2::point_xy<double> point_t;
typedef boost::geometry::model::polygon<point_t> polygon_t;
typedef boost::geometry::model::multi_polygon<polygon_t> multipolygon_t;
typedef boost::geometry::model::linestring<point_t> linestring_t;
typedef boost::geometry::model::multi_linestring<linestring_t> multilinestring_t;
typedef boost::geometry::model::segment<point_t> segment_t;
typedef boost::geometry::model::point<double, 2, boost::geometry::cs::spherical_equatorial<boost::geometry::degree>> sphere_t;

Expand Down Expand Up @@ -174,9 +175,6 @@ class OsmWay : public OsmObject {
/// Return the number of nodes in this way
int numPoints(void) { return boost::geometry::num_points(linestring); };

/// Add a point to the way's geometric data storage
// void makeLinestring(point_t point);

/// Calculate the length of the linestring in Kilometers
double getLength(void)
{
Expand Down Expand Up @@ -220,6 +218,10 @@ class OsmRelation : public OsmObject {
public:
OsmRelation(void) { type = relation; };

multilinestring_t multilinestring; ///< Store the members as a linestring
multipolygon_t multipolygon; ///< Store the members as a multipolygon
point_t center; ///< Store the centroid of the relation

/// Add a member to this relation
void addMember(long ref, osmtype_t _type, const std::string role) { members.push_back({ref, _type, role}); };

Expand Down
Loading

0 comments on commit ef79e72

Please sign in to comment.