diff --git a/setup/service/restart.sh b/setup/service/restart.sh old mode 100644 new mode 100755 diff --git a/setup/service/start.sh b/setup/service/start.sh old mode 100644 new mode 100755 diff --git a/src/bootstrap/bootstrap.cc b/src/bootstrap/bootstrap.cc index 4d1710c6..13b4c6b1 100644 --- a/src/bootstrap/bootstrap.cc +++ b/src/bootstrap/bootstrap.cc @@ -165,10 +165,8 @@ threadBootstrapTask(WayTask wayTask) // Proccesing ways for (int i = 0; i < PAGE_SIZE; ++i) { - // std::cout << "--> processing way " << i * taskIndex << std::endl; if (i * taskIndex < ways->size()) { auto way = ways->at(i * (taskIndex + 1)); - // std::cout << "[task " << taskIndex << "] way " << i * (taskIndex + 1) << std::endl; auto status = plugin->checkWay(way, "building"); for (auto status_it = status->status.begin(); status_it != status->status.end(); ++status_it) { task.query += queryvalidate->applyChange(*status, *status_it); diff --git a/src/data/pq.cc b/src/data/pq.cc index 5f5f9bf1..2125ac74 100644 --- a/src/data/pq.cc +++ b/src/data/pq.cc @@ -157,44 +157,24 @@ Pq::query(const std::string &query) return result; } -std::string Latin1ToUTF8(const std::string& latin1str) { - std::string utf8str; - for (char c : latin1str) { - if (static_cast(c) <= 0x7F) { - utf8str.push_back(c); - } else { - utf8str.push_back(0xC0 | static_cast(c) >> 6); - utf8str.push_back(0x80 | (static_cast(c) & 0x3F)); - } - } - return utf8str; -} - std::string Pq::escapedString(const std::string &s) { std::string newstr; int i = 0; while (i < s.size()) { + // Single quote (') if (s[i] == '\'') { - newstr += "'"; - } else if (s[i] == '\"') { - newstr += """; - } else if (s[i] == '\'') { - newstr += """; - } else if (s[i] == ')') { - newstr += ")"; - } else if (s[i] == '(') { - newstr += "("; - } else if ((s[i] == '\\') || (s[i] == '\n')) { - // drop this character + newstr += "''"; + // Slash (\) + } else if (s[i] == '\\') { + newstr += "\\\\"; } else { newstr += s[i]; } i++; } - - return sdb->esc(Latin1ToUTF8(newstr)); + return sdb->esc(newstr); } std::string diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc index 397d303d..c75738ab 100644 --- a/src/raw/queryraw.cc +++ b/src/raw/queryraw.cc @@ -62,6 +62,32 @@ QueryRaw::QueryRaw(std::shared_ptr db) { dbconn = db; } +std::string +QueryRaw::buildTagsQuery(std::map tags) const { + if (tags.size() > 0) { + std::string tagsStr = "jsonb_build_object("; + int count = 0; + for (auto it = std::begin(tags); it != std::end(tags); ++it) { + ++count; + // PostgreSQL has an argument limit for functions + if (count == 50) { + tagsStr.erase(tagsStr.size() - 1); + tagsStr += ") || jsonb_build_object("; + count = 0; + } + std::string tag_format = "'%s', '%s',"; + boost::format tag_fmt(tag_format); + tag_fmt % dbconn->escapedString(dbconn->escapedJSON(it->first)); + tag_fmt % dbconn->escapedString(dbconn->escapedJSON(it->second)); + tagsStr += tag_fmt.str(); + } + tagsStr.erase(tagsStr.size() - 1); + return tagsStr + ")"; + } else { + return "null"; + } +} + std::string QueryRaw::applyChange(const OsmNode &node) const { @@ -83,21 +109,7 @@ QueryRaw::applyChange(const OsmNode &node) const fmt % geometry; // tags - std::string tags = ""; - if (node.tags.size() > 0) { - for (auto it = std::begin(node.tags); it != std::end(node.tags); ++it) { - std::string tag_format = "\"%s\" : \"%s\","; - boost::format tag_fmt(tag_format); - tag_fmt % dbconn->escapedString(dbconn->escapedJSON(it->first)); - tag_fmt % dbconn->escapedString(dbconn->escapedJSON(it->second)); - tags += tag_fmt.str(); - } - tags.erase(tags.size() - 1); - tags = "'{" + tags + "}'"; - - } else { - tags = "null"; - } + auto tags = buildTagsQuery(node.tags); fmt % tags; // timestamp std::string timestamp = to_simple_string(boost::posix_time::microsec_clock::universal_time()); @@ -164,22 +176,8 @@ QueryRaw::applyChange(const OsmWay &way) const // osm_id fmt % way.id; - // tags - std::string tags = ""; - if (way.tags.size() > 0) { - for (auto it = std::begin(way.tags); it != std::end(way.tags); ++it) { - std::string tag_format = "\"%s\" : \"%s\","; - boost::format tag_fmt(tag_format); - tag_fmt % dbconn->escapedString(dbconn->escapedJSON(it->first)); - tag_fmt % dbconn->escapedString(dbconn->escapedJSON(it->second)); - tags += tag_fmt.str(); - } - tags.erase(tags.size() - 1); - tags = "'{" + tags + "}'"; - } else { - tags = "null"; - } - + //tags + auto tags = buildTagsQuery(way.tags); fmt % tags; // refs diff --git a/src/raw/queryraw.hh b/src/raw/queryraw.hh index 9a4820b3..20c99eda 100644 --- a/src/raw/queryraw.hh +++ b/src/raw/queryraw.hh @@ -33,6 +33,7 @@ #endif #include +#include #include "data/pq.hh" #include "osm/osmobjects.hh" #include "osm/osmchange.hh" @@ -75,6 +76,8 @@ class QueryRaw { std::shared_ptr dbconn; // Get ways count int getWaysCount(const std::string &tableName); + // Build tags query + std::string buildTagsQuery(std::map tags) const; // Get ways by page std::shared_ptr> getWaysFromDB(long lastid, int pageSize, const std::string &tableName); std::shared_ptr> getWaysFromDBWithoutRefs(long lastid, int pageSize, const std::string &tableName); diff --git a/src/stats/querystats.cc b/src/stats/querystats.cc index 21011723..30829923 100644 --- a/src/stats/querystats.cc +++ b/src/stats/querystats.cc @@ -71,7 +71,6 @@ QueryStats::applyChange(const osmchange::ChangeStats &change) const #ifdef TIMING_DEBUG_X boost::timer::auto_cpu_timer timer("applyChange(statistics): took %w seconds\n"); #endif - // std::cout << "Applying OsmChange data" << std::endl; if (change.closed_at != not_a_date_time) { diff --git a/src/underpass.cc b/src/underpass.cc index e4028ae8..9a5f4977 100644 --- a/src/underpass.cc +++ b/src/underpass.cc @@ -349,7 +349,6 @@ main(int argc, char *argv[]) std::cout << "the environment variable names and their current " "values (possibly defaults)." << std::endl; - // std::cout << config.dbConfigHelp() << std::endl; exit(0); } diff --git a/src/validate/semantic.cc b/src/validate/semantic.cc index 4ea26dae..ac5a955b 100644 --- a/src/validate/semantic.cc +++ b/src/validate/semantic.cc @@ -86,6 +86,12 @@ Semantic::checkTag(const std::string &key, const std::string &value, std::shared status->status.insert(badvalue); status->values.insert(key + "=" + value); } + // Check for a underscore at the beginning of the tag key + if(key.at(0) == '_') { + log_error("WARNING: underscore at the beginning of the tag key \"%1%\"", key); + status->status.insert(badvalue); + status->values.insert(key + "=" + value); + } } bool Semantic::isValidTag(const std::string &key, const std::string &value, yaml::Node tags) {