From f039ceada3e340f4cc600d3fa5e3728c7a8d167f Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Mon, 19 Feb 2024 17:40:21 -0300 Subject: [PATCH] + Base destination dir for local cache --- src/replicator/replication.cc | 23 ++++++++++++++--------- src/replicator/replication.hh | 7 ++++--- src/replicator/threads.cc | 9 ++++++--- src/underpass.cc | 10 +++++++++- src/underpassconfig.hh | 7 +++++++ 5 files changed, 40 insertions(+), 16 deletions(-) diff --git a/src/replicator/replication.cc b/src/replicator/replication.cc index 1495627b..5e63e089 100644 --- a/src/replicator/replication.cc +++ b/src/replicator/replication.cc @@ -284,17 +284,19 @@ Planet::processData(const std::string &dest, std::vector &data) // Download a file from planet RequestedFile -Planet::downloadFile(const std::string &url) +Planet::downloadFile(const std::string &url, const std::string &destdir_base) { RemoteURL remote(url); + remote.destdir_base = destdir_base; RequestedFile file; + std::string local_file_path = destdir_base + remote.filespec; - if (std::filesystem::exists(remote.filespec)) { - file = readFile(remote.filespec); + if (std::filesystem::exists(local_file_path)) { + file = readFile(local_file_path); // If local file doesn't work, remove it if (file.status == reqfile_t::localError) { - boost::filesystem::remove(remote.filespec); + boost::filesystem::remove(local_file_path); } return file; } @@ -434,19 +436,20 @@ Planet::readFile(std::string &filespec) { } void Planet::writeFile(RemoteURL &remote, std::shared_ptr> data) { + std::string local_file_path = remote.destdir_base + remote.destdir; try { - if (!boost::filesystem::exists(remote.destdir)) { - boost::filesystem::create_directories(remote.destdir); + if (!boost::filesystem::exists(local_file_path)) { + boost::filesystem::create_directories(local_file_path); } } catch (boost::system::system_error ex) { - log_error("Destdir corrupted!: %1%, %2%", remote.destdir, ex.what()); + log_error("Destdir corrupted!: %1%, %2%", local_file_path, ex.what()); } std::ofstream myfile; myfile.open(remote.filespec, std::ofstream::out | std::ios::binary); myfile.write(reinterpret_cast(data.get()->data()), data.get()->size()); myfile.flush(); myfile.close(); - log_debug("Wrote downloaded file %1% to disk from %2%", remote.filespec, remote.domain); + log_debug("Wrote downloaded file %1% to disk from %2%", local_file_path, remote.domain); } Planet::~Planet(void) @@ -740,6 +743,7 @@ RemoteURL::operator=(const RemoteURL &inr) index = inr.index; filespec = inr.filespec; destdir = inr.destdir; + destdir_base = inr.destdir_base; return *this; } @@ -761,6 +765,7 @@ RemoteURL::RemoteURL(const RemoteURL &inr) index = inr.index; filespec = inr.filespec; destdir = inr.destdir; + destdir_base = inr.destdir_base; } void @@ -781,7 +786,7 @@ RemoteURL::dump(void) std::cerr << "\t\"minor\": " << minor << "," << std::endl; std::cerr << "\t\"index\": " << index << "," << std::endl; std::cerr << "\t\"filespec\": \"" << filespec << "\"," << std::endl; - std::cerr << "\t\"destdir\": \"" << destdir << "\"" << std::endl; + std::cerr << "\t\"destdir\": \"" << destdir_base + destdir << "\"" << std::endl; std::cerr << "}" << std::endl; } diff --git a/src/replicator/replication.hh b/src/replicator/replication.hh index 9454deda..e59c0c1f 100644 --- a/src/replicator/replication.hh +++ b/src/replicator/replication.hh @@ -222,8 +222,9 @@ class RemoteURL { int major; ///< The first element in the numerical path int minor; ///< The second element in the numerical path int index; ///< The third element in the numerical path - std::string filespec; ///< The full fiespec + std::string filespec; ///< The full filespec std::string destdir; ///< The local directory used to cache files + std::string destdir_base; ///< A base local directory for caching files /// Dump internal data for debugging void dump(void); /// Increment the numerical part of the path by one file @@ -283,10 +284,10 @@ class Planet { /// \param file the full URL or the path part of the URL (such as: /// "/replication/changesets/000/001/633.osm.gz"), the host part is taken from remote.domain. /// \return RequestedFile object, which includes data and status - RequestedFile downloadFile(const std::string &file); + RequestedFile downloadFile(const std::string &file, const std::string &destdir_base); RequestedFile downloadFile(const RemoteURL &remote) { std::string str = "https://" + remote.domain + "/" + remote.filespec; - return downloadFile(str); + return downloadFile(str, remote.destdir_base); }; /// \brief readFile read a file from disk cache diff --git a/src/replicator/threads.cc b/src/replicator/threads.cc index 0686f70a..670d2565 100644 --- a/src/replicator/threads.cc +++ b/src/replicator/threads.cc @@ -182,8 +182,10 @@ startMonitorChangesets(std::shared_ptr &remote, (last_task->status == reqfile_t::remoteNotFound && !caughtUpWithNow)) { remote->Increment(); } + auto new_remote = std::make_shared(remote->getURL()); + new_remote->destdir_base = remote->destdir_base; auto task = boost::bind(threadChangeSet, - std::make_shared(remote->getURL()), + new_remote, std::ref(planets.front()), std::ref(poly), std::ref(tasks), @@ -308,9 +310,10 @@ startMonitorChanges(std::shared_ptr &remote, (last_task->status == reqfile_t::remoteNotFound && !caughtUpWithNow)) { remote->Increment(); } - + auto new_remote = std::make_shared(remote->getURL()); + new_remote->destdir_base = remote->destdir_base; OsmChangeTask osmChangeTask { - std::make_shared(remote->getURL()), + new_remote, std::ref(planets.front()), std::ref(poly), std::ref(validator), diff --git a/src/underpass.cc b/src/underpass.cc index 07c7a9e3..843193e3 100644 --- a/src/underpass.cc +++ b/src/underpass.cc @@ -112,7 +112,8 @@ main(int argc, char *argv[]) ("boundary,b", opts::value(), "Boundary polygon file name") ("osmnoboundary", "Disable boundary polygon for OsmChanges") ("oscnoboundary", "Disable boundary polygon for Changesets") - ("datadir", opts::value(), "Base directory for cached files (with ending slash)") + ("datadir", opts::value(), "Directory for remote and local cached files (with ending slash)") + ("destdir_base", opts::value(), "Base directory for local cached files (with ending slash)") ("verbose,v", "Enable verbosity") ("logstdout,l", "Enable logging to stdout, default is log to underpass.log") ("changefile", opts::value(), "Import change file") @@ -156,6 +157,10 @@ main(int argc, char *argv[]) config.underpass_db_url = vm["server"].as(); } + if (vm.count("destdir_base")) { + config.destdir_base = vm["destdir_base"].as(); + } + // Concurrency if (vm.count("concurrency")) { const auto concurrency = vm["concurrency"].as(); @@ -275,6 +280,7 @@ main(int argc, char *argv[]) // fullurl += "/" + vm["url"].as() + "/" + parts[2] + ".state.txt"; fullurl += "/" + vm["url"].as() + ".state.txt"; osmchange->parse(fullurl); + osmchange->destdir_base = config.destdir_base; auto data = replicator.downloadFile(*osmchange).data; StateFile start(osmchange->filespec, false); //start.dump(); @@ -291,11 +297,13 @@ main(int argc, char *argv[]) if (!vm.count("osmnoboundary")) { osmboundary = &geou.boundary; } + osmchange->destdir_base = config.destdir_base; osmChangeThread = std::thread(replicatorthreads::startMonitorChanges, std::ref(osmchange), std::ref(*osmboundary), std::ref(config)); } config.frequency = replication::changeset; auto changeset = replicator.findRemotePath(config, config.start_time); + changeset->destdir_base = config.destdir_base; if (vm.count("changeseturl")) { std::vector parts; boost::split(parts, vm["changeseturl"].as(), boost::is_any_of("/")); diff --git a/src/underpassconfig.hh b/src/underpassconfig.hh index 6ddaa2a4..acb288eb 100644 --- a/src/underpassconfig.hh +++ b/src/underpassconfig.hh @@ -119,11 +119,17 @@ struct UnderpassConfig { ); } }; + if (yaml.contains_key("destdir_base")) { + destdir_base = yamlConfig.get_value("destdir_base"); + } } if (getenv("REPLICATOR_UNDERPASS_DB_URL")) { underpass_db_url = getenv("REPLICATOR_UNDERPASS_DB_URL"); } + if (getenv("REPLICATOR_DESTDIR_BASE")) { + destdir_base = getenv("REPLICATOR_DESTDIR_BASE"); + } if (getenv("REPLICATOR_PLANET_SERVER")) { planet_server = getenv("REPLICATOR_PLANET_SERVER"); } @@ -145,6 +151,7 @@ struct UnderpassConfig { }; std::string underpass_db_url = "localhost/underpass"; + std::string destdir_base; std::string planet_server; std::string datadir; std::vector planet_servers;