+ Base destination dir for local cache

src/replicator/replication.cc

@@ -284,17 +284,19 @@ Planet::processData(const std::string &dest, std::vector<unsigned char> &data)
 
 // Download a file from planet
 RequestedFile
-Planet::downloadFile(const std::string &url)
+Planet::downloadFile(const std::string &url, const std::string &destdir_base)
 {
 
     RemoteURL remote(url);
+    remote.destdir_base = destdir_base;
     RequestedFile file;
+    std::string local_file_path = destdir_base + remote.filespec;
 
-    if (std::filesystem::exists(remote.filespec)) {
-        file = readFile(remote.filespec);
+    if (std::filesystem::exists(local_file_path)) {
+        file = readFile(local_file_path);
         // If local file doesn't work, remove it
         if (file.status == reqfile_t::localError) {
-            boost::filesystem::remove(remote.filespec);
+            boost::filesystem::remove(local_file_path);
         }
         return file;
     }
@@ -434,19 +436,20 @@ Planet::readFile(std::string &filespec) {
 }
 
 void Planet::writeFile(RemoteURL &remote, std::shared_ptr<std::vector<unsigned char>> data) {
+    std::string local_file_path = remote.destdir_base + remote.destdir;
     try {
-        if (!boost::filesystem::exists(remote.destdir)) {
-            boost::filesystem::create_directories(remote.destdir);
+        if (!boost::filesystem::exists(local_file_path)) {
+            boost::filesystem::create_directories(local_file_path);
         }
     } catch (boost::system::system_error ex) {
-        log_error("Destdir corrupted!: %1%, %2%", remote.destdir, ex.what());
+        log_error("Destdir corrupted!: %1%, %2%", local_file_path, ex.what());
     }
     std::ofstream myfile;
     myfile.open(remote.filespec, std::ofstream::out | std::ios::binary);
     myfile.write(reinterpret_cast<char *>(data.get()->data()), data.get()->size());
     myfile.flush();
     myfile.close();
-    log_debug("Wrote downloaded file %1% to disk from %2%", remote.filespec, remote.domain);
+    log_debug("Wrote downloaded file %1% to disk from %2%", local_file_path, remote.domain);
 }
 
 Planet::~Planet(void)
@@ -740,6 +743,7 @@ RemoteURL::operator=(const RemoteURL &inr)
     index = inr.index;
     filespec = inr.filespec;
     destdir = inr.destdir;
+    destdir_base = inr.destdir_base;
 
     return *this;
 }
@@ -761,6 +765,7 @@ RemoteURL::RemoteURL(const RemoteURL &inr)
     index = inr.index;
     filespec = inr.filespec;
     destdir = inr.destdir;
+    destdir_base = inr.destdir_base;
 }
 
 void
@@ -781,7 +786,7 @@ RemoteURL::dump(void)
     std::cerr << "\t\"minor\": " << minor << "," << std::endl;
     std::cerr << "\t\"index\": " <<  index << "," << std::endl;
     std::cerr << "\t\"filespec\": \"" << filespec << "\"," << std::endl;
-    std::cerr << "\t\"destdir\": \"" << destdir << "\"" << std::endl;
+    std::cerr << "\t\"destdir\": \"" << destdir_base + destdir << "\"" << std::endl;
     std::cerr << "}" << std::endl;
 
 }

src/replicator/replication.hh

@@ -222,8 +222,9 @@ class RemoteURL {
     int major;                ///< The first element in the numerical path
     int minor;            ///< The second element in the numerical path
     int index;            ///< The third element in the numerical path
-    std::string filespec;    ///< The full fiespec
+    std::string filespec;    ///< The full filespec
     std::string destdir;    ///< The local directory used to cache files
+    std::string destdir_base;    ///< A base local directory for caching files
     /// Dump internal data for debugging
     void dump(void);
     /// Increment the numerical part of the path by one file
@@ -283,10 +284,10 @@ class Planet {
     /// \param file the full URL or the path part of the URL (such as:
     /// "/replication/changesets/000/001/633.osm.gz"), the host part is taken from remote.domain.
     /// \return RequestedFile object, which includes data and status
-    RequestedFile downloadFile(const std::string &file);
+    RequestedFile downloadFile(const std::string &file, const std::string &destdir_base);
     RequestedFile downloadFile(const RemoteURL &remote) {
         std::string str = "https://" + remote.domain + "/" + remote.filespec;
-        return downloadFile(str);
+        return downloadFile(str, remote.destdir_base);
     };
 
     /// \brief readFile read a file from disk cache

src/replicator/threads.cc

@@ -182,8 +182,10 @@ startMonitorChangesets(std::shared_ptr<replication::RemoteURL> &remote,
                 (last_task->status == reqfile_t::remoteNotFound && !caughtUpWithNow)) {
                 remote->Increment();
             }
+            auto new_remote = std::make_shared<replication::RemoteURL>(remote->getURL());
+            new_remote->destdir_base = remote->destdir_base;
             auto task = boost::bind(threadChangeSet,
-                std::make_shared<replication::RemoteURL>(remote->getURL()),
+                new_remote,
                 std::ref(planets.front()),
                 std::ref(poly),
                 std::ref(tasks),
@@ -308,9 +310,10 @@ startMonitorChanges(std::shared_ptr<replication::RemoteURL> &remote,
                 (last_task->status == reqfile_t::remoteNotFound && !caughtUpWithNow)) {
                 remote->Increment();
             }
-
+            auto new_remote = std::make_shared<replication::RemoteURL>(remote->getURL());
+            new_remote->destdir_base = remote->destdir_base;
             OsmChangeTask osmChangeTask {
-                std::make_shared<replication::RemoteURL>(remote->getURL()),
+                new_remote,
                 std::ref(planets.front()),
                 std::ref(poly),
                 std::ref(validator),

src/underpass.cc

@@ -112,7 +112,8 @@ main(int argc, char *argv[])
             ("boundary,b", opts::value<std::string>(), "Boundary polygon file name")
             ("osmnoboundary", "Disable boundary polygon for OsmChanges")
             ("oscnoboundary", "Disable boundary polygon for Changesets")
-            ("datadir", opts::value<std::string>(), "Base directory for cached files (with ending slash)")
+            ("datadir", opts::value<std::string>(), "Directory for remote and local cached files (with ending slash)")
+            ("destdir_base", opts::value<std::string>(), "Base directory for local cached files (with ending slash)")
             ("verbose,v", "Enable verbosity")
             ("logstdout,l", "Enable logging to stdout, default is log to underpass.log")
             ("changefile", opts::value<std::string>(), "Import change file")
@@ -156,6 +157,10 @@ main(int argc, char *argv[])
         config.underpass_db_url = vm["server"].as<std::string>();
     }
 
+    if (vm.count("destdir_base")) {
+        config.destdir_base = vm["destdir_base"].as<std::string>();
+    }
+
     // Concurrency
     if (vm.count("concurrency")) {
         const auto concurrency = vm["concurrency"].as<std::string>();
@@ -275,6 +280,7 @@ main(int argc, char *argv[])
             // fullurl += "/" + vm["url"].as<std::string>() + "/" + parts[2] + ".state.txt";
             fullurl += "/" + vm["url"].as<std::string>() + ".state.txt";
             osmchange->parse(fullurl);
+            osmchange->destdir_base = config.destdir_base;
             auto data = replicator.downloadFile(*osmchange).data;
             StateFile start(osmchange->filespec, false);
             //start.dump();
@@ -291,11 +297,13 @@ main(int argc, char *argv[])
             if (!vm.count("osmnoboundary")) {
                 osmboundary = &geou.boundary;
             }
+            osmchange->destdir_base = config.destdir_base;
             osmChangeThread = std::thread(replicatorthreads::startMonitorChanges, std::ref(osmchange),
                             std::ref(*osmboundary), std::ref(config));
         }
         config.frequency = replication::changeset;
         auto changeset = replicator.findRemotePath(config, config.start_time);
+        changeset->destdir_base = config.destdir_base;
         if (vm.count("changeseturl")) {
             std::vector<std::string> parts;
             boost::split(parts, vm["changeseturl"].as<std::string>(), boost::is_any_of("/"));

src/underpassconfig.hh

@@ -119,11 +119,17 @@ struct UnderpassConfig {
                     );
                 }
             };
+            if (yaml.contains_key("destdir_base")) {
+                destdir_base = yamlConfig.get_value("destdir_base");
+            }
         }
 
         if (getenv("REPLICATOR_UNDERPASS_DB_URL")) {
             underpass_db_url = getenv("REPLICATOR_UNDERPASS_DB_URL");
         }
+        if (getenv("REPLICATOR_DESTDIR_BASE")) {
+            destdir_base = getenv("REPLICATOR_DESTDIR_BASE");
+        }
         if (getenv("REPLICATOR_PLANET_SERVER")) {
             planet_server = getenv("REPLICATOR_PLANET_SERVER");
         }
@@ -145,6 +151,7 @@ struct UnderpassConfig {
     };
 
     std::string underpass_db_url = "localhost/underpass";
+    std::string destdir_base;
     std::string planet_server;
     std::string datadir;
     std::vector<PlanetServer> planet_servers;