Get rid of regexp

Author: kssenii

programs/static-files-disk-uploader/static-files-disk-uploader.cpp

@@ -15,8 +15,7 @@
 
 namespace fs = std::filesystem;
 
-#define UUID_PATTERN "[\\w]{8}-[\\w]{4}-[\\w]{4}-[\\w]{4}-[\\w]{12}"
-#define EXTRACT_UUID_PATTERN fmt::format(".*\\/({})\\/.*", UUID_PATTERN)
+#define EXTRACT_PATH_PATTERN ".*\\/store/(.*)"
 
 
 namespace DB
@@ -32,40 +31,102 @@ namespace ErrorCodes
  * If test-mode option is added, files will be put by given url via PUT request.
  */
 
-void processTableFiles(const fs::path & path, String root_path, String uuid,
-        WriteBuffer & metadata_buf, std::function<std::shared_ptr<WriteBuffer>(const String &)> create_dst_buf)
+void processFile(const fs::path & file_path, const fs::path & dst_path, bool test_mode, WriteBuffer & metadata_buf)
 {
-    fs::directory_iterator dir_end;
-    auto process_file = [&](const String & file_name, const String & file_path)
+    String remote_path;
+    RE2::FullMatch(file_path.string(), EXTRACT_PATH_PATTERN, &remote_path);
+    bool is_directory = fs::is_directory(file_path);
+
+    writeText(file_path.filename(), metadata_buf);
+    writeChar('\t', metadata_buf);
+    writeBoolText(is_directory, metadata_buf);
+    if (!is_directory)
     {
-        auto remote_file_name = uuid + "/" + file_name;
-        writeText(remote_file_name, metadata_buf);
         writeChar('\t', metadata_buf);
         writeIntText(fs::file_size(file_path), metadata_buf);
-        writeChar('\n', metadata_buf);
+    }
+    writeChar('\n', metadata_buf);
+
+    if (is_directory)
+        return;
+
+    auto dst_file_path = fs::path(dst_path) / remote_path;
+
+    auto src_buf = createReadBufferFromFileBase(file_path, {}, fs::file_size(file_path));
+    std::shared_ptr<WriteBuffer> dst_buf;
+
+    /// test mode for integration tests.
+    if (test_mode)
+        dst_buf = std::make_shared<WriteBufferFromHTTP>(Poco::URI(dst_file_path), Poco::Net::HTTPRequest::HTTP_PUT);
+    else
+        dst_buf = std::make_shared<WriteBufferFromFile>(dst_file_path);
+
+    copyData(*src_buf, *dst_buf);
+    dst_buf->next();
+    dst_buf->finalize();
+};
 
-        auto src_buf = createReadBufferFromFileBase(file_path, {}, fs::file_size(file_path));
-        fs::create_directories((fs::path(root_path) / remote_file_name).parent_path());
-        auto dst_buf = create_dst_buf(remote_file_name);
 
-        copyData(*src_buf, *dst_buf);
-        dst_buf->next();
-        dst_buf->finalize();
-    };
+void processTableFiles(const fs::path & data_path, fs::path dst_path, bool test_mode)
+{
+    std::cerr << "Data path: " << data_path << ", destination path: " << dst_path << std::endl;
+
+    String prefix;
+    RE2::FullMatch(data_path.string(), EXTRACT_PATH_PATTERN, &prefix);
+
+    std::shared_ptr<WriteBuffer> root_meta;
+    if (test_mode)
+    {
+        dst_path /= "store";
+        auto files_root = dst_path / prefix;
+        root_meta = std::make_shared<WriteBufferFromHTTP>(Poco::URI(files_root / ".index"), Poco::Net::HTTPRequest::HTTP_PUT);
+    }
+    else
+    {
+        dst_path = fs::canonical(dst_path);
+        auto files_root = dst_path / prefix;
+        fs::create_directories(files_root);
+        root_meta = std::make_shared<WriteBufferFromFile>(files_root / ".index");
+    }
 
-    for (fs::directory_iterator dir_it(path); dir_it != dir_end; ++dir_it)
+    fs::directory_iterator dir_end;
+    for (fs::directory_iterator dir_it(data_path); dir_it != dir_end; ++dir_it)
     {
         if (dir_it->is_directory())
         {
+            processFile(dir_it->path(), dst_path, test_mode, *root_meta);
+
+            String directory_prefix;
+            RE2::FullMatch(dir_it->path().string(), EXTRACT_PATH_PATTERN, &directory_prefix);
+
+            std::shared_ptr<WriteBuffer> directory_meta;
+            if (test_mode)
+            {
+                auto files_root = dst_path / prefix;
+                directory_meta = std::make_shared<WriteBufferFromHTTP>(Poco::URI(dst_path / directory_prefix / ".index"), Poco::Net::HTTPRequest::HTTP_PUT);
+            }
+            else
+            {
+                dst_path = fs::canonical(dst_path);
+                auto files_root = dst_path / prefix;
+                fs::create_directories(dst_path / directory_prefix);
+                directory_meta = std::make_shared<WriteBufferFromFile>(dst_path / directory_prefix / ".index");
+            }
+
             fs::directory_iterator files_end;
             for (fs::directory_iterator file_it(dir_it->path()); file_it != files_end; ++file_it)
-                process_file(dir_it->path().filename() / file_it->path().filename(), file_it->path());
+                processFile(file_it->path(), dst_path, test_mode, *directory_meta);
+
+            directory_meta->next();
+            directory_meta->finalize();
         }
         else
         {
-            process_file(dir_it->path().filename(), dir_it->path());
+            processFile(dir_it->path(), dst_path, test_mode, *root_meta);
         }
     }
+    root_meta->next();
+    root_meta->finalize();
 }
 }
 
@@ -94,7 +155,7 @@ try
         exit(0);
     }
 
-    String url, metadata_path;
+    String metadata_path;
 
     if (options.count("metadata-path"))
         metadata_path = options["metadata-path"].as<std::string>();
@@ -108,46 +169,24 @@ try
         return 1;
     }
 
-    String uuid;
-    if (!RE2::Extract(metadata_path, EXTRACT_UUID_PATTERN, "\\1", &uuid))
-        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot extract uuid for: {}", metadata_path);
-
-    std::shared_ptr<WriteBuffer> metadata_buf;
-    std::function<std::shared_ptr<WriteBuffer>(const String &)> create_dst_buf;
     String root_path;
-
     auto test_mode = options.contains("test-mode");
     if (test_mode)
     {
         if (options.count("url"))
-            url = options["url"].as<std::string>();
+            root_path = options["url"].as<std::string>();
         else
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "No url option passed for test mode");
-
-        metadata_buf = std::make_shared<WriteBufferFromHTTP>(Poco::URI(fs::path(url) / (".index-" + uuid)), Poco::Net::HTTPRequest::HTTP_PUT);
-
-        create_dst_buf = [&](const String & remote_file_name)
-        {
-            return std::make_shared<WriteBufferFromHTTP>(Poco::URI(fs::path(url) / remote_file_name), Poco::Net::HTTPRequest::HTTP_PUT);
-        };
     }
     else
     {
         if (options.count("output-dir"))
             root_path = options["output-dir"].as<std::string>();
         else
             root_path = fs::current_path();
-
-        metadata_buf = std::make_shared<WriteBufferFromFile>(fs::path(root_path) / (".index-" + uuid));
-        create_dst_buf = [&](const String & remote_file_name)
-        {
-            return std::make_shared<WriteBufferFromFile>(fs::path(root_path) / remote_file_name);
-        };
     }
 
-    processTableFiles(fs_path, root_path, uuid, *metadata_buf, create_dst_buf);
-    metadata_buf->next();
-    metadata_buf->finalize();
+    processTableFiles(fs_path, root_path, test_mode);
 
     return 0;
 }