build error fix

microsoft · Oct 5, 2023 · 392c0ec · 392c0ec
1 parent eded185
commit 392c0ec
Show file tree

Hide file tree

Showing 8 changed files with 42 additions and 22 deletions.
diff --git a/apps/test_insert_deletes_consolidate.cpp b/apps/test_insert_deletes_consolidate.cpp
@@ -189,8 +189,7 @@ void build_incremental_index(const std::string &data_path, diskann::IndexWritePa
 
     if (universal_label != "")
     {
-        std::vector<std::string> unv_labels = {universal_label};
-        index->set_universal_labels(unv_labels);
+        index->set_universal_labels({universal_label}, /*dynamic index*/ true);
     }
 
     if (points_to_skip > num_points)

diff --git a/apps/test_streaming_scenario.cpp b/apps/test_streaming_scenario.cpp
@@ -254,7 +254,7 @@ void build_incremental_index(const std::string &data_path, const uint32_t L, con
 
     if (universal_label != "")
     {
-        index->set_universal_labels({universal_label});
+        index->set_universal_labels({universal_label}, true);
     }
 
     if (max_points_to_insert == 0)

diff --git a/include/abstract_filter_store.h b/include/abstract_filter_store.h
@@ -43,7 +43,8 @@ template <typename label_type> class AbstractFilterStore
 
     // TODO: in future we may accept a set or vector of universal labels
     // DISKANN_DLLEXPORT virtual void set_universal_label(label_type universal_label) = 0;
-    DISKANN_DLLEXPORT virtual void set_universal_labels(const std::vector<std::string> &universal_labels) = 0;
+    DISKANN_DLLEXPORT virtual void set_universal_labels(const std::vector<std::string> &universal_labels,
+                                                        bool dynamic_index = false) = 0;
     // DISKANN_DLLEXPORT virtual const label_type get_universal_label() const = 0;
 
     // takes raw label file and then genrate internal mapping file and keep the info of mapping

diff --git a/include/abstract_index.h b/include/abstract_index.h
@@ -103,7 +103,8 @@ class AbstractIndex
     // memory should be allocated for vec before calling this function
     template <typename tag_type, typename data_type> int get_vector_by_tag(tag_type &tag, data_type *vec);
 
-    virtual void set_universal_labels(const std::vector<std::string> &raw_universal_labels) = 0;
+    virtual void set_universal_labels(const std::vector<std::string> &raw_universal_labels,
+                                      bool dynamic_index = false) = 0;
 
   private:
     virtual void _build(const DataType &data, const size_t num_points_to_load, TagVector &tags) = 0;

diff --git a/include/in_mem_filter_store.h b/include/in_mem_filter_store.h
@@ -33,7 +33,8 @@ template <typename label_type> class InMemFilterStore : public AbstractFilterSto
     void calculate_best_medoids(const size_t num_points_to_load, const uint32_t num_candidates) override;
 
     // takes raw universal labels and map them internally.
-    void set_universal_labels(const std::vector<std::string> &raw_universal_labels) override;
+    void set_universal_labels(const std::vector<std::string> &raw_universal_labels,
+                              bool dyanmic_index = false) override;
     // const label_type get_universal_label() const;
 
     // ideally takes raw label file and then genrate internal mapping file and keep the info of mapping

diff --git a/include/index.h b/include/index.h
@@ -105,7 +105,7 @@ template <typename T, typename TagT = uint32_t, typename LabelT = uint32_t> clas
                                                 const std::vector<TagT> &tags = std::vector<TagT>());
 
     // DISKANN_DLLEXPORT void set_universal_label(const LabelT &label);
-    DISKANN_DLLEXPORT void set_universal_labels(const std::vector<std::string> &raw_labels);
+    DISKANN_DLLEXPORT void set_universal_labels(const std::vector<std::string> &raw_labels, bool dynamic_index = false);
 
     // Get converted integer label from string to int map (_label_map)
     DISKANN_DLLEXPORT LabelT get_converted_label(const std::string &raw_label);

diff --git a/src/in_mem_filter_store.cpp b/src/in_mem_filter_store.cpp
@@ -89,7 +89,8 @@ void InMemFilterStore<label_type>::add_label_to_location(const location_t point_
 // }
 
 template <typename label_type>
-void InMemFilterStore<label_type>::set_universal_labels(const std::vector<std::string> &raw_universal_labels)
+void InMemFilterStore<label_type>::set_universal_labels(const std::vector<std::string> &raw_universal_labels,
+                                                        bool dynamic_index)
 {
     for (auto label : raw_universal_labels)
     {
@@ -104,6 +105,13 @@ void InMemFilterStore<label_type>::set_universal_labels(const std::vector<std::s
     }
     if (!_raw_universal_labels.empty())
         _use_universal_label = true;
+    // populate the mapped universal label set for dynamic index
+    if (dynamic_index && _use_universal_label)
+    {
+        // WARNING: Please change as we have unified mapping logic from raw univ labels => mapped univ labels
+        // we are able to make above assumption that mapping is always consistent and not random.
+        _mapped_universal_labels.insert(0);
+    }
 }
 
 // template <typename label_type> const label_type InMemFilterStore<label_type>::get_universal_label() const
@@ -178,18 +186,28 @@ size_t InMemFilterStore<label_type>::load_medoids(const std::string &labels_to_m
 
 template <typename label_type> void InMemFilterStore<label_type>::load_label_map(const std::string &labels_map_file)
 {
-    std::ifstream map_reader(labels_map_file);
-    std::string line, token;
-    label_type token_as_num;
-    std::string label_str;
-    while (std::getline(map_reader, line))
+    if (file_exists(labels_map_file))
     {
-        std::istringstream iss(line);
-        getline(iss, token, '\t');
-        label_str = token;
-        getline(iss, token, '\t');
-        token_as_num = (label_type)std::stoul(token);
-        _label_map[label_str] = token_as_num;
+        std::ifstream map_reader(labels_map_file);
+        std::string line, token;
+        label_type token_as_num;
+        std::string label_str;
+        while (std::getline(map_reader, line))
+        {
+            std::istringstream iss(line);
+            getline(iss, token, '\t');
+            label_str = token;
+            getline(iss, token, '\t');
+            token_as_num = (label_type)std::stoul(token);
+            _label_map[label_str] = token_as_num;
+        }
+    }
+    else
+    {
+        throw diskann::ANNException(
+            "Can't load label map file please make sure it was generate, either by filter_store->load_raw_labels() "
+            "then index->save() or  convert_labels_string_to_int() method in case of dynamic index",
+            -1);
     }
 }
 
@@ -226,7 +244,7 @@ void InMemFilterStore<label_type>::save_labels(const std::string &save_path, con
         assert(label_writer.is_open());
         for (uint32_t i = 0; i < total_points; i++)
         {
-            for (uint32_t j = 0; j < (_location_to_labels[i].size() - 1); j++)
+            for (uint32_t j = 0; j + 1 < _location_to_labels[i].size(); j++)
             {
                 label_writer << _location_to_labels[i][j] << ",";
             }

diff --git a/src/index.cpp b/src/index.cpp
@@ -1715,9 +1715,9 @@ void Index<T, TagT, LabelT>::parse_label_file(const std::string &label_file, siz
 // }
 
 template <typename T, typename TagT, typename LabelT>
-void Index<T, TagT, LabelT>::set_universal_labels(const std::vector<std::string> &raw_labels)
+void Index<T, TagT, LabelT>::set_universal_labels(const std::vector<std::string> &raw_labels, bool dynamic_index)
 {
-    _filter_store->set_universal_labels(raw_labels);
+    _filter_store->set_universal_labels(raw_labels, dynamic_index);
 }
 
 template <typename T, typename TagT, typename LabelT>