Revert "parse_label_file change for memory"

This reverts commit 56f65a5.

include/index.h

@@ -247,7 +247,7 @@ template <typename T, typename TagT = uint32_t, typename LabelT = uint32_t> clas
     // determines navigating node of the graph by calculating medoid of datafopt
     uint32_t calculate_entry_point();
 
-    void parse_label_file(std::basic_istream<char> &label_file, size_t &num_pts);
+    void parse_label_file(const std::string &label_file, size_t &num_pts_labels);
 
     std::unordered_map<std::string, LabelT> load_label_map(const std::string &map_file);
 
@@ -339,8 +339,6 @@ template <typename T, typename TagT = uint32_t, typename LabelT = uint32_t> clas
                                                       IndexSearchContext<LabelT> &context);
 
   private:
-    void reset_stream_for_reading(std::basic_istream<char> &infile);
-
     // Distance functions
     Metric _dist_metric = diskann::L2;
     std::shared_ptr<Distance<T>> _distance;

src/index.cpp

@@ -603,12 +603,7 @@ void Index<T, TagT, LabelT>::load(const char *filename, uint32_t num_threads, ui
     if (file_exists(labels_file))
     {
         _label_map = load_label_map(labels_map_file);
-        std::ifstream infile(labels_file, std::ios::binary);
-        if (infile.fail())
-        {
-            throw diskann::ANNException(std::string("Failed to open file ") + labels_file, -1);
-        }
-        parse_label_file(infile, label_num_pts);
+        parse_label_file(labels_file, label_num_pts);
         assert(label_num_pts == data_file_num_pts);
         if (file_exists(labels_to_medoids))
         {
@@ -2023,88 +2018,49 @@ LabelT Index<T, TagT, LabelT>::get_converted_label(const std::string &raw_label)
 }
 
 template <typename T, typename TagT, typename LabelT>
-void Index<T, TagT, LabelT>::reset_stream_for_reading(std::basic_istream<char> &infile)
-{
-    infile.clear();
-    infile.seekg(0);
-}
-
-template <typename T, typename TagT, typename LabelT>
-void Index<T, TagT, LabelT>::parse_label_file(std::basic_istream<char> &label_file, size_t &num_pts)
+void Index<T, TagT, LabelT>::parse_label_file(const std::string &label_file, size_t &num_points)
 {
     // Format of Label txt file: filters with comma separators
-    label_file.seekg(0, std::ios::end);
-    size_t file_size = label_file.tellg();
-
-    std::string buffer(file_size, ' ');
 
-    label_file.seekg(0, std::ios::beg);
-    label_file.read(&buffer[0], file_size);
+    std::ifstream infile(label_file);
+    if (infile.fail())
+    {
+        throw diskann::ANNException(std::string("Failed to open file ") + label_file, -1);
+    }
 
+    std::string line, token;
     uint32_t line_cnt = 0;
-    std::string label_str;
-    size_t cur_pos = 0;
-    size_t next_pos = 0;
-    while (cur_pos < file_size && cur_pos != std::string::npos)
+
+    while (std::getline(infile, line))
     {
-        next_pos = buffer.find('\n', cur_pos);
-        if (next_pos == std::string::npos)
-        {
-            break;
-        }
-        cur_pos = next_pos + 1;
         line_cnt++;
     }
     _pts_to_labels.resize(line_cnt, std::vector<LabelT>());
 
-    cur_pos = 0;
-    next_pos = 0;
+    infile.clear();
+    infile.seekg(0, std::ios::beg);
     line_cnt = 0;
-    while (cur_pos < file_size && cur_pos != std::string::npos)
+
+    while (std::getline(infile, line))
     {
-        next_pos = buffer.find('\n', cur_pos);
-        if (next_pos == std::string::npos)
-        {
-            break;
-        }
-        std::vector<LabelT> lbls;
-        size_t lbl_pos = cur_pos;
-        size_t next_lbl_pos = 0;
-        while (lbl_pos < next_pos && lbl_pos != std::string::npos)
+        std::istringstream iss(line);
+        std::vector<LabelT> lbls(0);
+        getline(iss, token, '\t');
+        std::istringstream new_iss(token);
+        while (getline(new_iss, token, ','))
         {
-            next_lbl_pos = buffer.find(',', lbl_pos);
-            if (next_lbl_pos == std::string::npos) // the last label in the whole file
-            {
-                next_lbl_pos = next_pos;
-            }
-
-            if (next_lbl_pos > next_pos) // the last label in one line, just read to the end
-            {
-                next_lbl_pos = next_pos;
-            }
-
-            label_str.assign(buffer.c_str() + lbl_pos, next_lbl_pos - lbl_pos);
-            if (label_str[label_str.length() - 1] == '\t' ||
-                label_str[label_str.length() - 1] == '\r') // '\t' won't exist in label file?
-            {
-                label_str.erase(label_str.length() - 1);
-            }
-
-            LabelT token_as_num = (LabelT)std::stoul(label_str);
+            token.erase(std::remove(token.begin(), token.end(), '\n'), token.end());
+            token.erase(std::remove(token.begin(), token.end(), '\r'), token.end());
+            LabelT token_as_num = (LabelT)std::stoul(token);
             lbls.push_back(token_as_num);
             _labels.insert(token_as_num);
-
-            // move to next label
-            lbl_pos = next_lbl_pos + 1;
         }
+
+        std::sort(lbls.begin(), lbls.end());
         _pts_to_labels[line_cnt] = lbls;
-        line_cnt = line_cnt + 1;
-        // move to next line
-        cur_pos = next_pos + 1;
+        line_cnt++;
     }
-
-    num_pts = line_cnt;
-    reset_stream_for_reading(label_file);
+    num_points = (size_t)line_cnt;
     diskann::cout << "Identified " << _labels.size() << " distinct label(s)" << std::endl;
 }
 
@@ -2125,14 +2081,9 @@ void Index<T, TagT, LabelT>::build_filtered_index(const char *filename, const st
     _label_to_medoid_id.clear();
     size_t num_points_labels = 0;
 
-    std::ifstream infile(label_file, std::ios::binary);
-    if (infile.fail())
-    {
-        throw diskann::ANNException(std::string("Failed to open file ") + _labels_file, -1);
-    }
-
-    parse_label_file(infile, num_points_labels); // determines medoid for each label and identifies
-                                                 // the points to label mapping
+    parse_label_file(label_file,
+                     num_points_labels); // determines medoid for each label and identifies
+                                         // the points to label mapping
 
     std::unordered_map<LabelT, std::vector<uint32_t>> label_to_points;