Fixed Bug with infinite loop on init

Author: amigin

JavaScript/HtmlMain.js

@@ -12,4 +12,5 @@
 * Added   settings parameter
 
 ### 0.0.20-rc01
-* Sync to Readers now works in separated thread
+* Sync to Readers now works in a separate thread
+* Exposed sync to readers bytes to Prometheus reader by reader

JavaScript/HtmlMain.js.map

@@ -9,7 +9,9 @@ class HtmlMain {
 
     public static generateInit(model: INonInitializedModel): string {
 
-        var result = '<h1>Remains tables to load: ' + model.tablesRemains + '</h1><h2>Total loading time is: ' + this.formatSeconds(model.initializingSeconds) + '</h2>' +
+        var result = '<h1>Remains tables to load: ' + model.tablesRemains + '</h1>' +
+            '<h2>Table which files are being loaded: ' + model.tableBeingLoadedFiles + '</h2>' +
+            '<h2>Total loading time is: ' + this.formatSeconds(model.initializingSeconds) + '</h2>' +
             '<table class="table table-striped table-bordered"><tr><th>TableName</th><th>Partitions loaded</th><th>Partitions total</th><th>Time gone</th><th>Time estimation</th></tr>'
 
         for (let itm of model.progress.sort((a, b) => a.tableName > b.tableName ? 1 : -1)) {

JavaScript/main.js.map

@@ -26,6 +26,7 @@ interface INonInitializedModel {
     tablesRemains: number,
     initializingSeconds: number,
     progress: ITableLoadProgress[]
+    tableBeingLoadedFiles: String
 }
 
 

README.md

@@ -69,8 +69,6 @@ class main {
                     this.layoutElement.innerHTML = HtmlMain.generateInit(result.notInitialized);
                 }
 
-
-
                 HtmlStatusBar.updateStatusbar(result.statusBar);
 
             }).fail(() => {

TypeScript/HtmlMain.ts

@@ -12,6 +12,8 @@ pub struct NonInitializedModel {
     #[serde(rename = "initializingSeconds")]
     initializing_seconds: i64,
     progress: Vec<TableProgressModel>,
+    #[serde(rename = "tableBeingLoadedFiles")]
+    table_being_loaded_files: Option<String>,
 }
 
 impl NonInitializedModel {
@@ -30,6 +32,7 @@ impl NonInitializedModel {
             progress,
             tables_remains: snapshot.to_load.len(),
             initializing_seconds: now.seconds_before(app.created),
+            table_being_loaded_files: snapshot.table_being_loaded_files,
         };
     }
 }

TypeScript/contracts.d.ts

@@ -48,7 +48,7 @@ async fn load_table_files_loop(
                     .await;
                 }
                 PartitionToLoad::EndOfReading => {
-                    break;
+                    return;
                 }
             },
             None => {

TypeScript/main.ts

@@ -59,7 +59,8 @@ async fn load_tables_spawned(app: Arc<AppContext>) {
                 );
             }
             super::load_tasks::ProcessTableToLoad::NotReadyYet => {
-                tokio::time::sleep(Duration::from_millis(100)).await;
+                println!("We do not have table to load yet");
+                tokio::time::sleep(Duration::from_millis(1000)).await;
             }
             super::load_tasks::ProcessTableToLoad::TheEnd => {
                 return;

src/http/controllers/status_controller/non_initialized/model.rs

@@ -34,6 +34,11 @@ impl InitState {
         write_access.get_next_table_to_init_files()
     }
 
+    pub async fn loading_files_for_tables_is_done(&self) {
+        let mut write_access = self.data.lock().await;
+        write_access.loading_files_for_tables_is_done();
+    }
+
     pub async fn get_next_table_to_load(&self) -> ProcessTableToLoad {
         let mut write_access = self.data.lock().await;
         write_access.get_next_table_to_load()

src/persist_operations/data_initializer/load_table/load_table_files.rs

@@ -11,6 +11,7 @@ pub enum ProcessTableToLoad {
 }
 
 pub struct InitStateData {
+    pub table_being_loaded_files: Option<String>,
     pub tables_to_init_files: Vec<Arc<TableToLoad>>,
     pub tables_to_load: Vec<ProcessTableToLoad>,
     pub tables_being_loaded: Vec<Arc<TableToLoad>>,
@@ -24,6 +25,7 @@ impl InitStateData {
             tables_to_load: Vec::new(),
             tables_being_loaded: Vec::new(),
             tables_loaded: Vec::new(),
+            table_being_loaded_files: None,
         }
     }
 
@@ -34,13 +36,18 @@ impl InitStateData {
         }
 
         let result = self.tables_to_init_files.remove(0);
+        self.table_being_loaded_files = Some(result.table_name.to_string());
 
         self.tables_to_load
             .push(ProcessTableToLoad::Process(result.clone()));
 
         Some(result)
     }
 
+    pub fn loading_files_for_tables_is_done(&mut self) {
+        self.table_being_loaded_files = None;
+    }
+
     pub fn init_tables(&mut self, tables: Vec<String>, logs: &Logs) {
         for table_name in tables {
             if let Err(err) = validation::validate_table_name(table_name.as_str()) {
@@ -66,7 +73,14 @@ impl InitStateData {
             if self.tables_to_load.len() > 0 {
                 self.tables_to_load.remove(0)
             } else {
-                ProcessTableToLoad::NotReadyYet
+                if self.tables_being_loaded.len() == 0
+                    && self.tables_to_load.len() == 0
+                    && self.tables_to_init_files.len() == 0
+                {
+                    ProcessTableToLoad::TheEnd
+                } else {
+                    ProcessTableToLoad::NotReadyYet
+                }
             }
         };
 
@@ -101,6 +115,7 @@ impl InitStateData {
             to_load: convert_to_tables_snapshot_2(&self.tables_to_load),
             loading: convert_to_tables_snapshot(&self.tables_being_loaded),
             loaded: convert_to_tables_snapshot(&self.tables_loaded),
+            table_being_loaded_files: self.table_being_loaded_files.clone(),
         }
     }
 

src/persist_operations/data_initializer/load_tables.rs

@@ -12,4 +12,5 @@ pub struct InitStateSnapshot {
     pub to_load: Vec<InitTableStateSnapshot>,
     pub loading: Vec<InitTableStateSnapshot>,
     pub loaded: Vec<InitTableStateSnapshot>,
+    pub table_being_loaded_files: Option<String>,
 }

src/persist_operations/data_initializer/load_tasks/init_state.rs

@@ -6,4 +6,6 @@ pub async fn table_list_of_files_loader(app: Arc<AppContext>) {
     while let Some(table_to_load) = app.init_state.get_next_table_to_init_files().await {
         app.persist_io.get_table_files(&table_to_load).await;
     }
+
+    app.init_state.loading_files_for_tables_is_done().await;
 }