Merge pull request #118 from CSCfi/CSCTTV-3638-koivu-logging

CSCTTV-3638 send logs to Koivu
CSCfi · Feb 22, 2024 · 6335c2d · 6335c2d
2 parents cd40a9a + f737853
commit 6335c2d
Show file tree

Hide file tree

Showing 28 changed files with 546 additions and 89 deletions.
diff --git a/aspnetcore/openshift/indexer/template-indexer-devel.yml b/aspnetcore/openshift/indexer/template-indexer-devel.yml
@@ -95,6 +95,21 @@ objects:
                         configMapKeyRef:
                           name: publicapi-api-config-devel
                           key: "Serilog__WriteTo__HttpSink__Args__requestUri"
+                    - name: "Serilog__Properties__WoodLogProjectNumber"
+                      valueFrom:
+                        configMapKeyRef:
+                          name: publicapi-api-config-devel
+                          key: "Serilog__Properties__WoodLogProjectNumber"
+                    - name: "Serilog__Properties__WoodLogRetentionMonthsIndexer"
+                      valueFrom:
+                        configMapKeyRef:
+                          name: publicapi-api-config-devel
+                          key: "Serilog__Properties__WoodLogRetentionMonthsIndexer"
+                    - name: "Serilog__Properties__WoodLogUsecaseIndexer"
+                      valueFrom:
+                        configMapKeyRef:
+                          name: publicapi-api-config-devel
+                          key: "Serilog__Properties__WoodLogUsecaseIndexer"
                     - name: "QueryTimeout"
                       valueFrom:
                         configMapKeyRef:

diff --git a/aspnetcore/openshift/indexer/template-indexer-manualjob-devel.yml b/aspnetcore/openshift/indexer/template-indexer-manualjob-devel.yml
@@ -1,6 +1,6 @@
 # This file is part of the research.fi api
 #
-# Copyright 2022 Ministry of Education and Culture, Finland
+# Copyright 2024 Ministry of Education and Culture, Finland
 #
 # :author: CSC - IT Center for Science Ltd., Espoo Finland [email protected]
 # :license: MIT
@@ -90,6 +90,21 @@ objects:
                     configMapKeyRef:
                       name: publicapi-api-config-devel
                       key: "Serilog__WriteTo__HttpSink__Args__requestUri"
+                - name: "Serilog__Properties__WoodLogProjectNumber"
+                  valueFrom:
+                    configMapKeyRef:
+                      name: publicapi-api-config-devel
+                      key: "Serilog__Properties__WoodLogProjectNumber"
+                - name: "Serilog__Properties__WoodLogRetentionMonthsIndexer"
+                  valueFrom:
+                    configMapKeyRef:
+                      name: publicapi-api-config-devel
+                      key: "Serilog__Properties__WoodLogRetentionMonthsIndexer"
+                - name: "Serilog__Properties__WoodLogUsecaseIndexer"
+                  valueFrom:
+                    configMapKeyRef:
+                      name: publicapi-api-config-devel
+                      key: "Serilog__Properties__WoodLogUsecaseIndexer"
                 - name: "QueryTimeout"
                   valueFrom:
                     configMapKeyRef:

diff --git a/aspnetcore/openshift/indexer/template-indexer-manualjob-production.yml b/aspnetcore/openshift/indexer/template-indexer-manualjob-production.yml
@@ -90,6 +90,21 @@ objects:
                     configMapKeyRef:
                       name: publicapi-api-config-production
                       key: "Serilog__WriteTo__HttpSink__Args__requestUri"
+                - name: "Serilog__Properties__WoodLogProjectNumber"
+                  valueFrom:
+                    configMapKeyRef:
+                      name: publicapi-api-config-production
+                      key: "Serilog__Properties__WoodLogProjectNumber"
+                - name: "Serilog__Properties__WoodLogRetentionMonthsIndexer"
+                  valueFrom:
+                    configMapKeyRef:
+                      name: publicapi-api-config-production
+                      key: "Serilog__Properties__WoodLogRetentionMonthsIndexer"
+                - name: "Serilog__Properties__WoodLogUsecaseIndexer"
+                  valueFrom:
+                    configMapKeyRef:
+                      name: publicapi-api-config-production
+                      key: "Serilog__Properties__WoodLogUsecaseIndexer"
                 - name: "QueryTimeout"
                   valueFrom:
                     configMapKeyRef:

diff --git a/aspnetcore/openshift/indexer/template-indexer-manualjob-qa.yml b/aspnetcore/openshift/indexer/template-indexer-manualjob-qa.yml
@@ -90,6 +90,21 @@ objects:
                     configMapKeyRef:
                       name: publicapi-api-config-qa
                       key: "Serilog__WriteTo__HttpSink__Args__requestUri"
+                - name: "Serilog__Properties__WoodLogProjectNumber"
+                  valueFrom:
+                    configMapKeyRef:
+                      name: publicapi-api-config-qa
+                      key: "Serilog__Properties__WoodLogProjectNumber"
+                - name: "Serilog__Properties__WoodLogRetentionMonthsIndexer"
+                  valueFrom:
+                    configMapKeyRef:
+                      name: publicapi-api-config-qa
+                      key: "Serilog__Properties__WoodLogRetentionMonthsIndexer"
+                - name: "Serilog__Properties__WoodLogUsecaseIndexer"
+                  valueFrom:
+                    configMapKeyRef:
+                      name: publicapi-api-config-qa
+                      key: "Serilog__Properties__WoodLogUsecaseIndexer"
                 - name: "QueryTimeout"
                   valueFrom:
                     configMapKeyRef:

diff --git a/aspnetcore/openshift/indexer/template-indexer-production.yml b/aspnetcore/openshift/indexer/template-indexer-production.yml
@@ -95,6 +95,21 @@ objects:
                         configMapKeyRef:
                           name: publicapi-api-config-production
                           key: "Serilog__WriteTo__HttpSink__Args__requestUri"
+                    - name: "Serilog__Properties__WoodLogProjectNumber"
+                      valueFrom:
+                        configMapKeyRef:
+                          name: publicapi-api-config-production
+                          key: "Serilog__Properties__WoodLogProjectNumber"
+                    - name: "Serilog__Properties__WoodLogRetentionMonthsIndexer"
+                      valueFrom:
+                        configMapKeyRef:
+                          name: publicapi-api-config-production
+                          key: "Serilog__Properties__WoodLogRetentionMonthsIndexer"
+                    - name: "Serilog__Properties__WoodLogUsecaseIndexer"
+                      valueFrom:
+                        configMapKeyRef:
+                          name: publicapi-api-config-production
+                          key: "Serilog__Properties__WoodLogUsecaseIndexer"
                     - name: "QueryTimeout"
                       valueFrom:
                         configMapKeyRef:

diff --git a/aspnetcore/openshift/indexer/template-indexer-qa.yml b/aspnetcore/openshift/indexer/template-indexer-qa.yml
@@ -95,6 +95,21 @@ objects:
                         configMapKeyRef:
                           name: publicapi-api-config-qa
                           key: "Serilog__WriteTo__HttpSink__Args__requestUri"
+                    - name: "Serilog__Properties__WoodLogProjectNumber"
+                      valueFrom:
+                        configMapKeyRef:
+                          name: publicapi-api-config-qa
+                          key: "Serilog__Properties__WoodLogProjectNumber"
+                    - name: "Serilog__Properties__WoodLogRetentionMonthsIndexer"
+                      valueFrom:
+                        configMapKeyRef:
+                          name: publicapi-api-config-qa
+                          key: "Serilog__Properties__WoodLogRetentionMonthsIndexer"
+                    - name: "Serilog__Properties__WoodLogUsecaseIndexer"
+                      valueFrom:
+                        configMapKeyRef:
+                          name: publicapi-api-config-qa
+                          key: "Serilog__Properties__WoodLogUsecaseIndexer"
                     - name: "QueryTimeout"
                       valueFrom:
                         configMapKeyRef:

diff --git a/aspnetcore/src/ElasticService/ElasticSearchIndexService.cs b/aspnetcore/src/ElasticService/ElasticSearchIndexService.cs
@@ -34,9 +34,9 @@ public async Task IndexAsync(string indexName, List<object> entities, Type model
         // Switch indexes
         await SwitchIndexes(indexName, indexToCreate, indexToDelete);
 
-        _logger.LogDebug("{EntityType}: Indexing to {IndexName} complete", modelType.Name, indexName);
+        _logger.LogDebug("{EntityType:l}: Indexing to {IndexName:l} complete", modelType.Name, indexName);
     }
-    
+
     public async Task IndexChunkAsync(string indexToCreate, List<object> entities, Type modelType)
     {
         // Add entities to the index.
@@ -89,28 +89,27 @@ await _elasticClient.Indices.BulkAliasAsync(r => r
 
     private async Task IndexEntities<T>(string indexName, List<T> entities, Type modelType) where T : class
     {
-        var indexedCount = 0;
-
         // Split entities into batches to avoid one big request.
         var documentBatches = new List<List<T>>();
         for (var docIndex = 0; docIndex < entities.Count; docIndex += BatchSize)
         {
             documentBatches.Add(entities.GetRange(docIndex, Math.Min(BatchSize, entities.Count - docIndex)));
         }
-
+
+        int batchCounter = 0;
         foreach (var batchToIndex in documentBatches)
         {
+            ++batchCounter;
+            _logger.LogInformation("{EntityType:l}: Indexing {ElasticsearchBatchSize} documents. Batch {ElasticsearchBatchCurrent}/{ElasticsearchBatchCount}", modelType.Name, batchToIndex.Count, batchCounter, documentBatches.Count);
             var indexBatchResponse = await _elasticClient.BulkAsync(b => b
                 .Index(indexName)
                 .IndexMany(batchToIndex));
 
             if (!indexBatchResponse.IsValid)
             {
-                _logger.LogError(indexBatchResponse.OriginalException, "{EntityType}: Indexing documents to {IndexName} failed", modelType, indexName);
+                _logger.LogError("{EntityType:l}: Indexing documents to {IndexName:l} failed: {IndexerException}", modelType, indexName, indexBatchResponse.OriginalException.ToString());
                 throw new InvalidOperationException($"Indexing documents to {indexName} failed.", indexBatchResponse.OriginalException);
             }
-            indexedCount = indexedCount + batchToIndex.Count;
-            _logger.LogInformation("{EntityType}: Indexed {BatchSize} documents to {IndexName}", modelType.Name, batchToIndex.Count, indexName);
         }
     }
 
@@ -136,6 +135,6 @@ await _elasticClient.Indices.DeleteAsync(indexName,
             throw new InvalidOperationException($"Creating index {indexName} failed.", createResponse.OriginalException);
         }
 
-        _logger.LogDebug("{EntityType}: Index {IndexName} created", type.Name, indexName);
+        _logger.LogDebug("{EntityType:l}: Index {IndexName:l} created", type.Name, indexName);
     }
 }