Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CSCTTV-3638 send logs to Koivu #118

Merged
merged 11 commits into from
Feb 22, 2024
Merged

CSCTTV-3638 send logs to Koivu #118

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
17a8f54
Add Koivu metadata to all log messages
sarkikos Feb 14, 2024
03353f9
Add logging parameters in OpenShift templates
sarkikos Feb 14, 2024
e971833
Always include rendered message in Indexer logs
sarkikos Feb 15, 2024
2158bf4
Flush logs after indexer process completes
sarkikos Feb 15, 2024
35d9b43
Modify Indexer log messages. Unify structured log property names.
sarkikos Feb 15, 2024
d9a5b7e
Modify Indexer log messages
sarkikos Feb 15, 2024
b4d4bc2
Modify Indexer log messages
sarkikos Feb 15, 2024
01a8470
Modify Indexer log messages
sarkikos Feb 15, 2024
a96cce3
Use own text formatter for Indexer and Interface. Add QueryString to …
sarkikos Feb 16, 2024
0a5ae14
Add ResourceType parameter in interface controller logs
sarkikos Feb 21, 2024
f737853
Separate logging parameters for Indexer and Interface
sarkikos Feb 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions aspnetcore/openshift/indexer/template-indexer-devel.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,21 @@ objects:
configMapKeyRef:
name: publicapi-api-config-devel
key: "Serilog__WriteTo__HttpSink__Args__requestUri"
- name: "Serilog__Properties__WoodLogProjectNumber"
valueFrom:
configMapKeyRef:
name: publicapi-api-config-devel
key: "Serilog__Properties__WoodLogProjectNumber"
- name: "Serilog__Properties__WoodLogRetentionMonthsIndexer"
valueFrom:
configMapKeyRef:
name: publicapi-api-config-devel
key: "Serilog__Properties__WoodLogRetentionMonthsIndexer"
- name: "Serilog__Properties__WoodLogUsecaseIndexer"
valueFrom:
configMapKeyRef:
name: publicapi-api-config-devel
key: "Serilog__Properties__WoodLogUsecaseIndexer"
- name: "QueryTimeout"
valueFrom:
configMapKeyRef:
Expand Down
17 changes: 16 additions & 1 deletion aspnetcore/openshift/indexer/template-indexer-manualjob-devel.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# This file is part of the research.fi api
#
# Copyright 2022 Ministry of Education and Culture, Finland
# Copyright 2024 Ministry of Education and Culture, Finland
#
# :author: CSC - IT Center for Science Ltd., Espoo Finland [email protected]
# :license: MIT
Expand Down Expand Up @@ -90,6 +90,21 @@ objects:
configMapKeyRef:
name: publicapi-api-config-devel
key: "Serilog__WriteTo__HttpSink__Args__requestUri"
- name: "Serilog__Properties__WoodLogProjectNumber"
valueFrom:
configMapKeyRef:
name: publicapi-api-config-devel
key: "Serilog__Properties__WoodLogProjectNumber"
- name: "Serilog__Properties__WoodLogRetentionMonthsIndexer"
valueFrom:
configMapKeyRef:
name: publicapi-api-config-devel
key: "Serilog__Properties__WoodLogRetentionMonthsIndexer"
- name: "Serilog__Properties__WoodLogUsecaseIndexer"
valueFrom:
configMapKeyRef:
name: publicapi-api-config-devel
key: "Serilog__Properties__WoodLogUsecaseIndexer"
- name: "QueryTimeout"
valueFrom:
configMapKeyRef:
Expand Down
15 changes: 15 additions & 0 deletions aspnetcore/openshift/indexer/template-indexer-manualjob-production.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,21 @@ objects:
configMapKeyRef:
name: publicapi-api-config-production
key: "Serilog__WriteTo__HttpSink__Args__requestUri"
- name: "Serilog__Properties__WoodLogProjectNumber"
valueFrom:
configMapKeyRef:
name: publicapi-api-config-production
key: "Serilog__Properties__WoodLogProjectNumber"
- name: "Serilog__Properties__WoodLogRetentionMonthsIndexer"
valueFrom:
configMapKeyRef:
name: publicapi-api-config-production
key: "Serilog__Properties__WoodLogRetentionMonthsIndexer"
- name: "Serilog__Properties__WoodLogUsecaseIndexer"
valueFrom:
configMapKeyRef:
name: publicapi-api-config-production
key: "Serilog__Properties__WoodLogUsecaseIndexer"
- name: "QueryTimeout"
valueFrom:
configMapKeyRef:
Expand Down
15 changes: 15 additions & 0 deletions aspnetcore/openshift/indexer/template-indexer-manualjob-qa.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,21 @@ objects:
configMapKeyRef:
name: publicapi-api-config-qa
key: "Serilog__WriteTo__HttpSink__Args__requestUri"
- name: "Serilog__Properties__WoodLogProjectNumber"
valueFrom:
configMapKeyRef:
name: publicapi-api-config-qa
key: "Serilog__Properties__WoodLogProjectNumber"
- name: "Serilog__Properties__WoodLogRetentionMonthsIndexer"
valueFrom:
configMapKeyRef:
name: publicapi-api-config-qa
key: "Serilog__Properties__WoodLogRetentionMonthsIndexer"
- name: "Serilog__Properties__WoodLogUsecaseIndexer"
valueFrom:
configMapKeyRef:
name: publicapi-api-config-qa
key: "Serilog__Properties__WoodLogUsecaseIndexer"
- name: "QueryTimeout"
valueFrom:
configMapKeyRef:
Expand Down
15 changes: 15 additions & 0 deletions aspnetcore/openshift/indexer/template-indexer-production.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,21 @@ objects:
configMapKeyRef:
name: publicapi-api-config-production
key: "Serilog__WriteTo__HttpSink__Args__requestUri"
- name: "Serilog__Properties__WoodLogProjectNumber"
valueFrom:
configMapKeyRef:
name: publicapi-api-config-production
key: "Serilog__Properties__WoodLogProjectNumber"
- name: "Serilog__Properties__WoodLogRetentionMonthsIndexer"
valueFrom:
configMapKeyRef:
name: publicapi-api-config-production
key: "Serilog__Properties__WoodLogRetentionMonthsIndexer"
- name: "Serilog__Properties__WoodLogUsecaseIndexer"
valueFrom:
configMapKeyRef:
name: publicapi-api-config-production
key: "Serilog__Properties__WoodLogUsecaseIndexer"
- name: "QueryTimeout"
valueFrom:
configMapKeyRef:
Expand Down
15 changes: 15 additions & 0 deletions aspnetcore/openshift/indexer/template-indexer-qa.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,21 @@ objects:
configMapKeyRef:
name: publicapi-api-config-qa
key: "Serilog__WriteTo__HttpSink__Args__requestUri"
- name: "Serilog__Properties__WoodLogProjectNumber"
valueFrom:
configMapKeyRef:
name: publicapi-api-config-qa
key: "Serilog__Properties__WoodLogProjectNumber"
- name: "Serilog__Properties__WoodLogRetentionMonthsIndexer"
valueFrom:
configMapKeyRef:
name: publicapi-api-config-qa
key: "Serilog__Properties__WoodLogRetentionMonthsIndexer"
- name: "Serilog__Properties__WoodLogUsecaseIndexer"
valueFrom:
configMapKeyRef:
name: publicapi-api-config-qa
key: "Serilog__Properties__WoodLogUsecaseIndexer"
- name: "QueryTimeout"
valueFrom:
configMapKeyRef:
Expand Down
17 changes: 8 additions & 9 deletions aspnetcore/src/ElasticService/ElasticSearchIndexService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,9 @@ public async Task IndexAsync(string indexName, List<object> entities, Type model
// Switch indexes
await SwitchIndexes(indexName, indexToCreate, indexToDelete);

_logger.LogDebug("{EntityType}: Indexing to {IndexName} complete", modelType.Name, indexName);
_logger.LogDebug("{EntityType:l}: Indexing to {IndexName:l} complete", modelType.Name, indexName);
}

public async Task IndexChunkAsync(string indexToCreate, List<object> entities, Type modelType)
{
// Add entities to the index.
Expand Down Expand Up @@ -89,28 +89,27 @@ await _elasticClient.Indices.BulkAliasAsync(r => r

private async Task IndexEntities<T>(string indexName, List<T> entities, Type modelType) where T : class
{
var indexedCount = 0;

// Split entities into batches to avoid one big request.
var documentBatches = new List<List<T>>();
for (var docIndex = 0; docIndex < entities.Count; docIndex += BatchSize)
{
documentBatches.Add(entities.GetRange(docIndex, Math.Min(BatchSize, entities.Count - docIndex)));
}


int batchCounter = 0;
foreach (var batchToIndex in documentBatches)
{
++batchCounter;
_logger.LogInformation("{EntityType:l}: Indexing {ElasticsearchBatchSize} documents. Batch {ElasticsearchBatchCurrent}/{ElasticsearchBatchCount}", modelType.Name, batchToIndex.Count, batchCounter, documentBatches.Count);
var indexBatchResponse = await _elasticClient.BulkAsync(b => b
.Index(indexName)
.IndexMany(batchToIndex));

if (!indexBatchResponse.IsValid)
{
_logger.LogError(indexBatchResponse.OriginalException, "{EntityType}: Indexing documents to {IndexName} failed", modelType, indexName);
_logger.LogError("{EntityType:l}: Indexing documents to {IndexName:l} failed: {IndexerException}", modelType, indexName, indexBatchResponse.OriginalException.ToString());
throw new InvalidOperationException($"Indexing documents to {indexName} failed.", indexBatchResponse.OriginalException);
}
indexedCount = indexedCount + batchToIndex.Count;
_logger.LogInformation("{EntityType}: Indexed {BatchSize} documents to {IndexName}", modelType.Name, batchToIndex.Count, indexName);
}
}

Expand All @@ -136,6 +135,6 @@ await _elasticClient.Indices.DeleteAsync(indexName,
throw new InvalidOperationException($"Creating index {indexName} failed.", createResponse.OriginalException);
}

_logger.LogDebug("{EntityType}: Index {IndexName} created", type.Name, indexName);
_logger.LogDebug("{EntityType:l}: Index {IndexName:l} created", type.Name, indexName);
}
}
Loading
Loading