From 25970d90c3cf45fd2c227b758108e6a8922ba63c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20Sa=CC=88rkikoski?= Date: Tue, 26 Sep 2023 11:30:10 +0300 Subject: [PATCH 1/5] Add thesis type code in publication --- .../src/ApiModels/CodeList/CountryCode.cs | 2 +- .../src/ApiModels/CodeList/ThesisType.cs | 10 ++++++++ .../src/ApiModels/Publication/Publication.cs | 7 ++++++ .../Repositories/Maps/PublicationProfile.cs | 1 + .../Service.Models/Publication/Publication.cs | 23 +++++++++++-------- 5 files changed, 33 insertions(+), 10 deletions(-) create mode 100644 aspnetcore/src/ApiModels/CodeList/ThesisType.cs diff --git a/aspnetcore/src/ApiModels/CodeList/CountryCode.cs b/aspnetcore/src/ApiModels/CodeList/CountryCode.cs index f7998be..fd76d9c 100644 --- a/aspnetcore/src/ApiModels/CodeList/CountryCode.cs +++ b/aspnetcore/src/ApiModels/CodeList/CountryCode.cs @@ -1,7 +1,7 @@ namespace ResearchFi.CodeList; /// -/// Countriy codes +/// Country codes /// /// http://uri.suomi.fi/codelist/jhs/valtio_1_20120101 /// diff --git a/aspnetcore/src/ApiModels/CodeList/ThesisType.cs b/aspnetcore/src/ApiModels/CodeList/ThesisType.cs new file mode 100644 index 0000000..ceb672e --- /dev/null +++ b/aspnetcore/src/ApiModels/CodeList/ThesisType.cs @@ -0,0 +1,10 @@ +namespace ResearchFi.CodeList; + +/// +/// Thesis types +/// +/// http://uri.suomi.fi/codelist/research/Opinnaytetyyppi +/// +public class ThesisType : CodeList +{ +} \ No newline at end of file diff --git a/aspnetcore/src/ApiModels/Publication/Publication.cs b/aspnetcore/src/ApiModels/Publication/Publication.cs index cd0e861..000f935 100644 --- a/aspnetcore/src/ApiModels/Publication/Publication.cs +++ b/aspnetcore/src/ApiModels/Publication/Publication.cs @@ -70,6 +70,13 @@ public class Publication /// public PublicationType? Type { get; set; } + /// + /// Thesis type + /// + /// http://uri.suomi.fi/codelist/research/Opinnaytetyyppi + /// + public ThesisType? ThesisType { get; set; } + /// /// Journal name /// diff --git a/aspnetcore/src/Repositories/Maps/PublicationProfile.cs b/aspnetcore/src/Repositories/Maps/PublicationProfile.cs index c496e44..2af27f5 100644 --- a/aspnetcore/src/Repositories/Maps/PublicationProfile.cs +++ b/aspnetcore/src/Repositories/Maps/PublicationProfile.cs @@ -32,6 +32,7 @@ public PublicationProfile() .ForMember(dst => dst.DatabasePeerReviewed, opt => opt.MapFrom(src => src.PeerReviewed)) .ForMember(dst => dst.TargetAudience, opt => opt.MapFrom(src => src.TargetAudienceCodeNavigation)) .ForMember(dst => dst.Type, opt => opt.MapFrom(src => src.PublicationTypeCodeNavigation)) + .ForMember(dst => dst.ThesisType, opt => opt.MapFrom(src => src.ThesisTypeCodeNavigation)) .ForMember(dst => dst.JournalName, opt => opt.MapFrom(src => src.JournalName)) .ForMember(dst => dst.IssueNumber, opt => opt.MapFrom(src => src.IssueNumber)) .ForMember(dst => dst.ConferenceName, opt => opt.MapFrom(src => src.ConferenceName)) diff --git a/aspnetcore/src/Service.Models/Publication/Publication.cs b/aspnetcore/src/Service.Models/Publication/Publication.cs index d7a97eb..9d8debc 100644 --- a/aspnetcore/src/Service.Models/Publication/Publication.cs +++ b/aspnetcore/src/Service.Models/Publication/Publication.cs @@ -24,7 +24,7 @@ public class Publication public DateTime? PublicationYear { get; set; } /// - /// Tekijäteksti + /// Tekij�teksti /// public string? AuthorsText { get; set; } @@ -34,7 +34,7 @@ public class Publication public List? Organizations { get; set; } /// - /// Tekijät + /// Tekij�t /// public List? Authors { get; set; } @@ -49,7 +49,7 @@ public class Publication public ReferenceData? PeerReviewed { get; set; } /// - /// Yleisö + /// Yleis� /// public ReferenceData? TargetAudience { get; set; } @@ -58,6 +58,11 @@ public class Publication /// public ReferenceData? Type { get; set; } + /// + /// Opinnäytetyön tyyppi + /// + public ReferenceData? ThesisType { get; set; } + /// /// Lehti /// @@ -74,7 +79,7 @@ public class Publication public string? ConferenceName { get; set; } /// - /// Julkaisuun liittyvät ISSN tunnisteet + /// Julkaisuun liittyv�t ISSN tunnisteet /// [Keyword] public List? Issn { get; set; } @@ -100,7 +105,7 @@ public class Publication public ParentPublication? ParentPublication { get; set; } /// - /// Julkaisuun liittyvät ISBN tunnisteet + /// Julkaisuun liittyv�t ISBN tunnisteet /// [Keyword] public List? Isbn { get; set; } @@ -132,7 +137,7 @@ public class Publication public string? Doi { get; set; } /// - /// Pysyvä osoite + /// Pysyv� osoite /// [Keyword] public string? DoiHandle { get; set; } @@ -173,7 +178,7 @@ public class Publication public List? Keywords { get; set; } /// - /// Julkaisun kansainvälisyys + /// Julkaisun kansainv�lisyys /// public bool? InternationalPublication { get; set; } @@ -188,7 +193,7 @@ public class Publication public ReferenceData? Language { get; set; } /// - /// Kansainvälinen yhteisjulkaisu + /// Kansainv�linen yhteisjulkaisu /// public bool? InternationalCollaboration { get; set; } @@ -238,7 +243,7 @@ public class Publication public List? ArtPublicationTypeCategory { get; set; } /// - /// Tiivistelmä + /// Tiivistelm� /// public string? Abstract { get; set; } From a10bbbaded59a7963f97a8a8077dd0516e29776c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20Sa=CC=88rkikoski?= Date: Fri, 29 Sep 2023 08:49:40 +0300 Subject: [PATCH 2/5] Enable chunked database queries in indexing --- .../ElasticSearchIndexService.cs | 13 +-- aspnetcore/src/Indexer/Indexer.cs | 97 ++++++++++++++----- .../src/Indexer/appsettings.Development.json | 7 +- aspnetcore/src/Indexer/appsettings.json | 7 +- .../FundingCallIndexRepository.cs | 12 +++ .../FundingDecisionIndexRepository.cs | 18 ++++ .../src/Repositories/IIndexRepository.cs | 19 ++++ .../src/Repositories/IndexRepositoryBase.cs | 22 +++++ .../InfrastructureIndexRepository.cs | 10 ++ .../OrganizationIndexRepository.cs | 10 ++ .../PublicationIndexRepository.cs | 26 +++++ .../ResearchDatasetIndexRepository.cs | 14 +++ 12 files changed, 222 insertions(+), 33 deletions(-) diff --git a/aspnetcore/src/ElasticService/ElasticSearchIndexService.cs b/aspnetcore/src/ElasticService/ElasticSearchIndexService.cs index 430ea01..0ad3f7c 100644 --- a/aspnetcore/src/ElasticService/ElasticSearchIndexService.cs +++ b/aspnetcore/src/ElasticService/ElasticSearchIndexService.cs @@ -8,7 +8,7 @@ public class ElasticSearchIndexService : IElasticSearchIndexService private readonly IElasticClient _elasticClient; private readonly ILogger _logger; - private const int BatchSize = 1000; + private const int BatchSize = 2000; public ElasticSearchIndexService(IElasticClient elasticClient, ILogger logger) { @@ -78,6 +78,8 @@ await _elasticClient.Indices.BulkAliasAsync(r => r private async Task IndexEntities(string indexName, List entities, Type modelType) where T : class { + var indexedCount = 0; + // Split entities into batches to avoid one big request. var documentBatches = new List>(); for (var docIndex = 0; docIndex < entities.Count; docIndex += BatchSize) @@ -93,11 +95,11 @@ private async Task IndexEntities(string indexName, List entities, Type mod if (!indexBatchResponse.IsValid) { - _logger.LogError(indexBatchResponse.OriginalException, "{EntityType}: Indexing entities to {IndexName} failed", modelType, indexName); - throw new InvalidOperationException($"Indexing entities to {indexName} failed.", indexBatchResponse.OriginalException); + _logger.LogError(indexBatchResponse.OriginalException, "{EntityType}: Indexing documents to {IndexName} failed", modelType, indexName); + throw new InvalidOperationException($"Indexing documents to {indexName} failed.", indexBatchResponse.OriginalException); } - - _logger.LogDebug("{EntityType}: Indexed {BatchSize} documents to {IndexName}", modelType.Name, batchToIndex.Count, indexName); + indexedCount = indexedCount + batchToIndex.Count; + _logger.LogInformation("{EntityType}: Indexed {BatchSize} documents to {IndexName}. Progress {IndexedCount}/{TotalCount}", modelType.Name, batchToIndex.Count, indexName, indexedCount, entities.Count); } } @@ -125,5 +127,4 @@ await _elasticClient.Indices.DeleteAsync(indexName, _logger.LogDebug("{EntityType}: Index {IndexName} created", type.Name, indexName); } - } \ No newline at end of file diff --git a/aspnetcore/src/Indexer/Indexer.cs b/aspnetcore/src/Indexer/Indexer.cs index e0aa719..d8bcecd 100644 --- a/aspnetcore/src/Indexer/Indexer.cs +++ b/aspnetcore/src/Indexer/Indexer.cs @@ -39,23 +39,22 @@ public async Task Start() { Stopwatch stopWatch = new(); stopWatch.Start(); - - await PopulateOrganizationCache(); - await PopulateFundingCallCache(); _logger.LogInformation("Starting indexing..."); _logger.LogInformation("Using ElasticSearch at '{ElasticSearchAddress}'", _configuration["ELASTICSEARCH:URL"]); var configuredTypesAndIndexNames = _indexNameSettings.GetTypesAndIndexNames(); + await PopulateOrganizationCache(); + await PopulateFundingCallCache(); + foreach (var (indexName, modelType) in configuredTypesAndIndexNames) { var repositoryForType = _indexRepositories.SingleOrDefault(repo => repo.ModelType == modelType); if (repositoryForType is null) { - _logger.LogError("{EntityType}: Unable to find database repository for index {IndexName}", modelType.Name, indexName); - + _logger.LogError("{EntityType}: Unable to find database repository for index {IndexName}", modelType.Name, indexName); continue; } @@ -73,7 +72,7 @@ public async Task Start() /// private async Task PopulateOrganizationCache() { - _logger.LogInformation("Populating Organization cache..."); + _logger.LogInformation("Populating Organization cache"); var organizationRepository = _indexRepositories.SingleOrDefault(repo => repo.ModelType == typeof(Organization)); if (organizationRepository != null) @@ -84,6 +83,8 @@ private async Task PopulateOrganizationCache() _memoryCache.Set(MemoryCacheKeys.OrganizationById(organization.Id), organization); } } + + _logger.LogInformation("Populated Organization cache"); } /// @@ -91,7 +92,7 @@ private async Task PopulateOrganizationCache() /// private async Task PopulateFundingCallCache() { - _logger.LogInformation("Populating Funding Call cache..."); + _logger.LogInformation("Populating Funding Call cache"); var fundingCallRepository = _indexRepositories.SingleOrDefault(repo => repo.ModelType == typeof(FundingCall)); if (fundingCallRepository != null) @@ -116,38 +117,84 @@ private async Task PopulateFundingCallCache() } } } + + _logger.LogInformation("Populated Funding Call cache"); } private async Task IndexEntities(string indexName, IIndexRepository repository, Type type) { + _logger.LogInformation("{EntityType}: Recreating '{IndexName}' index...", type.Name, indexName); + Stopwatch stopWatch = new(); stopWatch.Start(); try { - _logger.LogDebug("{EntityType}: Recreating '{IndexName}' index...", type.Name, indexName); - - var indexModels = await repository.GetAllAsync().ToListAsync(); - - var databaseElapsed = stopWatch.Elapsed; - - _logger.LogDebug("{EntityType}: Retrieved {DatabaseCount} entities from the database in {ElapsedDatabase}...", type.Name, indexModels.Count, databaseElapsed); + List finalized = new(); + + if (indexName.Contains("publication")) { + /* + * Process database result in smaller chunks to keep memory requirement low. + * Chunking is based on skip/take query. + * Currently this is done only for publications, because their dataset is much + * larger than others. + */ + + int skipAmount = 0; + int takeAmount = 200000; + int numOfResults = 0; - var finalized = repository.PerformInMemoryOperations(indexModels); - + do + { + _logger.LogInformation("{EntityType}: Requested {takeAmount} entities from database...", type.Name, takeAmount); + var indexModels = await repository.GetChunkAsync(skipAmount: skipAmount, takeAmount: takeAmount).ToListAsync(); + numOfResults = indexModels.Count; + _logger.LogInformation("{EntityType}: ...received {numOfResults} entities", type.Name, numOfResults); + + if (numOfResults > 0) + { + _logger.LogInformation("{EntityType}: In-memory operations start", type.Name); + foreach (object entity in indexModels) { + finalized.Add(repository.PerformInMemoryOperation(entity)); + } + _logger.LogInformation("{EntityType}: In-memory operations complete", type.Name); + } + skipAmount = skipAmount + takeAmount; + } while(numOfResults >= takeAmount-1); + } + else + { + /* + * Process complete database result at once. + * Suitable for small result sets. + */ + _logger.LogInformation("{EntityType}: Requested all entities from database...", type.Name); + var indexModels = await repository.GetAllAsync().ToListAsync(); + var databaseElapsed = stopWatch.Elapsed; + _logger.LogInformation("{EntityType}: ..received {DatabaseCount} entities in {ElapsedDatabase}", type.Name, indexModels.Count, databaseElapsed); + + if (indexModels.Count > 0) + { + _logger.LogInformation("{EntityType}: Start in-memory operations", type.Name); + finalized = repository.PerformInMemoryOperations(indexModels); + } + } var inMemoryElapsed = stopWatch.Elapsed; - _logger.LogDebug("{EntityType}: Performed in-memory operations in {ElapsedInMemory}...", type.Name, inMemoryElapsed - databaseElapsed); - - await _indexService.IndexAsync(indexName, finalized, type); - - var indexingElapsed = stopWatch.Elapsed; - - _logger.LogDebug("{EntityType}: Indexed {IndexCount} entities in {ElapsedIndexing}...", type.Name, indexModels.Count, indexingElapsed - inMemoryElapsed); - - _logger.LogInformation("{EntityType}: Index '{IndexName}' recreated successfully with {IndexCount} entities in {ElapsedTotal}", type.Name, indexName, indexModels.Count, stopWatch.Elapsed); + if (finalized.Count > 0) + { + _logger.LogInformation("{EntityType}: Retrieved and performed in-memory operations to {FinalizedCount} entities in {Elapsed}. Start indexing.", type.Name, finalized.Count, inMemoryElapsed); + await _indexService.IndexAsync(indexName, finalized, type); + var indexingElapsed = stopWatch.Elapsed; + _logger.LogInformation("{EntityType}: Indexed total of {IndexCount} documents in {ElapsedIndexing}...", type.Name, finalized.Count, indexingElapsed - inMemoryElapsed); + _logger.LogInformation("{EntityType}: Index '{IndexName}' recreated successfully in {ElapsedTotal}", type.Name, indexName, stopWatch.Elapsed); + } + else + { + _logger.LogInformation("{EntityType}: Nothing to index", type.Name); + } } catch (Exception ex) { diff --git a/aspnetcore/src/Indexer/appsettings.Development.json b/aspnetcore/src/Indexer/appsettings.Development.json index ab1679d..a90f9fc 100644 --- a/aspnetcore/src/Indexer/appsettings.Development.json +++ b/aspnetcore/src/Indexer/appsettings.Development.json @@ -16,7 +16,12 @@ } }, "WriteTo": { - "ConsoleSink": "Console", + "ConsoleSink": { + "Name": "Console", + "Args": { + "outputTemplate": "[{Timestamp:yyyy-MM-dd HH:mm:ss} {Level:u3}] {Message:lj}{NewLine}{Exception}" + } + }, "HttpSink": { "Name": "Http", "Args": { diff --git a/aspnetcore/src/Indexer/appsettings.json b/aspnetcore/src/Indexer/appsettings.json index b034dbb..56d3db7 100644 --- a/aspnetcore/src/Indexer/appsettings.json +++ b/aspnetcore/src/Indexer/appsettings.json @@ -13,7 +13,12 @@ } }, "WriteTo": { - "ConsoleSink": "Console", + "ConsoleSink": { + "Name": "Console", + "Args": { + "outputTemplate": "[{Timestamp:yyyy-MM-dd HH:mm:ss} {Level:u3}] {Message:lj}{NewLine}{Exception}" + } + }, "HttpSink": { "Name": "Http", "Args": { diff --git a/aspnetcore/src/Repositories/FundingCallIndexRepository.cs b/aspnetcore/src/Repositories/FundingCallIndexRepository.cs index 90c93ec..d34fdef 100644 --- a/aspnetcore/src/Repositories/FundingCallIndexRepository.cs +++ b/aspnetcore/src/Repositories/FundingCallIndexRepository.cs @@ -26,6 +26,18 @@ protected override IQueryable GetAll() .Where(callProgramme => callProgramme.Id != -1) .ProjectTo(_mapper.ConfigurationProvider); } + + protected override IQueryable GetChunk(int skipAmount, int takeAmount) + { + return _context.DimCallProgrammes + .OrderBy(callProgramme => callProgramme.Id) + .Skip(skipAmount) + .Take(takeAmount) + .AsNoTracking() + .AsSplitQuery() + .Where(callProgramme => callProgramme.Id != -1) + .ProjectTo(_mapper.ConfigurationProvider); + } public override List PerformInMemoryOperations(List entities) { diff --git a/aspnetcore/src/Repositories/FundingDecisionIndexRepository.cs b/aspnetcore/src/Repositories/FundingDecisionIndexRepository.cs index e505ca0..ff4a43c 100644 --- a/aspnetcore/src/Repositories/FundingDecisionIndexRepository.cs +++ b/aspnetcore/src/Repositories/FundingDecisionIndexRepository.cs @@ -36,6 +36,24 @@ protected override IQueryable GetAll() .ProjectTo(_mapper.ConfigurationProvider); } + protected override IQueryable GetChunk(int skipAmount, int takeAmount) + { + return _context + .Set() + .OrderBy(fd => fd.Id) + .Skip(skipAmount) + .Take(takeAmount) + .AsNoTracking() + .AsSplitQuery() + .Where(fd => fd.Id != -1) + .Where(fd => + (fd.SourceDescription == "eu_funding" + && fd.BrFundingConsortiumParticipations.Any(fc => fc.DimOrganization.DimPids.Any(pid => pid.PidType == "PIC"))) + || + fd.SourceDescription != "eu_funding") + .ProjectTo(_mapper.ConfigurationProvider); + } + public override List PerformInMemoryOperations(List objects) { objects.ForEach(o => diff --git a/aspnetcore/src/Repositories/IIndexRepository.cs b/aspnetcore/src/Repositories/IIndexRepository.cs index 6224a76..4021d72 100644 --- a/aspnetcore/src/Repositories/IIndexRepository.cs +++ b/aspnetcore/src/Repositories/IIndexRepository.cs @@ -11,6 +11,12 @@ public interface IIndexRepository : IIndexRepository where T : class /// /// IQueryable GetAll(); + + /// + /// Gets a chunk of database entities for indexing, projected as T. + /// + /// + IQueryable GetChunk(int skipAmount, int takeAmount); } /// @@ -30,10 +36,23 @@ public interface IIndexRepository /// IAsyncEnumerable GetAllAsync(); + /// + /// Returns models of the type ModelType. + /// + /// + IAsyncEnumerable GetChunkAsync(int skipAmount, int takeAmount); + /// /// Perform data manipulations which are hard to do in the db query phase. /// /// /// List PerformInMemoryOperations(List objects); + + /// + /// Perform to single entity data manipulations which are hard to do in the db query phase. + /// + /// + /// + object PerformInMemoryOperation(object entity); } \ No newline at end of file diff --git a/aspnetcore/src/Repositories/IndexRepositoryBase.cs b/aspnetcore/src/Repositories/IndexRepositoryBase.cs index cbaa2e4..001bea0 100644 --- a/aspnetcore/src/Repositories/IndexRepositoryBase.cs +++ b/aspnetcore/src/Repositories/IndexRepositoryBase.cs @@ -21,13 +21,23 @@ protected IndexRepositoryBase(IMemoryCache memoryCache) /// /// protected abstract IQueryable GetAll(); + + /// + /// Method which every IndexRepository must override, provides every T as Queryable. + /// + /// + protected abstract IQueryable GetChunk(int skipAmount, int takeAmount); Type IIndexRepository.ModelType => typeof(T); IQueryable IIndexRepository.GetAll() => GetAll(); + IQueryable IIndexRepository.GetChunk(int skipAmount, int takeAmount) => GetChunk(skipAmount, takeAmount); + IAsyncEnumerable IIndexRepository.GetAllAsync() => GetAll().AsAsyncEnumerable(); + IAsyncEnumerable IIndexRepository.GetChunkAsync(int skipAmount, int takeAmount) => GetChunk(skipAmount, takeAmount).AsAsyncEnumerable(); + /// /// If the data type needs special data manipulations after data has been fetched, /// this can be overriden in derived classes. @@ -38,6 +48,18 @@ public virtual List PerformInMemoryOperations(List entities) { return entities; } + + /// + /// If the data type needs special data manipulations after data has been fetched, + /// this can be overriden in derived classes. + /// Single entity version. + /// + /// + /// + public virtual object PerformInMemoryOperation(object entity) + { + return entity; + } protected Service.Models.Organization.Organization? GetOrganization(int organizationId) { diff --git a/aspnetcore/src/Repositories/InfrastructureIndexRepository.cs b/aspnetcore/src/Repositories/InfrastructureIndexRepository.cs index 912f74a..0f67a5f 100644 --- a/aspnetcore/src/Repositories/InfrastructureIndexRepository.cs +++ b/aspnetcore/src/Repositories/InfrastructureIndexRepository.cs @@ -23,4 +23,14 @@ protected override IQueryable GetAll() .Where(infrastructure => infrastructure.Id != -1) .ProjectTo(_mapper.ConfigurationProvider); } + + protected override IQueryable GetChunk(int skipAmount, int takeAmount) + { + return _context.DimInfrastructures + .OrderBy(infrastructure => infrastructure.Id) + .Skip(skipAmount) + .Take(takeAmount) + .Where(infrastructure => infrastructure.Id != -1) + .ProjectTo(_mapper.ConfigurationProvider); + } } \ No newline at end of file diff --git a/aspnetcore/src/Repositories/OrganizationIndexRepository.cs b/aspnetcore/src/Repositories/OrganizationIndexRepository.cs index 6016acf..98a0781 100644 --- a/aspnetcore/src/Repositories/OrganizationIndexRepository.cs +++ b/aspnetcore/src/Repositories/OrganizationIndexRepository.cs @@ -23,4 +23,14 @@ protected override IQueryable GetAll() .Where(organization => organization.Id != -1) .ProjectTo(_mapper.ConfigurationProvider); } + + protected override IQueryable GetChunk(int skipAmount, int takeAmount) + { + return _context.DimOrganizations + .OrderBy(organization => organization.Id) + .Skip(skipAmount) + .Take(takeAmount) + .Where(organization => organization.Id != -1) + .ProjectTo(_mapper.ConfigurationProvider); + } } \ No newline at end of file diff --git a/aspnetcore/src/Repositories/PublicationIndexRepository.cs b/aspnetcore/src/Repositories/PublicationIndexRepository.cs index 249b0ae..316d392 100644 --- a/aspnetcore/src/Repositories/PublicationIndexRepository.cs +++ b/aspnetcore/src/Repositories/PublicationIndexRepository.cs @@ -44,6 +44,19 @@ protected override IQueryable GetAll() .Where(publication => publication.Id != -1) .ProjectTo(_mapper.ConfigurationProvider); } + + protected override IQueryable GetChunk(int skipAmount, int takeAmount) + { + return _context.DimPublications + .OrderBy(publication => publication.Id) + .Skip(skipAmount) + .Take(takeAmount) + .AsNoTracking() + .AsSplitQuery() + .Where(publication => publication.Id != -1) + .ProjectTo(_mapper.ConfigurationProvider); + } + public override List PerformInMemoryOperations(List entities) { @@ -65,6 +78,19 @@ public override List PerformInMemoryOperations(List entities) return entities; } + public override object PerformInMemoryOperation(object entity) + { + Publication publication = (Publication)entity; + + HandleIssnAndIsbn(publication); + HandleEmptyCollections(publication); + HandleOrganizations(publication); + HandleAuthors(publication); + HandleParentPublications(publication); + HandlePeerReviewed(publication); + return publication; + } + private static void HandlePeerReviewed(Publication publication) { if (publication.DatabasePeerReviewed is not null) diff --git a/aspnetcore/src/Repositories/ResearchDatasetIndexRepository.cs b/aspnetcore/src/Repositories/ResearchDatasetIndexRepository.cs index 86fb051..d7eade7 100644 --- a/aspnetcore/src/Repositories/ResearchDatasetIndexRepository.cs +++ b/aspnetcore/src/Repositories/ResearchDatasetIndexRepository.cs @@ -29,6 +29,20 @@ protected override IQueryable GetAll() .ThenInclude(f => f.DimOrganization) .ProjectTo(_mapper.ConfigurationProvider); } + + protected override IQueryable GetChunk(int skipAmount, int takeAmount) + { + return _context.DimResearchDatasets + .OrderBy(dataset => dataset.Id) + .Skip(skipAmount) + .Take(takeAmount) + .AsNoTracking() + .AsSplitQuery() + .Where(dataset => dataset.Id != -1) + .Include(d => d.FactContributions) + .ThenInclude(f => f.DimOrganization) + .ProjectTo(_mapper.ConfigurationProvider); + } public override List PerformInMemoryOperations(List entities) { From c174d655dc22f9719ac72c8850d986b663fb885c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20Sa=CC=88rkikoski?= Date: Fri, 6 Oct 2023 15:07:08 +0300 Subject: [PATCH 3/5] Publication.Created and Publication.Modified are nullable fields --- aspnetcore/src/ApiModels/Publication/Publication.cs | 4 ++-- aspnetcore/src/Service.Models/Publication/Publication.cs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/aspnetcore/src/ApiModels/Publication/Publication.cs b/aspnetcore/src/ApiModels/Publication/Publication.cs index 000f935..f73e31a 100644 --- a/aspnetcore/src/ApiModels/Publication/Publication.cs +++ b/aspnetcore/src/ApiModels/Publication/Publication.cs @@ -279,10 +279,10 @@ public class Publication /// /// Creation time /// - public DateTime Created { get; set; } + public DateTime? Created { get; set; } /// /// Modification time /// - public DateTime Modified { get; set; } + public DateTime? Modified { get; set; } } \ No newline at end of file diff --git a/aspnetcore/src/Service.Models/Publication/Publication.cs b/aspnetcore/src/Service.Models/Publication/Publication.cs index 9d8debc..3c98453 100644 --- a/aspnetcore/src/Service.Models/Publication/Publication.cs +++ b/aspnetcore/src/Service.Models/Publication/Publication.cs @@ -250,12 +250,12 @@ public class Publication /// /// Luontiaika /// - public DateTime Created { get; set; } + public DateTime? Created { get; set; } /// /// Muokkausaika /// - public DateTime Modified { get; set; } + public DateTime? Modified { get; set; } [Ignore] public string? Isbn1 { get; set; } From 2dd1dc030b74951e753e56c7c7ea1c986fa3fa60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20Sa=CC=88rkikoski?= Date: Mon, 9 Oct 2023 12:56:49 +0300 Subject: [PATCH 4/5] Add ThesisType in in publication search response data --- aspnetcore/src/Interface/Maps/PublicationProfile.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/aspnetcore/src/Interface/Maps/PublicationProfile.cs b/aspnetcore/src/Interface/Maps/PublicationProfile.cs index 88f08b7..cbf5b45 100644 --- a/aspnetcore/src/Interface/Maps/PublicationProfile.cs +++ b/aspnetcore/src/Interface/Maps/PublicationProfile.cs @@ -49,6 +49,7 @@ public PublicationProfile() CreateMap(); CreateMap(); CreateMap(); + CreateMap(); CreateMap(); } } \ No newline at end of file From bc35c8455da220e2365ac5234fcfc1181b37fe18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20Sa=CC=88rkikoski?= Date: Mon, 9 Oct 2023 14:35:33 +0300 Subject: [PATCH 5/5] Fix corrupted finnish alphabets in publication field description --- .../Service.Models/Publication/Publication.cs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/aspnetcore/src/Service.Models/Publication/Publication.cs b/aspnetcore/src/Service.Models/Publication/Publication.cs index 3c98453..6983f58 100644 --- a/aspnetcore/src/Service.Models/Publication/Publication.cs +++ b/aspnetcore/src/Service.Models/Publication/Publication.cs @@ -24,7 +24,7 @@ public class Publication public DateTime? PublicationYear { get; set; } /// - /// Tekij�teksti + /// Tekijäteksti /// public string? AuthorsText { get; set; } @@ -34,7 +34,7 @@ public class Publication public List? Organizations { get; set; } /// - /// Tekij�t + /// Tekijät /// public List? Authors { get; set; } @@ -49,7 +49,7 @@ public class Publication public ReferenceData? PeerReviewed { get; set; } /// - /// Yleis� + /// Yleisö /// public ReferenceData? TargetAudience { get; set; } @@ -79,7 +79,7 @@ public class Publication public string? ConferenceName { get; set; } /// - /// Julkaisuun liittyv�t ISSN tunnisteet + /// Julkaisuun liittyvät ISSN tunnisteet /// [Keyword] public List? Issn { get; set; } @@ -105,7 +105,7 @@ public class Publication public ParentPublication? ParentPublication { get; set; } /// - /// Julkaisuun liittyv�t ISBN tunnisteet + /// Julkaisuun liittyvät ISBN tunnisteet /// [Keyword] public List? Isbn { get; set; } @@ -137,7 +137,7 @@ public class Publication public string? Doi { get; set; } /// - /// Pysyv� osoite + /// Pysyvä osoite /// [Keyword] public string? DoiHandle { get; set; } @@ -178,7 +178,7 @@ public class Publication public List? Keywords { get; set; } /// - /// Julkaisun kansainv�lisyys + /// Julkaisun kansainvälisyys /// public bool? InternationalPublication { get; set; } @@ -193,7 +193,7 @@ public class Publication public ReferenceData? Language { get; set; } /// - /// Kansainv�linen yhteisjulkaisu + /// Kansainvälinen yhteisjulkaisu /// public bool? InternationalCollaboration { get; set; } @@ -243,7 +243,7 @@ public class Publication public List? ArtPublicationTypeCategory { get; set; } /// - /// Tiivistelm� + /// Tiivistelmä /// public string? Abstract { get; set; }