From c1a7a70c5e83dfa9e750f723f2a78a7d80bf46d8 Mon Sep 17 00:00:00 2001 From: Eric BREHAULT Date: Wed, 8 Feb 2023 14:42:08 +0100 Subject: [PATCH] use catalog endpoint in resource list (#568) --- .../app/resources/resource-list.component.ts | 35 ++++++++++--------- libs/sdk-core/CHANGELOG.md | 5 +-- libs/sdk-core/package.json | 2 +- libs/sdk-core/src/lib/db/kb/kb.models.ts | 2 ++ libs/sdk-core/src/lib/db/kb/kb.ts | 6 +++- libs/sdk-core/src/lib/db/search/search.ts | 32 +++++++++++++---- 6 files changed, 56 insertions(+), 26 deletions(-) diff --git a/apps/dashboard/src/app/resources/resource-list.component.ts b/apps/dashboard/src/app/resources/resource-list.component.ts index 3ca2dea49..d1806ea00 100644 --- a/apps/dashboard/src/app/resources/resource-list.component.ts +++ b/apps/dashboard/src/app/resources/resource-list.component.ts @@ -44,6 +44,7 @@ import { resourceToAlgoliaFormat, Search, SortOrder, + SearchOptions, } from '@nuclia/core'; import { BackendConfigurationService, SDKService, StateService, STFUtils } from '@flaps/core'; import { SisModalService, SisToastService } from '@nuclia/sistema'; @@ -584,21 +585,25 @@ export class ResourceListComponent implements AfterViewInit, OnInit, OnDestroy { }), switchMap(() => this.sdk.currentKb.pipe(take(1))), switchMap((kb) => { - const searchFeatures = - hasQuery && !titleOnly - ? [Search.Features.PARAGRAPH, Search.Features.VECTOR, Search.Features.DOCUMENT] - : [Search.Features.DOCUMENT]; const status = this.statusDisplayed.value; + const searchOptions: SearchOptions = { + page_number: page, + page_size: this.pageSize, + sort: { field: 'created' }, + filters: status === RESOURCE_STATUS.PROCESSED ? undefined : [`/n/s/${status}`], + with_status: status === RESOURCE_STATUS.PROCESSED ? status : undefined, + }; return forkJoin([ of(kb), - kb.search(query, searchFeatures, { - inTitleOnly: titleOnly, - page_number: page, - page_size: this.pageSize, - sort: { field: 'created' }, - filters: status === RESOURCE_STATUS.PROCESSED ? undefined : [`/n/s/${status}`], - with_status: status === RESOURCE_STATUS.PROCESSED ? status : undefined, - }), + titleOnly + ? kb.catalog(query, searchOptions) + : kb.search( + query, + hasQuery + ? [Search.Features.PARAGRAPH, Search.Features.VECTOR, Search.Features.DOCUMENT] + : [Search.Features.DOCUMENT], + searchOptions, + ), this.labelSets$.pipe(take(1)), this.statusCount.pipe(take(1)), ]); @@ -826,8 +831,7 @@ export class ResourceListComponent implements AfterViewInit, OnInit, OnDestroy { return this.sdk.currentKb.pipe( take(1), switchMap((kb) => - kb.search('', [Search.Features.DOCUMENT], { - inTitleOnly: true, + kb.catalog('', { faceted: [statusFacet], }), ), @@ -884,8 +888,7 @@ export class ResourceListComponent implements AfterViewInit, OnInit, OnDestroy { } private getResourcesInError(kb: KnowledgeBox, page_number = 0): Observable { - return kb.search('', [Search.Features.DOCUMENT], { - inTitleOnly: true, + return kb.catalog('', { page_number, page_size: 20, sort: { field: 'created' }, diff --git a/libs/sdk-core/CHANGELOG.md b/libs/sdk-core/CHANGELOG.md index ae40d7d8d..58364116b 100644 --- a/libs/sdk-core/CHANGELOG.md +++ b/libs/sdk-core/CHANGELOG.md @@ -1,4 +1,4 @@ -# 1.1.3 (unreleased) +# 1.1.3 (2023-02-08) ### Improvements @@ -11,10 +11,11 @@ - Update training types: labeler split in two `resoure-labeler` and `paragraph-labeler` - Add `field` property to `Search.SmartResult` interface - Add support for `/train/executions` endpoint -- Add `SHORT_FIELD_TYPE` enum in search models and update `Paragraph` and `Sentence` interfaces accordingly +- Add `SHORT_FIELD_TYPE` enum in search models and update `Paragraph` and `Sentence` interfaces accordingly - Add mappers to translate `SHORT_FIELD_TYPE` into `FIELD_TYPE` and _vice versa_ - Add mapper `getDataKeyFromFieldType` to translate `FIELD_TYPE` into key of `ResourceData` - Don't load entities by default when loading entity families, add `withEntities` parameter to `getEntities` method to load entities as well +- Add the `/catalog` endpoint # 1.1.2 (2023-01-20) diff --git a/libs/sdk-core/package.json b/libs/sdk-core/package.json index 2e56bb5f1..74b5878d4 100644 --- a/libs/sdk-core/package.json +++ b/libs/sdk-core/package.json @@ -1,6 +1,6 @@ { "name": "@nuclia/core", - "version": "1.1.2", + "version": "1.1.3", "description": "SDK allowing to integrate Nuclia services in your frontend application", "license": "MIT", "keywords": [ diff --git a/libs/sdk-core/src/lib/db/kb/kb.models.ts b/libs/sdk-core/src/lib/db/kb/kb.models.ts index d3df70938..b4e4b78aa 100644 --- a/libs/sdk-core/src/lib/db/kb/kb.models.ts +++ b/libs/sdk-core/src/lib/db/kb/kb.models.ts @@ -68,6 +68,8 @@ export interface IKnowledgeBox extends IKnowledgeBoxCreation { search(query: string, features?: Search.Features[], options?: SearchOptions): Observable; + catalog(query: string, options?: SearchOptions): Observable; + suggest(query: string): Observable; counters(): Observable; diff --git a/libs/sdk-core/src/lib/db/kb/kb.ts b/libs/sdk-core/src/lib/db/kb/kb.ts index db70bbf1d..be9284a7d 100644 --- a/libs/sdk-core/src/lib/db/kb/kb.ts +++ b/libs/sdk-core/src/lib/db/kb/kb.ts @@ -16,7 +16,7 @@ import type { ICreateResource, IResource, LinkField, UserMetadata } from '../res import { Resource } from '../resource'; import type { UploadResponse } from '../upload'; import { batchUpload, FileMetadata, FileWithMetadata, upload, UploadStatus } from '../upload'; -import type { Search, SearchOptions } from '../search'; +import { catalog, Search, SearchOptions } from '../search'; import { search } from '../search'; import { Training } from '../training'; @@ -129,6 +129,10 @@ export class KnowledgeBox implements IKnowledgeBox { return search(this.nuclia, this.id, this.path, query, features, options); } + catalog(query: string, options?: SearchOptions): Observable { + return catalog(this.nuclia, this.id, query, options); + } + suggest(query: string, inTitleOnly = false): Observable { const params = `query=${encodeURIComponent(query)}${inTitleOnly ? '&fields=a/title' : ''}`; return this.nuclia.rest.get(`${this.path}/suggest?${params}`).pipe( diff --git a/libs/sdk-core/src/lib/db/search/search.ts b/libs/sdk-core/src/lib/db/search/search.ts index 0e310ba88..0b491830f 100644 --- a/libs/sdk-core/src/lib/db/search/search.ts +++ b/libs/sdk-core/src/lib/db/search/search.ts @@ -35,14 +35,34 @@ export const search = ( ); }; +export const catalog = (nuclia: INuclia, kbid: string, query: string, options?: SearchOptions) => { + const params: { [key: string]: string | string[] } = {}; + params.query = query || ''; + params.shards = nuclia.currentShards?.[kbid] || []; + const searchMethod = nuclia.rest.get( + `/kb/${kbid}/catalog?${options ? serialize(params, options) : ''}`, + ); + return searchMethod.pipe( + catchError(() => of({ error: true } as Search.Results)), + map((res) => (Object.keys(res).includes('detail') ? ({ error: true } as Search.Results) : (res as Search.Results))), + tap((res) => { + if (res.shards) { + nuclia.currentShards = { ...nuclia.currentShards, [kbid]: res.shards }; + } + }), + ); +}; + const serialize = (params: { [key: string]: string | string[] }, others: SearchOptions): string => { Object.entries(others || {}).forEach(([key, value]) => { - if (Array.isArray(value)) { - params[key] = value.map((el) => `${el}`); - } else if (typeof value === 'object') { - Object.entries(value).forEach(([k, v]) => (params[`${key}_${k}`] = `${v}`)); - } else { - params[key] = `${value}`; + if (value !== undefined && value !== null) { + if (Array.isArray(value)) { + params[key] = value.map((el) => `${el}`); + } else if (typeof value === 'object') { + Object.entries(value).forEach(([k, v]) => (params[`${key}_${k}`] = `${v}`)); + } else { + params[key] = `${value}`; + } } }); const queryParams = new URLSearchParams();