Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

pref: search index #1241

Closed
wants to merge 8 commits into from
Closed

pref: search index #1241

Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
628cb4a
feat: add tsvector for search
caoxing9 Jan 12, 2025
757d362
feat: add tsvector trigger
caoxing9 Jan 12, 2025
284252e
feat: search hits with full text search
caoxing9 Jan 13, 2025
eb6a77a
perf: optimize search query logic for speed
caoxing9 Jan 13, 2025
86aa21b
fix: search ts vector with blank
caoxing9 Jan 13, 2025
ea7acbe
fix: search e2e test
caoxing9 Jan 13, 2025
61f41d9
fix: index error when enable hidden match field for search
caoxing9 Jan 13, 2025
0d7f64f
perf: optimize search sql
caoxing9 Jan 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 50 additions & 26 deletions apps/nestjs-backend/src/db-provider/search-query/search-fts-query.postgres.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,13 @@ export class VectorTransform {
constructor(
public field: IFieldInstance,
public knex: Knex.Client,
public dbTableName: string
public dbTableName: string,
public columnPrefix?: string
) {
this.field = field;
this.knex = knex;
this.dbTableName = dbTableName;
this.columnPrefix = columnPrefix;
}

getRawSql() {
Expand Down Expand Up @@ -42,6 +44,11 @@ export class VectorTransform {
}
}

getPrefix() {
const { columnPrefix } = this;
return `${columnPrefix ? `${columnPrefix}.` : ''}`;
}

getMultipleRawSql() {
const { field } = this;
const { isStructuredCellValue, cellValueType } = field;
Expand Down Expand Up @@ -70,8 +77,9 @@ export class VectorTransform {
field: { dbFieldName },
} = this;
const tsColumnName = FullTextSearchQueryPostgresBuilder.getTsVectorColumnName(dbFieldName);
const prefix = this.getPrefix();
return knex
.raw(`?? = to_tsvector('simple', COALESCE(??, ''))`, [tsColumnName, dbFieldName])
.raw(`?? = to_tsvector('simple', COALESCE(${prefix}??, ''))`, [tsColumnName, dbFieldName])
.toQuery();
}

Expand All @@ -82,11 +90,12 @@ export class VectorTransform {
} = this;
const tsColumnName = FullTextSearchQueryPostgresBuilder.getTsVectorColumnName(dbFieldName);
const precision = get(this.field, ['options', 'formatting', 'precision']) ?? 0;
const prefix = this.getPrefix();
return knex
.raw(`?? = to_tsvector('simple', COALESCE(ROUND(??::numeric, ${precision})::text, ''))`, [
tsColumnName,
dbFieldName,
])
.raw(
`?? = to_tsvector('simple', COALESCE(ROUND(${prefix}??::numeric, ${precision})::text, ''))`,
[tsColumnName, dbFieldName]
)
.toQuery();
}

Expand All @@ -98,11 +107,12 @@ export class VectorTransform {
const timeZone = (options as IDateFieldOptions).formatting.timeZone;
const tsColumnName = FullTextSearchQueryPostgresBuilder.getTsVectorColumnName(dbFieldName);
// "TO_CHAR(TIMEZONE(?, ??), 'YYYY-MM-DD HH24:MI')
const prefix = this.getPrefix();

return knex
.raw(
`?? = to_tsvector('simple',
to_char((??)::timestamp AT TIME ZONE ?, 'YYYY-MM-DD HH24:MI:SS')
to_char((${prefix}??)::timestamp AT TIME ZONE ?, 'YYYY-MM-DD HH24:MI:SS')
)`,
[tsColumnName, dbFieldName, timeZone]
)
Expand All @@ -114,9 +124,13 @@ export class VectorTransform {
knex,
field: { dbFieldName },
} = this;
const prefix = this.getPrefix();
const tsColumnName = FullTextSearchQueryPostgresBuilder.getTsVectorColumnName(dbFieldName);
return knex
.raw(`?? = to_tsvector('simple', COALESCE(??->>'title', ''))`, [tsColumnName, dbFieldName])
.raw(`?? = to_tsvector('simple', COALESCE(${prefix}??->>'title', ''))`, [
tsColumnName,
dbFieldName,
])
.toQuery();
}

Expand All @@ -126,13 +140,14 @@ export class VectorTransform {
field: { dbFieldName },
} = this;
const tsColumnName = FullTextSearchQueryPostgresBuilder.getTsVectorColumnName(dbFieldName);
const prefix = this.getPrefix();
return knex
.raw(
`?? = to_tsvector('simple',
COALESCE(
(
SELECT string_agg(elem::text, ' ')
FROM jsonb_array_elements_text(??::jsonb) as elem
FROM jsonb_array_elements_text(${prefix}??::jsonb) as elem
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

all ::jsonb is not nessesary

),
''
)
Expand All @@ -149,13 +164,14 @@ export class VectorTransform {
} = this;
const tsColumnName = FullTextSearchQueryPostgresBuilder.getTsVectorColumnName(dbFieldName);
const precision = get(this.field, ['options', 'formatting', 'precision']) ?? 0;
const prefix = this.getPrefix();
return knex
.raw(
`?? = to_tsvector('simple',
COALESCE(
(
SELECT string_agg(ROUND(elem::numeric, ?)::text, ' ')
FROM jsonb_array_elements_text(??::jsonb) as elem
FROM jsonb_array_elements_text(${prefix}::jsonb) as elem
),
''
)
Expand All @@ -172,6 +188,7 @@ export class VectorTransform {
} = this;
const timeZone = (options as IDateFieldOptions).formatting.timeZone;
const tsColumnName = FullTextSearchQueryPostgresBuilder.getTsVectorColumnName(dbFieldName);
const prefix = this.getPrefix();

return knex
.raw(
Expand All @@ -182,7 +199,7 @@ export class VectorTransform {
to_char((elem::timestamp AT TIME ZONE ?), 'YYYY-MM-DD HH24:MI:SS'),
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe we should use the formatter in the field to perform the conversion.

' '
)
FROM jsonb_array_elements_text(??::jsonb) as elem
FROM jsonb_array_elements_text(${prefix}??::jsonb) as elem
),
''
)
Expand All @@ -198,13 +215,14 @@ export class VectorTransform {
field: { dbFieldName },
} = this;
const tsColumnName = FullTextSearchQueryPostgresBuilder.getTsVectorColumnName(dbFieldName);
const prefix = this.getPrefix();
return knex
.raw(
`?? = to_tsvector('simple',
COALESCE(
(
SELECT string_agg(elem->>'title', ' ')
FROM jsonb_array_elements(??::jsonb) as elem
FROM jsonb_array_elements(${prefix}??::jsonb) as elem
),
''
)
Expand Down Expand Up @@ -305,22 +323,27 @@ export class FullTextSearchQueryPostgresBuilder {
return knex.raw(`UPDATE ?? Set ${sqls.join(',')}`, [dbTableName]).toQuery();
}

getCreateTriggerFunctionSql(dbFieldName: string) {
const { queryBuilder } = this;
getColumnTriggerFunctionSql(field: IFieldInstance) {
const { dbFieldName } = field;
const { queryBuilder, dbTableName } = this;
const knex = queryBuilder.client;
const tsName = FullTextSearchQueryPostgresBuilder.getTsVectorColumnName(dbFieldName);
// const tsName = FullTextSearchQueryPostgresBuilder.getTsVectorColumnName(dbFieldName);
const transformer = new VectorTransform(field, knex, dbTableName, 'NEW');
const sql = transformer.getRawSql();
return knex
.raw(
`
CREATE OR REPLACE FUNCTION update_${dbFieldName}_tsvector()
RETURNS trigger AS
$$
BEGIN
NEW.?? = to_tsvector('simple', NEW.??);
IF TG_OP = 'INSERT' OR NEW.?? IS DISTINCT FROM OLD.?? THEN
NEW.${sql};
END IF;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;`,
[tsName, dbFieldName]
[dbFieldName, dbFieldName]
)
.toQuery();
}
Expand All @@ -331,11 +354,11 @@ $$ LANGUAGE plpgsql;`,
return knex
.raw(
`
CREATE TRIGGER update_${dbFieldName}_tsvector
BEFORE INSERT OR UPDATE
ON ??
FOR EACH ROW
EXECUTE FUNCTION update_${dbFieldName}_tsvector();`,
CREATE TRIGGER update_${dbFieldName}_tsvector
BEFORE INSERT OR UPDATE
ON ??
FOR EACH ROW
EXECUTE FUNCTION update_${dbFieldName}_tsvector();`,
[dbTableName]
)
.toQuery();
Expand All @@ -348,12 +371,13 @@ EXECUTE FUNCTION update_${dbFieldName}_tsvector();`,
excSqls.push(extensionSql);
excSqls.push(this.getCreateTsVectorsSql(searchFields));
excSqls.push(this.getUpdateVectorsSql(searchFields));
// excSqls.push(this.getCreateTriggerSql(dbFieldName));
// excSqls.push(this.getCreateTriggerFunctionSql(dbFieldName));
searchFields.forEach((field) => {
const { dbFieldName } = field;
excSqls.push(this.getColumnTriggerFunctionSql(field));
excSqls.push(this.getCreateTriggerSql(dbFieldName));
excSqls.push(this.getCreateGinIndexSql(dbFieldName));
});
console.log('ppppppp', excSqls);
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

remove

return excSqls;
}

Expand Down Expand Up @@ -392,10 +416,10 @@ EXECUTE FUNCTION update_${dbFieldName}_tsvector();`,
const { searchFields } = this;
const excSqls = [] as string[];
searchFields.forEach(({ dbFieldName }) => {
excSqls.push(this.getDropTriggerSql(dbFieldName));
excSqls.push(this.getDropTriggerFnSql(dbFieldName));
excSqls.push(this.getDropTsIndexSql(dbFieldName));
excSqls.push(this.getDropGinIndexSql(dbFieldName));
// excSqls.push(this.getDropTriggerSql(dbFieldName));
// excSqls.push(this.getDropTriggerFnSql(dbFieldName));
});

return excSqls;
Expand Down
Loading