Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Query optimizations #11244

Merged
merged 8 commits into from
Mar 5, 2025
Merged

Query optimizations #11244

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
ae8a8d1
remove XSS attempted communities
timolegros Mar 3, 2025
fa4a7db
clean-up query
timolegros Mar 4, 2025
631295a
improve searchProfiles query + uses
timolegros Mar 4, 2025
1008cbc
fix user mentions
timolegros Mar 4, 2025
afd3fee
fix migration
timolegros Mar 4, 2025
ebed75c
fixes
timolegros Mar 4, 2025
d9ff0ef
cache updates
timolegros Mar 5, 2025
7e1eb8f
lint
timolegros Mar 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 31 additions & 12 deletions libs/adapters/src/trpc/handlers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ export const query = <
tag: Tag,
options?: {
forceSecure?: boolean;
ttlSecs?: number;
ttlSecs?: number | ((input: z.infer<Input>) => number | undefined);
},
outMiddlewares?: Array<OutputMiddleware<Input, Output>>,
) => {
Expand All @@ -115,17 +115,30 @@ export const query = <
forceSecure: options?.forceSecure,
}).query(async ({ ctx, input }) => {
try {
const cacheKey = options?.ttlSecs
const cacheTTL =
typeof options?.ttlSecs === 'function'
? options.ttlSecs(input)
: options?.ttlSecs;

const cacheKey = cacheTTL
? `${factory.name}_${JSON.stringify(input)}`
: undefined;
if (cacheKey) {
const cachedReponse = await cache().getKey(
CacheNamespaces.Query_Response,
cacheKey,
);
if (cachedReponse) {
log.info(`Returning cached response for ${cacheKey}`);
return JSON.parse(cachedReponse);
// best effort return from cache (if cache goes down keep serving requests)
try {
const cachedResponse = await cache().getKey(
CacheNamespaces.Query_Response,
cacheKey,
);
if (cachedResponse) {
log.info(`Returning cached response for ${cacheKey}`);
return JSON.parse(cachedResponse);
}
} catch (e) {
log.error(
`Failed to return cached response for ${cacheKey}`,
e instanceof Error ? e : new Error(JSON.stringify(e)),
);
}
}
const response = await coreQuery(
Expand All @@ -137,14 +150,20 @@ export const query = <
false,
);
if (cacheKey) {
void cache()
cache()
.setKey(
CacheNamespaces.Query_Response,
cacheKey,
JSON.stringify(response),
options?.ttlSecs,
cacheTTL,
)
.then(() => log.info(`Cached response for ${cacheKey}`));
.then(() => log.info(`Cached response for ${cacheKey}`))
.catch((e) => {
log.error(
`Failed to cache response for ${cacheKey}`,
e instanceof Error ? e : new Error(JSON.stringify(e)),
);
});
}
return response;
} catch (error) {
Expand Down
196 changes: 85 additions & 111 deletions libs/model/src/community/GetCommunities.query.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ import { z } from 'zod';
import { models } from '../database';
import { buildChainNodeUrl } from '../utils/utils';

function iQ(conditional: unknown, query: string) {
return conditional ? query : '';
}

export function GetCommunities(): Query<typeof schemas.GetCommunities> {
return {
...schemas.GetCommunities,
Expand All @@ -17,7 +21,7 @@ export function GetCommunities(): Query<typeof schemas.GetCommunities> {
base,
network,
include_node_info,
include_last_30_day_thread_count,
include_last_30_day_thread_count: threadFilter,
stake_enabled,
has_groups,
tag_ids,
Expand Down Expand Up @@ -51,7 +55,14 @@ export function GetCommunities(): Query<typeof schemas.GetCommunities> {

const date30DaysAgo = new Date(+new Date() - 1000 * 24 * 60 * 60 * 30);
const communityCTE = `
WITH "community_CTE" AS (
WITH thread_counts AS (
SELECT community_id, COUNT(id)::int as thread_count
FROM "Threads"
WHERE created_at > '${date30DaysAgo.toISOString()}'
AND deleted_at IS NULL
GROUP BY community_id
),
"community_CTE" AS (
SELECT "Community"."id",
"Community"."chain_node_id",
"Community"."name",
Expand Down Expand Up @@ -90,65 +101,37 @@ export function GetCommunities(): Query<typeof schemas.GetCommunities> {
"Community"."redirect",
"Community"."snapshot_spaces",
"Community"."include_in_digest_email"
${
include_last_30_day_thread_count
? // eslint-disable-next-line max-len
`,(SELECT COUNT("Threads".id)::int FROM "Threads" WHERE "Threads".community_id = "Community".id AND "Threads".created_at > '${date30DaysAgo.toISOString()}' AND "Threads".deleted_at IS NULL) as last_30_day_thread_count`
: ''
}
${iQ(threadFilter, `, COALESCE(tc.thread_count, 0) as last_30_day_thread_count`)}
FROM "Communities" AS "Community"
${iQ(threadFilter, 'LEFT JOIN thread_counts tc ON tc.community_id = "Community".id')}
WHERE "Community"."active" = true
${
relevance_by === 'membership'
? `
AND name NOT LIKE '%<%'
AND name NOT LIKE '%>%'
AND name NOT LIKE '%\`%'
AND name NOT LIKE '%"%'
AND name NOT LIKE '%''%'
`
: ''
}
${
base
? `
AND "Community"."base" = '${base}'
`
: ''
}${
network
? `
AND "Community"."network" = '${network}'
`
: ''
}
${
community_type
? `
${iQ(base, `AND "Community"."base" = '${base}'`)}
${iQ(network, `AND "Community"."network" = '${network}'`)}
${iQ(
community_type,
`
AND (
SELECT "community_id"
SELECT 1
FROM "LaunchpadTokens" AS "LaunchpadTokens"
WHERE ( "LaunchpadTokens"."community_id" = "Community"."id" )
WHERE ( "LaunchpadTokens"."namespace" = "Community"."namespace_address" )
LIMIT 1
) IS ${community_type === CommunityType.Launchpad ? 'NOT' : ''} NULL
`
: ''
}
${
has_groups
? `
`,
)}
${iQ(
has_groups,
`
AND (
SELECT "community_id"
FROM "Groups" AS "Groups"
WHERE ( "Groups"."community_id" = "Community"."id" )
LIMIT 1
) IS NOT NULL
`
: ''
}
${
stake_enabled
? `
`,
)}
${iQ(
stake_enabled,
`
AND (
SELECT "community_id"
FROM "CommunityStakes" AS "CommunityStakes"
Expand All @@ -158,26 +141,25 @@ export function GetCommunities(): Query<typeof schemas.GetCommunities> {
)
LIMIT 1
) IS NOT NULL
`
: ''
}
${
filtering_tags && relevance_by !== 'tag_ids'
? `
`,
)}
${iQ(
filtering_tags && relevance_by !== 'tag_ids',
`
AND (
SELECT COUNT ( DISTINCT "CommunityTags"."tag_id" )
FROM "CommunityTags" AS "CommunityTags"
WHERE (
"CommunityTags"."community_id" = "Community"."id" AND
"CommunityTags"."tag_id" IN (:tag_ids)
)
) = ${tag_ids.length}
`
: ''
}
) = ${tag_ids?.length}
`,
)}
)
`,
communityTagsCTE = `
`;

const communityTagsCTE = `
, "CommunityTags_CTE" AS (
SELECT "community_id",
json_agg(json_build_object(
Expand All @@ -190,8 +172,8 @@ export function GetCommunities(): Query<typeof schemas.GetCommunities> {
WHERE "CommunityTags"."community_id" IN (SELECT "id" FROM "community_CTE")
GROUP BY "community_id"
)
`,
communityStakesCTE = `
`;
const communityStakesCTE = `
, "CommunityStakes_CTE" AS (
SELECT "community_id",
json_agg(json_build_object(
Expand All @@ -207,9 +189,11 @@ export function GetCommunities(): Query<typeof schemas.GetCommunities> {
WHERE "CommunityStakes"."community_id" IN (SELECT "id" FROM "community_CTE")
GROUP BY "community_id"
)
`,
groupsCTE = has_groups
? `
`;

const groupsCTE = iQ(
has_groups,
`
, "Groups_CTE" AS (
SELECT "community_id",
json_agg(json_build_object(
Expand All @@ -225,8 +209,8 @@ export function GetCommunities(): Query<typeof schemas.GetCommunities> {
WHERE "Groups"."community_id" IN (SELECT "id" FROM "community_CTE")
GROUP BY "community_id"
)
`
: '';
`,
);

const sql = `
${communityCTE}
Expand All @@ -238,10 +222,10 @@ export function GetCommunities(): Query<typeof schemas.GetCommunities> {
count(*) OVER() AS total,
"CommunityTags_CTE"."CommunityTags" as "CommunityTags",
"CommunityStakes_CTE"."CommunityStakes" as "CommunityStakes"
${has_groups ? `, "Groups_CTE"."Groups" as "groups"` : ''}
${
include_node_info
? `
${iQ(has_groups, `, "Groups_CTE"."Groups" as "groups"`)}
${iQ(
include_node_info,
`
, "ChainNode"."id" AS "ChainNode.id",
"ChainNode"."url" AS "ChainNode.url",
"ChainNode"."eth_chain_id" AS "ChainNode.eth_chain_id",
Expand All @@ -258,39 +242,29 @@ export function GetCommunities(): Query<typeof schemas.GetCommunities> {
"ChainNode"."slip44" AS "ChainNode.slip44",
"ChainNode"."created_at" AS "ChainNode.created_at",
"ChainNode"."updated_at" AS "ChainNode.updated_at"
`
: ''
}
`,
)}
FROM "community_CTE"
LEFT OUTER JOIN "CommunityTags_CTE" ON "community_CTE"."id" = "CommunityTags_CTE"."community_id"
LEFT OUTER JOIN "CommunityStakes_CTE" ON "community_CTE"."id" = "CommunityStakes_CTE"."community_id"
${
relevance_by === 'membership' && replacements.user_id
? // eslint-disable-next-line max-len
`LEFT OUTER JOIN
(SELECT DISTINCT ON (community_id) * FROM "Addresses" WHERE user_id = :user_id) authUserAddresses
ON "community_CTE"."id" = authUserAddresses.community_id
AND authUserAddresses.user_id = :user_id`
: ``
}
${
has_groups
? 'LEFT OUTER JOIN "Groups_CTE" ON "community_CTE"."id" = "Groups_CTE"."community_id"'
: ''
}
${
include_node_info || eth_chain_id || cosmos_chain_id
? `
LEFT OUTER JOIN "ChainNodes" AS "ChainNode" ON "community_CTE"."chain_node_id" = "ChainNode"."id"
${eth_chain_id && !cosmos_chain_id ? `WHERE "ChainNode".eth_chain_id = :eth_chain_id` : ''}
${cosmos_chain_id && !eth_chain_id ? `WHERE "ChainNode".cosmos_chain_id = :cosmos_chain_id` : ''}
`
: ''
}
ORDER BY
${
filtering_tags && relevance_by === 'tag_ids'
? `CASE
${iQ(
relevance_by === 'membership' && replacements.user_id,
`LEFT OUTER JOIN
(SELECT DISTINCT ON (community_id) * FROM "Addresses" WHERE user_id = :user_id) authUserAddresses
ON "community_CTE"."id" = authUserAddresses.community_id
AND authUserAddresses.user_id = :user_id`,
)}
${iQ(has_groups, 'LEFT OUTER JOIN "Groups_CTE" ON "community_CTE"."id" = "Groups_CTE"."community_id"')}
${iQ(
include_node_info || eth_chain_id || cosmos_chain_id,
`LEFT OUTER JOIN "ChainNodes" AS "ChainNode" ON "community_CTE"."chain_node_id" = "ChainNode"."id"
${iQ(eth_chain_id && !cosmos_chain_id, `WHERE "ChainNode".eth_chain_id = :eth_chain_id`)}
${iQ(cosmos_chain_id && !eth_chain_id, `WHERE "ChainNode".cosmos_chain_id = :cosmos_chain_id`)}`,
)}
ORDER BY
${iQ(
filtering_tags && relevance_by === 'tag_ids',
`CASE
WHEN jsonb_array_length("CommunityTags_CTE"."CommunityTags"::jsonb) = 0 IS NULL THEN 2
WHEN EXISTS (
SELECT 1
Expand All @@ -299,17 +273,17 @@ export function GetCommunities(): Query<typeof schemas.GetCommunities> {
) THEN 0
ELSE 1
END,
"CommunityTags_CTE"."CommunityTags"::jsonb,`
: ''
}
${
relevance_by === 'membership' && replacements.user_id
? `CASE
"CommunityTags_CTE"."CommunityTags"::jsonb,`,
)}
${iQ(
relevance_by === 'membership' && replacements.user_id,
`
CASE
WHEN authUserAddresses.user_id IS NOT NULL THEN 1
ELSE 0
END DESC,`
: ''
}
END DESC,
`,
)}
"community_CTE"."${order_col}" ${direction}
LIMIT ${limit}
OFFSET ${offset};
Expand Down
Loading
Loading