From ec66e7a24c0a8bef8579186765da937b811e6a50 Mon Sep 17 00:00:00 2001 From: Maxime Cyr <2829180+Redm4x@users.noreply.github.com> Date: Thu, 4 Apr 2024 11:45:00 -0400 Subject: [PATCH] Bugfixes/fix provider gpus UI (#143) * Return gpus in provider endpoints * Display gpus from feature discovery instead of capabilities * Add missing indexes * Change to inner join * . * Code improvements * . * Add array helper * Add createFilterUnique to deploy-web project --- api/src/routes/v1/providers/byAddress.ts | 8 ++ api/src/routes/v1/providers/list.ts | 8 ++ api/src/services/db/providerStatusService.ts | 74 ++++++++++++++----- api/src/types/provider.ts | 1 + api/src/utils/array/array.spec.ts | 19 +++++ api/src/utils/array/array.ts | 7 ++ api/src/utils/map/provider.ts | 21 +++++- .../components/providers/ProviderListRow.tsx | 4 +- .../components/providers/ProviderSpecs.tsx | 15 ++-- .../src/pages/providers/[owner]/index.tsx | 2 +- deploy-web/src/types/provider.ts | 1 + deploy-web/src/utils/array.ts | 7 ++ shared/dbSchemas/akash/providerSnapshot.ts | 5 +- .../dbSchemas/akash/providerSnapshotNode.ts | 10 ++- .../akash/providerSnapshotNodeCPU.ts | 3 +- .../akash/providerSnapshotNodeGPU.ts | 3 +- 16 files changed, 154 insertions(+), 34 deletions(-) create mode 100644 api/src/utils/array/array.spec.ts create mode 100644 api/src/utils/array/array.ts create mode 100644 deploy-web/src/utils/array.ts diff --git a/api/src/routes/v1/providers/byAddress.ts b/api/src/routes/v1/providers/byAddress.ts index 4267a66c7..1359e527a 100644 --- a/api/src/routes/v1/providers/byAddress.ts +++ b/api/src/routes/v1/providers/byAddress.ts @@ -62,6 +62,14 @@ const route = createRoute({ memory: z.number(), storage: z.number() }), + gpuModels: z.array( + z.object({ + vendor: z.string(), + model: z.string(), + ram: z.string(), + interface: z.string() + }) + ), attributes: z.array( z.object({ key: z.string(), diff --git a/api/src/routes/v1/providers/list.ts b/api/src/routes/v1/providers/list.ts index 6349b58eb..4f15cd086 100644 --- a/api/src/routes/v1/providers/list.ts +++ b/api/src/routes/v1/providers/list.ts @@ -55,6 +55,14 @@ const route = createRoute({ memory: z.number(), storage: z.number() }), + gpuModels: z.array( + z.object({ + vendor: z.string(), + model: z.string(), + ram: z.string(), + interface: z.string() + }) + ), attributes: z.array( z.object({ key: z.string(), diff --git a/api/src/services/db/providerStatusService.ts b/api/src/services/db/providerStatusService.ts index 08c901e7e..2f5c1435a 100644 --- a/api/src/services/db/providerStatusService.ts +++ b/api/src/services/db/providerStatusService.ts @@ -1,4 +1,4 @@ -import { Provider, ProviderAttribute, ProviderAttributeSignature } from "@shared/dbSchemas/akash"; +import { Provider, ProviderAttribute, ProviderAttributeSignature, ProviderSnapshotNode, ProviderSnapshotNodeGPU } from "@shared/dbSchemas/akash"; import { ProviderSnapshot } from "@shared/dbSchemas/akash/providerSnapshot"; import { toUTC } from "@src/utils"; import { add } from "date-fns"; @@ -42,7 +42,7 @@ export async function getNetworkCapacity() { } export const getProviderList = async () => { - const providers = await Provider.findAll({ + const providersWithAttributesAndAuditors = await Provider.findAll({ where: { deletedHeight: null }, @@ -56,13 +56,40 @@ export const getProviderList = async () => { } ] }); - const filteredProviders = providers.filter((value, index, self) => self.map((x) => x.hostUri).lastIndexOf(value.hostUri) === index); + + const providerWithNodes = await Provider.findAll({ + attributes: ["owner"], + where: { + deletedHeight: null + }, + include: [ + { + model: ProviderSnapshot, + attributes: ["id"], + required: true, + as: "lastSnapshot", + include: [ + { + model: ProviderSnapshotNode, + attributes: ["id"], + required: true, + include: [{ model: ProviderSnapshotNodeGPU, required: true }] + } + ] + } + ] + }); + + const distinctProviders = providersWithAttributesAndAuditors.filter((value, index, self) => self.map((x) => x.hostUri).lastIndexOf(value.hostUri) === index); const providerAttributeSchemaQuery = getProviderAttributesSchema(); const auditorsQuery = getAuditors(); const [auditors, providerAttributeSchema] = await Promise.all([auditorsQuery, providerAttributeSchemaQuery]); - return filteredProviders.map((x) => mapProviderToList(x, providerAttributeSchema, auditors)); + return distinctProviders.map((x) => { + const nodes = providerWithNodes.find((p) => p.owner === x.owner)?.lastSnapshot?.nodes; + return mapProviderToList(x, providerAttributeSchema, auditors, nodes); + }); }; export const getProviderDetail = async (address: string): Promise => { @@ -78,32 +105,43 @@ export const getProviderDetail = async (address: string): Promise ({ + ...mapProviderToList(provider, providerAttributeSchema, auditors, lastSnapshot?.nodes), + uptime: uptimeSnapshots.map((ps) => ({ id: ps.id, isOnline: ps.isOnline, checkDate: ps.checkDate diff --git a/api/src/types/provider.ts b/api/src/types/provider.ts index fbbf86deb..c7a942ea6 100644 --- a/api/src/types/provider.ts +++ b/api/src/types/provider.ts @@ -22,6 +22,7 @@ export interface ProviderList { isValidVersion: boolean; isOnline: boolean; isAudited: boolean; + gpuModels: { vendor: string; model: string; ram: string; interface: string }[]; activeStats: { cpu: number; gpu: number; diff --git a/api/src/utils/array/array.spec.ts b/api/src/utils/array/array.spec.ts new file mode 100644 index 000000000..ff1263d2c --- /dev/null +++ b/api/src/utils/array/array.spec.ts @@ -0,0 +1,19 @@ +import { createFilterUnique } from "./array"; + +describe("array helpers", () => { + describe("createFilterUnique", () => { + it("should return a functionning unique filter with default equality matcher", () => { + const arrayWithDuplicate = [1, 2, 2, 3, 3, 3]; + const expected = [1, 2, 3]; + + expect(arrayWithDuplicate.filter(createFilterUnique())).toEqual(expected); + }); + + it("should return a functionning unique filter with custom matcher", () => { + const arrayWithDuplicate = [{ v: 1 }, { v: 2 }, { v: 2 }, { v: 3 }, { v: 3 }, { v: 3 }]; + const expected = [{ v: 1 }, { v: 2 }, { v: 3 }]; + + expect(arrayWithDuplicate.filter(createFilterUnique((a, b) => a.v === b.v))).toEqual(expected); + }); + }); +}); diff --git a/api/src/utils/array/array.ts b/api/src/utils/array/array.ts new file mode 100644 index 000000000..3a22c4800 --- /dev/null +++ b/api/src/utils/array/array.ts @@ -0,0 +1,7 @@ +type Matcher = (a: T, b: T) => boolean; + +export function createFilterUnique(matcher: Matcher = (a, b) => a === b): (value: T, index: number, array: T[]) => boolean { + return (value, index, array) => { + return array.findIndex((other) => matcher(value, other)) === index; + }; +} diff --git a/api/src/utils/map/provider.ts b/api/src/utils/map/provider.ts index 353db4479..f8b91f24b 100644 --- a/api/src/utils/map/provider.ts +++ b/api/src/utils/map/provider.ts @@ -1,10 +1,17 @@ -import { Provider } from "@shared/dbSchemas/akash"; +import { Provider, ProviderSnapshotNode } from "@shared/dbSchemas/akash"; import { Auditor, ProviderAttributesSchema, ProviderList } from "@src/types/provider"; +import { createFilterUnique } from "../array/array"; import semver from "semver"; -export const mapProviderToList = (provider: Provider, providerAttributeSchema: ProviderAttributesSchema, auditors: Array): ProviderList => { +export const mapProviderToList = ( + provider: Provider, + providerAttributeSchema: ProviderAttributesSchema, + auditors: Array, + nodes?: ProviderSnapshotNode[] +): ProviderList => { const isValidVersion = provider.cosmosSdkVersion ? semver.gte(provider.cosmosSdkVersion, "v0.45.9") : false; const name = provider.isOnline ? new URL(provider.hostUri).hostname : null; + const gpuModels = getDistinctGpuModelsFromNodes(nodes || []); return { owner: provider.owner, @@ -42,6 +49,7 @@ export const mapProviderToList = (provider: Provider, providerAttributeSchema: P memory: isValidVersion ? provider.availableMemory : 0, storage: isValidVersion ? provider.availableStorage : 0 }, + gpuModels: gpuModels, uptime1d: provider.uptime1d, uptime7d: provider.uptime7d, uptime30d: provider.uptime30d, @@ -83,6 +91,15 @@ export const mapProviderToList = (provider: Provider, providerAttributeSchema: P } as ProviderList; }; +function getDistinctGpuModelsFromNodes(nodes: ProviderSnapshotNode[]) { + const gpuModels = nodes.flatMap((x) => x.gpus).map((x) => ({ vendor: x.vendor, model: x.name, ram: x.memorySize, interface: x.interface })); + const distinctGpuModels = gpuModels.filter( + createFilterUnique((a, b) => a.vendor === b.vendor && a.model === b.model && a.ram === b.ram && a.interface === b.interface) + ); + + return distinctGpuModels; +} + export const getProviderAttributeValue = ( key: keyof ProviderAttributesSchema, provider: Provider, diff --git a/deploy-web/src/components/providers/ProviderListRow.tsx b/deploy-web/src/components/providers/ProviderListRow.tsx index 19937ad6f..e9846b960 100644 --- a/deploy-web/src/components/providers/ProviderListRow.tsx +++ b/deploy-web/src/components/providers/ProviderListRow.tsx @@ -16,8 +16,8 @@ import { Uptime } from "./Uptime"; import React from "react"; import { hasSomeParentTheClass } from "@src/utils/domUtils"; import { cx } from "@emotion/css"; -import CheckIcon from "@mui/icons-material/Check"; import WarningIcon from "@mui/icons-material/Warning"; +import { createFilterUnique } from "@src/utils/array"; const useStyles = makeStyles()(theme => ({ root: { @@ -62,7 +62,7 @@ export const ProviderListRow: React.FunctionComponent = ({ provider }) => const _totalStorage = provider.isOnline ? bytesToShrink(provider.availableStats.storage + provider.pendingStats.storage + provider.activeStats.storage) : null; - const gpuModels = provider.hardwareGpuModels.map(gpu => gpu.substring(gpu.lastIndexOf(" ") + 1, gpu.length)); + const gpuModels = provider.gpuModels.map(x => x.model).filter(createFilterUnique()); const onStarClick = event => { event.preventDefault(); diff --git a/deploy-web/src/components/providers/ProviderSpecs.tsx b/deploy-web/src/components/providers/ProviderSpecs.tsx index bacbdc164..78515fb86 100644 --- a/deploy-web/src/components/providers/ProviderSpecs.tsx +++ b/deploy-web/src/components/providers/ProviderSpecs.tsx @@ -1,10 +1,9 @@ import { Box, Chip, Paper } from "@mui/material"; import { makeStyles } from "tss-react/mui"; -import { useRouter } from "next/router"; import { ClientProviderDetailWithStatus } from "@src/types/provider"; import { LabelValue } from "../shared/LabelValue"; import CheckIcon from "@mui/icons-material/Check"; -import { ProviderAttributesSchema } from "@src/types/providerAttributes"; +import { createFilterUnique } from "@src/utils/array"; const useStyles = makeStyles()(theme => ({ root: { @@ -20,12 +19,16 @@ const useStyles = makeStyles()(theme => ({ type Props = { provider: Partial; - providerAttributesSchema: ProviderAttributesSchema; }; -export const ProviderSpecs: React.FunctionComponent = ({ provider, providerAttributesSchema }) => { +export const ProviderSpecs: React.FunctionComponent = ({ provider }) => { const { classes } = useStyles(); - const router = useRouter(); + + const gpuModels = + provider?.gpuModels + ?.map(x => x.model + " " + x.ram) + .filter(createFilterUnique()) + .sort((a, b) => a.localeCompare(b)) || []; return ( @@ -41,7 +44,7 @@ export const ProviderSpecs: React.FunctionComponent = ({ provider, provid ( + value={gpuModels.map(x => ( ))} /> diff --git a/deploy-web/src/pages/providers/[owner]/index.tsx b/deploy-web/src/pages/providers/[owner]/index.tsx index 6dbb106e3..49d5943d0 100644 --- a/deploy-web/src/pages/providers/[owner]/index.tsx +++ b/deploy-web/src/pages/providers/[owner]/index.tsx @@ -186,7 +186,7 @@ const ProviderDetailPage: React.FunctionComponent = ({ owner, _provider } Specs - + Features diff --git a/deploy-web/src/types/provider.ts b/deploy-web/src/types/provider.ts index 4e131c5be..e73b81744 100644 --- a/deploy-web/src/types/provider.ts +++ b/deploy-web/src/types/provider.ts @@ -206,6 +206,7 @@ export interface ApiProviderList { isValidVersion: boolean; isOnline: boolean; isAudited: boolean; + gpuModels: { vendor: string; model: string; ram: string; interface: string }[]; activeStats: { cpu: number; gpu: number; diff --git a/deploy-web/src/utils/array.ts b/deploy-web/src/utils/array.ts new file mode 100644 index 000000000..37190595b --- /dev/null +++ b/deploy-web/src/utils/array.ts @@ -0,0 +1,7 @@ +type Matcher = (a: T, b: T) => boolean; + +export function createFilterUnique(matcher: Matcher = (a, b) => a === b): (value: T, index: number, array: T[]) => boolean { + return (value, index, array) => { + return array.findIndex(other => matcher(value, other)) === index; + }; +} diff --git a/shared/dbSchemas/akash/providerSnapshot.ts b/shared/dbSchemas/akash/providerSnapshot.ts index 96919df51..1ed675d1d 100644 --- a/shared/dbSchemas/akash/providerSnapshot.ts +++ b/shared/dbSchemas/akash/providerSnapshot.ts @@ -1,6 +1,7 @@ -import { Column, Default, Model, PrimaryKey, Table } from "sequelize-typescript"; +import { Column, Default, HasMany, Model, PrimaryKey, Table } from "sequelize-typescript"; import { DataTypes } from "sequelize"; import { Required } from "../decorators/requiredDecorator"; +import { ProviderSnapshotNode } from "./providerSnapshotNode"; @Table({ modelName: "providerSnapshot", @@ -33,4 +34,6 @@ export class ProviderSnapshot extends Model { @Column(DataTypes.BIGINT) availableGPU?: number; @Column(DataTypes.BIGINT) availableMemory?: number; @Column(DataTypes.BIGINT) availableStorage?: number; + + @HasMany(() => ProviderSnapshotNode, "snapshotId") nodes: ProviderSnapshotNode[]; } diff --git a/shared/dbSchemas/akash/providerSnapshotNode.ts b/shared/dbSchemas/akash/providerSnapshotNode.ts index 280b10b27..8526e09bb 100644 --- a/shared/dbSchemas/akash/providerSnapshotNode.ts +++ b/shared/dbSchemas/akash/providerSnapshotNode.ts @@ -1,9 +1,12 @@ -import { Column, Default, Model, PrimaryKey, Table } from "sequelize-typescript"; +import { Column, Default, HasMany, Model, PrimaryKey, Table } from "sequelize-typescript"; import { DataTypes } from "sequelize"; import { Required } from "../decorators/requiredDecorator"; +import { ProviderSnapshotNodeGPU } from "./providerSnapshotNodeGPU"; +import { ProviderSnapshotNodeCPU } from "./providerSnapshotNodeCPU"; @Table({ - modelName: "providerSnapshotNode" + modelName: "providerSnapshotNode", + indexes: [{ unique: false, fields: ["snapshotId"] }] }) export class ProviderSnapshotNode extends Model { @Required @PrimaryKey @Default(DataTypes.UUIDV4) @Column(DataTypes.UUID) id: string; @@ -26,4 +29,7 @@ export class ProviderSnapshotNode extends Model { @Column(DataTypes.BIGINT) gpuAllocatable: number; @Column(DataTypes.BIGINT) gpuAllocated: number; + + @HasMany(() => ProviderSnapshotNodeGPU, "snapshotNodeId") gpus: ProviderSnapshotNodeGPU[]; + @HasMany(() => ProviderSnapshotNodeCPU, "snapshotNodeId") cpus: ProviderSnapshotNodeCPU[]; } diff --git a/shared/dbSchemas/akash/providerSnapshotNodeCPU.ts b/shared/dbSchemas/akash/providerSnapshotNodeCPU.ts index ca54f6029..3186c9edb 100644 --- a/shared/dbSchemas/akash/providerSnapshotNodeCPU.ts +++ b/shared/dbSchemas/akash/providerSnapshotNodeCPU.ts @@ -3,7 +3,8 @@ import { DataTypes } from "sequelize"; import { Required } from "../decorators/requiredDecorator"; @Table({ - modelName: "providerSnapshotNodeCPU" + modelName: "providerSnapshotNodeCPU", + indexes: [{ unique: false, fields: ["snapshotNodeId"] }] }) export class ProviderSnapshotNodeCPU extends Model { @Required @PrimaryKey @Default(DataTypes.UUIDV4) @Column(DataTypes.UUID) id: string; diff --git a/shared/dbSchemas/akash/providerSnapshotNodeGPU.ts b/shared/dbSchemas/akash/providerSnapshotNodeGPU.ts index 58aba0437..079477350 100644 --- a/shared/dbSchemas/akash/providerSnapshotNodeGPU.ts +++ b/shared/dbSchemas/akash/providerSnapshotNodeGPU.ts @@ -3,7 +3,8 @@ import { DataTypes } from "sequelize"; import { Required } from "../decorators/requiredDecorator"; @Table({ - modelName: "providerSnapshotNodeGPU" + modelName: "providerSnapshotNodeGPU", + indexes: [{ unique: false, fields: ["snapshotNodeId"] }] }) export class ProviderSnapshotNodeGPU extends Model { @Required @PrimaryKey @Default(DataTypes.UUIDV4) @Column(DataTypes.UUID) id: string;