diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupPropertiesMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupPropertiesMapper.java index a6cfded9865d90..2da2fa2a58a6af 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupPropertiesMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupPropertiesMapper.java @@ -3,8 +3,11 @@ import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.MLModelGroupProperties; +import com.linkedin.datahub.graphql.generated.MLModelLineageInfo; import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper; +import com.linkedin.datahub.graphql.types.common.mappers.TimeStampToAuditStampMapper; import com.linkedin.datahub.graphql.types.mappers.EmbeddedModelMapper; +import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -33,10 +36,40 @@ public MLModelGroupProperties apply( result.setVersion(VersionTagMapper.map(context, mlModelGroupProperties.getVersion())); } result.setCreatedAt(mlModelGroupProperties.getCreatedAt()); + if (mlModelGroupProperties.hasCreated()) { + result.setCreated( + TimeStampToAuditStampMapper.map(context, mlModelGroupProperties.getCreated())); + } + if (mlModelGroupProperties.getName() != null) { + result.setName(mlModelGroupProperties.getName()); + } else { + // backfill name from URN for backwards compatibility + result.setName(entityUrn.getEntityKey().get(1)); // indexed access is safe here + } + + if (mlModelGroupProperties.hasLastModified()) { + result.setLastModified( + TimeStampToAuditStampMapper.map(context, mlModelGroupProperties.getLastModified())); + } result.setCustomProperties( CustomPropertiesMapper.map(mlModelGroupProperties.getCustomProperties(), entityUrn)); + final MLModelLineageInfo lineageInfo = new MLModelLineageInfo(); + if (mlModelGroupProperties.hasTrainingJobs()) { + lineageInfo.setTrainingJobs( + mlModelGroupProperties.getTrainingJobs().stream() + .map(urn -> urn.toString()) + .collect(Collectors.toList())); + } + if (mlModelGroupProperties.hasDownstreamJobs()) { + lineageInfo.setDownstreamJobs( + mlModelGroupProperties.getDownstreamJobs().stream() + .map(urn -> urn.toString()) + .collect(Collectors.toList())); + } + result.setMlModelLineageInfo(lineageInfo); + return result; } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelPropertiesMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelPropertiesMapper.java index 7b00fe88f2d683..1f1003dea720c3 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelPropertiesMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelPropertiesMapper.java @@ -5,6 +5,7 @@ import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.MLModelGroup; +import com.linkedin.datahub.graphql.generated.MLModelLineageInfo; import com.linkedin.datahub.graphql.generated.MLModelProperties; import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper; import com.linkedin.datahub.graphql.types.common.mappers.TimeStampToAuditStampMapper; @@ -87,6 +88,20 @@ public MLModelProperties apply( .collect(Collectors.toList())); } result.setTags(mlModelProperties.getTags()); + final MLModelLineageInfo lineageInfo = new MLModelLineageInfo(); + if (mlModelProperties.hasTrainingJobs()) { + lineageInfo.setTrainingJobs( + mlModelProperties.getTrainingJobs().stream() + .map(urn -> urn.toString()) + .collect(Collectors.toList())); + } + if (mlModelProperties.hasDownstreamJobs()) { + lineageInfo.setDownstreamJobs( + mlModelProperties.getDownstreamJobs().stream() + .map(urn -> urn.toString()) + .collect(Collectors.toList())); + } + result.setMlModelLineageInfo(lineageInfo); return result; } diff --git a/datahub-graphql-core/src/main/resources/lineage.graphql b/datahub-graphql-core/src/main/resources/lineage.graphql index 975d013a448058..abb1446421858f 100644 --- a/datahub-graphql-core/src/main/resources/lineage.graphql +++ b/datahub-graphql-core/src/main/resources/lineage.graphql @@ -25,3 +25,32 @@ input LineageEdge { """ upstreamUrn: String! } + +""" +Represents lineage information for ML entities. +""" +type MLModelLineageInfo { + """ + List of jobs or processes used to train the model. + """ + trainingJobs: [String!] + + """ + List of jobs or processes that use this model. + """ + downstreamJobs: [String!] +} + +extend type MLModelProperties { + """ + Information related to lineage to this model group + """ + mlModelLineageInfo: MLModelLineageInfo +} + +extend type MLModelGroupProperties { + """ + Information related to lineage to this model group + """ + mlModelLineageInfo: MLModelLineageInfo +} \ No newline at end of file diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupPropertiesMapperTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupPropertiesMapperTest.java new file mode 100644 index 00000000000000..fc738837c09d17 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupPropertiesMapperTest.java @@ -0,0 +1,68 @@ +package com.linkedin.datahub.graphql.types.mlmodel.mappers; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; + +import com.linkedin.common.urn.Urn; +import com.linkedin.ml.metadata.MLModelGroupProperties; +import java.net.URISyntaxException; +import org.testng.annotations.Test; + +public class MLModelGroupPropertiesMapperTest { + + @Test + public void testMapMLModelGroupProperties() throws URISyntaxException { + // Create backend ML Model Group Properties + MLModelGroupProperties input = new MLModelGroupProperties(); + + // Set description + input.setDescription("a ml trust model group"); + + // Set Name + input.setName("ML trust model group"); + + // Create URN + Urn groupUrn = + Urn.createFromString( + "urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,another-group,PROD)"); + + // Map the properties + com.linkedin.datahub.graphql.generated.MLModelGroupProperties result = + MLModelGroupPropertiesMapper.map(null, input, groupUrn); + + // Verify mapped properties + assertNotNull(result); + assertEquals(result.getDescription(), "a ml trust model group"); + assertEquals(result.getName(), "ML trust model group"); + + // Verify lineage info is null as in the mock data + assertNotNull(result.getMlModelLineageInfo()); + assertNull(result.getMlModelLineageInfo().getTrainingJobs()); + assertNull(result.getMlModelLineageInfo().getDownstreamJobs()); + } + + @Test + public void testMapWithMinimalProperties() throws URISyntaxException { + // Create backend ML Model Group Properties with minimal information + MLModelGroupProperties input = new MLModelGroupProperties(); + + // Create URN + Urn groupUrn = + Urn.createFromString( + "urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,another-group,PROD)"); + + // Map the properties + com.linkedin.datahub.graphql.generated.MLModelGroupProperties result = + MLModelGroupPropertiesMapper.map(null, input, groupUrn); + + // Verify basic mapping with minimal properties + assertNotNull(result); + assertNull(result.getDescription()); + + // Verify lineage info is null + assertNotNull(result.getMlModelLineageInfo()); + assertNull(result.getMlModelLineageInfo().getTrainingJobs()); + assertNull(result.getMlModelLineageInfo().getDownstreamJobs()); + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelPropertiesMapperTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelPropertiesMapperTest.java new file mode 100644 index 00000000000000..17fa7a0abe1396 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelPropertiesMapperTest.java @@ -0,0 +1,187 @@ +package com.linkedin.datahub.graphql.types.mlmodel.mappers; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; + +import com.linkedin.common.MLFeatureUrnArray; +import com.linkedin.common.TimeStamp; +import com.linkedin.common.VersionTag; +import com.linkedin.common.url.Url; +import com.linkedin.common.urn.MLFeatureUrn; +import com.linkedin.common.urn.MLModelUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.StringArray; +import com.linkedin.data.template.StringMap; +import com.linkedin.ml.metadata.MLHyperParam; +import com.linkedin.ml.metadata.MLHyperParamArray; +import com.linkedin.ml.metadata.MLMetric; +import com.linkedin.ml.metadata.MLMetricArray; +import com.linkedin.ml.metadata.MLModelProperties; +import java.net.URISyntaxException; +import org.testng.annotations.Test; + +public class MLModelPropertiesMapperTest { + + @Test + public void testMapMLModelProperties() throws URISyntaxException { + MLModelProperties input = new MLModelProperties(); + + // Set basic properties + input.setName("TestModel"); + input.setDescription("A test ML model"); + input.setType("Classification"); + + // Set version + VersionTag versionTag = new VersionTag(); + versionTag.setVersionTag("1.0.0"); + input.setVersion(versionTag); + + // Set external URL + Url externalUrl = new Url("https://example.com/model"); + input.setExternalUrl(externalUrl); + + // Set created and last modified timestamps + TimeStamp createdTimeStamp = new TimeStamp(); + createdTimeStamp.setTime(1000L); + Urn userUrn = Urn.createFromString("urn:li:corpuser:test"); + createdTimeStamp.setActor(userUrn); + input.setCreated(createdTimeStamp); + + TimeStamp lastModifiedTimeStamp = new TimeStamp(); + lastModifiedTimeStamp.setTime(2000L); + lastModifiedTimeStamp.setActor(userUrn); + input.setLastModified(lastModifiedTimeStamp); + + // Set custom properties + StringMap customProps = new StringMap(); + customProps.put("key1", "value1"); + customProps.put("key2", "value2"); + input.setCustomProperties(customProps); + + // Set hyper parameters + MLHyperParamArray hyperParams = new MLHyperParamArray(); + MLHyperParam hyperParam1 = new MLHyperParam(); + hyperParam1.setName("learning_rate"); + hyperParam1.setValue("0.01"); + hyperParams.add(hyperParam1); + input.setHyperParams(hyperParams); + + // Set training metrics + MLMetricArray trainingMetrics = new MLMetricArray(); + MLMetric metric1 = new MLMetric(); + metric1.setName("accuracy"); + metric1.setValue("0.95"); + trainingMetrics.add(metric1); + input.setTrainingMetrics(trainingMetrics); + + // Set ML features + MLFeatureUrnArray mlFeatures = new MLFeatureUrnArray(); + MLFeatureUrn featureUrn = MLFeatureUrn.createFromString("urn:li:mlFeature:(dataset,feature)"); + mlFeatures.add(featureUrn); + input.setMlFeatures(mlFeatures); + + // Set tags + StringArray tags = new StringArray(); + tags.add("tag1"); + tags.add("tag2"); + input.setTags(tags); + + // Set training and downstream jobs + input.setTrainingJobs( + new com.linkedin.common.UrnArray(Urn.createFromString("urn:li:dataJob:train"))); + input.setDownstreamJobs( + new com.linkedin.common.UrnArray(Urn.createFromString("urn:li:dataJob:predict"))); + + // Create ML Model URN + MLModelUrn modelUrn = + MLModelUrn.createFromString( + "urn:li:mlModel:(urn:li:dataPlatform:sagemaker,unittestmodel,PROD)"); + + // Map the properties + com.linkedin.datahub.graphql.generated.MLModelProperties result = + MLModelPropertiesMapper.map(null, input, modelUrn); + + // Verify mapped properties + assertNotNull(result); + assertEquals(result.getName(), "TestModel"); + assertEquals(result.getDescription(), "A test ML model"); + assertEquals(result.getType(), "Classification"); + assertEquals(result.getVersion(), "1.0.0"); + assertEquals(result.getExternalUrl(), "https://example.com/model"); + + // Verify audit stamps + assertNotNull(result.getCreated()); + assertEquals(result.getCreated().getTime().longValue(), 1000L); + assertEquals(result.getCreated().getActor(), userUrn.toString()); + + assertNotNull(result.getLastModified()); + assertEquals(result.getLastModified().getTime().longValue(), 2000L); + assertEquals(result.getLastModified().getActor(), userUrn.toString()); + + // Verify custom properties + assertNotNull(result.getCustomProperties()); + + // Verify hyper parameters + assertNotNull(result.getHyperParams()); + assertEquals(result.getHyperParams().size(), 1); + assertEquals(result.getHyperParams().get(0).getName(), "learning_rate"); + assertEquals(result.getHyperParams().get(0).getValue(), "0.01"); + + // Verify training metrics + assertNotNull(result.getTrainingMetrics()); + assertEquals(result.getTrainingMetrics().size(), 1); + assertEquals(result.getTrainingMetrics().get(0).getName(), "accuracy"); + assertEquals(result.getTrainingMetrics().get(0).getValue(), "0.95"); + + // Verify ML features + assertNotNull(result.getMlFeatures()); + assertEquals(result.getMlFeatures().size(), 1); + assertEquals(result.getMlFeatures().get(0), featureUrn.toString()); + + // Verify tags + assertNotNull(result.getTags()); + assertEquals(result.getTags().get(0), "tag1"); + assertEquals(result.getTags().get(1), "tag2"); + + // Verify lineage info + assertNotNull(result.getMlModelLineageInfo()); + assertEquals(result.getMlModelLineageInfo().getTrainingJobs().size(), 1); + assertEquals(result.getMlModelLineageInfo().getTrainingJobs().get(0), "urn:li:dataJob:train"); + assertEquals(result.getMlModelLineageInfo().getDownstreamJobs().size(), 1); + assertEquals( + result.getMlModelLineageInfo().getDownstreamJobs().get(0), "urn:li:dataJob:predict"); + } + + @Test + public void testMapWithMissingName() throws URISyntaxException { + MLModelProperties input = new MLModelProperties(); + MLModelUrn modelUrn = + MLModelUrn.createFromString( + "urn:li:mlModel:(urn:li:dataPlatform:sagemaker,missingnamemodel,PROD)"); + + com.linkedin.datahub.graphql.generated.MLModelProperties result = + MLModelPropertiesMapper.map(null, input, modelUrn); + + // Verify that name is extracted from URN when not present in input + assertEquals(result.getName(), "missingnamemodel"); + } + + @Test + public void testMapWithMinimalProperties() throws URISyntaxException { + MLModelProperties input = new MLModelProperties(); + MLModelUrn modelUrn = + MLModelUrn.createFromString( + "urn:li:mlModel:(urn:li:dataPlatform:sagemaker,minimalmodel,PROD)"); + + com.linkedin.datahub.graphql.generated.MLModelProperties result = + MLModelPropertiesMapper.map(null, input, modelUrn); + + // Verify basic mapping with minimal properties + assertNotNull(result); + assertEquals(result.getName(), "minimalmodel"); + assertNull(result.getDescription()); + assertNull(result.getType()); + assertNull(result.getVersion()); + } +} diff --git a/datahub-web-react/src/app/entity/EntityPage.tsx b/datahub-web-react/src/app/entity/EntityPage.tsx index 916fa417954126..d05f75694ab94e 100644 --- a/datahub-web-react/src/app/entity/EntityPage.tsx +++ b/datahub-web-react/src/app/entity/EntityPage.tsx @@ -66,6 +66,7 @@ export const EntityPage = ({ entityType }: Props) => { entityType === EntityType.MlfeatureTable || entityType === EntityType.MlmodelGroup || entityType === EntityType.GlossaryTerm || + entityType === EntityType.DataProcessInstance || entityType === EntityType.GlossaryNode; return ( diff --git a/datahub-web-react/src/app/entity/dataProcessInstance/DataProcessInstanceEntity.tsx b/datahub-web-react/src/app/entity/dataProcessInstance/DataProcessInstanceEntity.tsx index 9bb9bd745d1ee6..bdf77959e97c7f 100644 --- a/datahub-web-react/src/app/entity/dataProcessInstance/DataProcessInstanceEntity.tsx +++ b/datahub-web-react/src/app/entity/dataProcessInstance/DataProcessInstanceEntity.tsx @@ -1,12 +1,7 @@ import React from 'react'; import { ApiOutlined } from '@ant-design/icons'; -import { - DataProcessInstance, - Entity as GeneratedEntity, - EntityType, - OwnershipType, - SearchResult, -} from '../../../types.generated'; +import { Entity as GraphQLEntity } from '@types'; +import { DataProcessInstance, EntityType, OwnershipType, SearchResult } from '../../../types.generated'; import { Preview } from './preview/Preview'; import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from '../Entity'; import { EntityProfile } from '../shared/containers/profile/EntityProfile'; @@ -23,32 +18,21 @@ import { EntityMenuItems } from '../shared/EntityDropdown/EntityDropdown'; import { capitalizeFirstLetterOnly } from '../../shared/textUtil'; import DataProductSection from '../shared/containers/profile/sidebar/DataProduct/DataProductSection'; import { getDataProduct } from '../shared/utils'; -// import SummaryTab from './profile/DataProcessInstaceSummary'; +import SummaryTab from './profile/DataProcessInstanceSummary'; -// const getProcessPlatformName = (data?: DataProcessInstance): string => { -// return ( -// data?.dataPlatformInstance?.platform?.properties?.displayName || -// capitalizeFirstLetterOnly(data?.dataPlatformInstance?.platform?.name) || -// '' -// ); -// }; - -const getParentEntities = (data: DataProcessInstance): GeneratedEntity[] => { +const getParentEntities = (data: DataProcessInstance): GraphQLEntity[] => { const parentEntity = data?.relationships?.relationships?.find( (rel) => rel.type === 'InstanceOf' && rel.entity?.type === EntityType.DataJob, ); - if (!parentEntity?.entity) return []; + if (!parentEntity || !parentEntity.entity) { + return []; + } - // Convert to GeneratedEntity - return [ - { - type: parentEntity.entity.type, - urn: (parentEntity.entity as any).urn, // Make sure urn exists - relationships: (parentEntity.entity as any).relationships, - }, - ]; + // First cast to unknown, then to Entity with proper type + return [parentEntity.entity]; }; + /** * Definition of the DataHub DataProcessInstance entity. */ @@ -97,18 +81,13 @@ export class DataProcessInstanceEntity implements Entity { urn={urn} entityType={EntityType.DataProcessInstance} useEntityQuery={this.useEntityQuery} - // useUpdateQuery={useUpdateDataProcessInstanceMutation} getOverrideProperties={this.getOverridePropertiesFromEntity} headerDropdownItems={new Set([EntityMenuItems.UPDATE_DEPRECATION, EntityMenuItems.RAISE_INCIDENT])} tabs={[ - // { - // name: 'Documentation', - // component: DocumentationTab, - // }, - // { - // name: 'Summary', - // component: SummaryTab, - // }, + { + name: 'Summary', + component: SummaryTab, + }, { name: 'Lineage', component: LineageTab, @@ -117,14 +96,6 @@ export class DataProcessInstanceEntity implements Entity { name: 'Properties', component: PropertiesTab, }, - // { - // name: 'Incidents', - // component: IncidentTab, - // getDynamicName: (_, processInstance) => { - // const activeIncidentCount = processInstance?.dataProcessInstance?.activeIncidents.total; - // return `Incidents${(activeIncidentCount && ` (${activeIncidentCount})`) || ''}`; - // }, - // }, ]} sidebarSections={this.getSidebarSections()} /> @@ -181,13 +152,11 @@ export class DataProcessInstanceEntity implements Entity { platformLogo={data?.dataPlatformInstance?.platform?.properties?.logoUrl} owners={null} globalTags={null} - // domain={data.domain?.domain} dataProduct={getDataProduct(genericProperties?.dataProduct)} externalUrl={data.properties?.externalUrl} parentContainers={data.parentContainers} parentEntities={parentEntities} container={data.container || undefined} - // health={data.health} /> ); }; @@ -196,6 +165,9 @@ export class DataProcessInstanceEntity implements Entity { const data = result.entity as DataProcessInstance; const genericProperties = this.getGenericEntityProperties(data); const parentEntities = getParentEntities(data); + + const firstState = data?.state && data.state.length > 0 ? data.state[0] : undefined; + return ( { platformInstanceId={data.dataPlatformInstance?.instanceId} owners={null} globalTags={null} - // domain={data.domain?.domain} dataProduct={getDataProduct(genericProperties?.dataProduct)} - // deprecation={data.deprecation} insights={result.insights} externalUrl={data.properties?.externalUrl} degree={(result as any).degree} @@ -220,10 +190,9 @@ export class DataProcessInstanceEntity implements Entity { parentContainers={data.parentContainers} parentEntities={parentEntities} container={data.container || undefined} - // duration={data?.state?.[0]?.durationMillis} - // status={data?.state?.[0]?.result?.resultType} - // startTime={data?.state?.[0]?.timestampMillis} - // health={data.health} + duration={firstState?.durationMillis} + status={firstState?.result?.resultType} + startTime={firstState?.timestampMillis} /> ); }; @@ -237,7 +206,6 @@ export class DataProcessInstanceEntity implements Entity { icon: entity?.dataPlatformInstance?.platform?.properties?.logoUrl || undefined, platform: entity?.dataPlatformInstance?.platform, container: entity?.container, - // health: entity?.health || undefined, }; }; diff --git a/datahub-web-react/src/app/entity/dataProcessInstance/preview/Preview.tsx b/datahub-web-react/src/app/entity/dataProcessInstance/preview/Preview.tsx index 3a3b0340695d96..9a2acbe11c0845 100644 --- a/datahub-web-react/src/app/entity/dataProcessInstance/preview/Preview.tsx +++ b/datahub-web-react/src/app/entity/dataProcessInstance/preview/Preview.tsx @@ -39,10 +39,10 @@ export const Preview = ({ health, parentEntities, parentContainers, -}: // duration, -// status, -// startTime, -{ + duration, + status, + startTime, +}: { urn: string; name: string; subType?: string | null; @@ -64,9 +64,9 @@ export const Preview = ({ health?: Health[] | null; parentEntities?: Array | null; parentContainers?: ParentContainersResult | null; - // duration?: number | null; - // status?: string | null; - // startTime?: number | null; + duration?: number | null; + status?: string | null; + startTime?: number | null; }): JSX.Element => { const entityRegistry = useEntityRegistry(); return ( @@ -95,9 +95,9 @@ export const Preview = ({ paths={paths} health={health || undefined} parentEntities={parentEntities} - // duration={duration} - // status={status} - // startTime={startTime} + duration={duration} + status={status} + startTime={startTime} /> ); }; diff --git a/datahub-web-react/src/app/entity/dataProcessInstance/profile/DataProcessInstanceSummary.tsx b/datahub-web-react/src/app/entity/dataProcessInstance/profile/DataProcessInstanceSummary.tsx new file mode 100644 index 00000000000000..c6591d4f5faa1d --- /dev/null +++ b/datahub-web-react/src/app/entity/dataProcessInstance/profile/DataProcessInstanceSummary.tsx @@ -0,0 +1,102 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Space, Table, Typography } from 'antd'; +import { formatDetailedDuration } from '@src/app/shared/time/timeUtils'; +import { capitalize } from 'lodash'; +import moment from 'moment'; +import { MlHyperParam, MlMetric, DataProcessInstanceRunResultType } from '../../../../types.generated'; +import { useBaseEntity } from '../../shared/EntityContext'; +import { InfoItem } from '../../shared/components/styled/InfoItem'; +import { GetDataProcessInstanceQuery } from '../../../../graphql/dataProcessInstance.generated'; +import { Pill } from '../../../../alchemy-components/components/Pills'; + +const TabContent = styled.div` + padding: 16px; +`; + +const InfoItemContainer = styled.div<{ justifyContent }>` + display: flex; + position: relative; + justify-content: ${(props) => props.justifyContent}; + padding: 0px 2px; +`; + +const InfoItemContent = styled.div` + padding-top: 8px; + width: 100px; +`; + +const propertyTableColumns = [ + { + title: 'Name', + dataIndex: 'name', + width: 450, + }, + { + title: 'Value', + dataIndex: 'value', + }, +]; + +export default function MLModelSummary() { + const baseEntity = useBaseEntity(); + const dpi = baseEntity?.dataProcessInstance; + + const formatStatus = (state) => { + if (!state || state.length === 0) return '-'; + const result = state[0]?.result?.resultType; + const statusColor = result === DataProcessInstanceRunResultType.Success ? 'green' : 'red'; + return ; + }; + + const formatDuration = (state) => { + if (!state || state.length === 0) return '-'; + return formatDetailedDuration(state[0]?.durationMillis); + }; + + return ( + + + Details + + + + {dpi?.properties?.created?.time + ? moment(dpi.properties.created.time).format('YYYY-MM-DD HH:mm:ss') + : '-'} + + + + {formatStatus(dpi?.state)} + + + {formatDuration(dpi?.state)} + + + {dpi?.mlTrainingRunProperties?.id} + + + {dpi?.properties?.created?.actor} + + + + + {dpi?.mlTrainingRunProperties?.outputUrls} + + + Training Metrics + + Hyper Parameters +
+ + + ); +} diff --git a/datahub-web-react/src/app/entity/mlModel/MLModelEntity.tsx b/datahub-web-react/src/app/entity/mlModel/MLModelEntity.tsx index b77f6a19436a51..5e75b4680e427f 100644 --- a/datahub-web-react/src/app/entity/mlModel/MLModelEntity.tsx +++ b/datahub-web-react/src/app/entity/mlModel/MLModelEntity.tsx @@ -151,7 +151,7 @@ export class MLModelEntity implements Entity { }; displayName = (data: MlModel) => { - return data.name || data.urn; + return data.properties?.name || data.name || data.urn; }; getGenericEntityProperties = (mlModel: MlModel) => { diff --git a/datahub-web-react/src/app/entity/mlModel/preview/Preview.tsx b/datahub-web-react/src/app/entity/mlModel/preview/Preview.tsx index 4b57976dfe1a27..7ea33ba4c15f6f 100644 --- a/datahub-web-react/src/app/entity/mlModel/preview/Preview.tsx +++ b/datahub-web-react/src/app/entity/mlModel/preview/Preview.tsx @@ -21,7 +21,8 @@ export const Preview = ({ return ( ` + display: flex; + position: relative; + justify-content: ${(props) => props.justifyContent}; + padding: 0px 2px; +`; + +const InfoItemContent = styled.div` + padding-top: 8px; + width: 100px; + display: flex; + flex-wrap: wrap; + gap: 5px; +`; + +const JobLink = styled(Link)` + color: ${colors.blue[700]}; + &:hover { + text-decoration: underline; + } +`; + export default function MLModelSummary() { const baseEntity = useBaseEntity(); const model = baseEntity?.mlModel; + const entityRegistry = useEntityRegistry(); const propertyTableColumns = [ { @@ -26,9 +55,72 @@ export default function MLModelSummary() { }, ]; + const renderTrainingJobs = () => { + const trainingJobs = + model?.trainedBy?.relationships?.map((relationship) => relationship.entity).filter(notEmpty) || []; + + if (trainingJobs.length === 0) return '-'; + + return ( +
+ {trainingJobs.map((job, index) => { + const { urn, name } = job as { urn: string; name?: string }; + return ( + + + {name || urn} + + {index < trainingJobs.length - 1 && ', '} + + ); + })} +
+ ); + }; + return ( + Model Details + + + {model?.versionProperties?.version?.versionTag} + + + + {model?.properties?.created?.time + ? moment(model.properties.created.time).format('YYYY-MM-DD HH:mm:ss') + : '-'} + + + + + {model?.properties?.lastModified?.time + ? moment(model.properties.lastModified.time).format('YYYY-MM-DD HH:mm:ss') + : '-'} + + + + {model?.properties?.created?.actor} + + + + + + {model?.versionProperties?.aliases?.map((alias) => ( + + ))} + + + + {renderTrainingJobs()} + + Training Metrics
{ }; displayName = (data: MlModelGroup) => { - return data.name || data.urn; + return data.properties?.name || data.name || data.urn; }; getGenericEntityProperties = (mlModelGroup: MlModelGroup) => { diff --git a/datahub-web-react/src/app/entity/mlModelGroup/preview/Preview.tsx b/datahub-web-react/src/app/entity/mlModelGroup/preview/Preview.tsx index 910397af899f57..76ad9c06daece3 100644 --- a/datahub-web-react/src/app/entity/mlModelGroup/preview/Preview.tsx +++ b/datahub-web-react/src/app/entity/mlModelGroup/preview/Preview.tsx @@ -19,7 +19,8 @@ export const Preview = ({ return ( ` + display: flex; + position: relative; + justify-content: ${(props) => props.justifyContent}; + padding: 12px 2px 20px 2px; +`; + +const InfoItemContent = styled.div` + padding-top: 8px; + width: 100px; +`; + +const NameContainer = styled.div` + display: flex; + align-items: center; +`; + +const NameLink = styled.a` + font-weight: 700; + color: inherit; + font-size: 0.9rem; + &:hover { + color: ${colors.blue[400]} !important; + } +`; + +const TagContainer = styled.div` + display: inline-flex; + margin-left: 0px; + margin-top: 3px; + flex-wrap: wrap; + margin-right: 8px; + backgroundcolor: white; + gap: 5px; +`; + +const StyledTable = styled(Table)` + &&& .ant-table-cell { + padding: 16px; + } +` as typeof Table; + +const ModelsContainer = styled.div` + width: 100%; + padding: 20px; +`; + +const VersionContainer = styled.div` + display: flex; + align-items: center; +`; export default function MLGroupModels() { const baseEntity = useBaseEntity(); - const models = baseEntity?.mlModelGroup?.incoming?.relationships?.map((relationship) => relationship.entity) || []; - const entityRegistry = useEntityRegistry(); + const modelGroup = baseEntity?.mlModelGroup; + + const models = + baseEntity?.mlModelGroup?.incoming?.relationships + ?.map((relationship) => relationship.entity) + .filter(notEmpty) || []; + + const columns = [ + { + title: 'Name', + dataIndex: 'name', + key: 'name', + width: 300, + render: (_: any, record) => ( + + + {record?.properties?.propertiesName || record?.name} + + + ), + }, + { + title: 'Version', + key: 'version', + width: 70, + render: (_: any, record: any) => ( + {record.versionProperties?.version?.versionTag || '-'} + ), + }, + { + title: 'Created At', + key: 'createdAt', + width: 150, + render: (_: any, record: any) => ( + + {record.properties?.createdTS?.time + ? moment(record.properties.createdTS.time).format('YYYY-MM-DD HH:mm:ss') + : '-'} + + ), + }, + { + title: 'Aliases', + key: 'aliases', + width: 200, + render: (_: any, record: any) => { + const aliases = record.versionProperties?.aliases || []; + + return ( + + {aliases.map((alias) => ( + + ))} + + ); + }, + }, + { + title: 'Tags', + key: 'tags', + width: 200, + render: (_: any, record: any) => { + const tags = record.properties?.tags || []; + + return ( + + {tags.map((tag) => ( + + ))} + + ); + }, + }, + { + title: 'Description', + dataIndex: 'description', + key: 'description', + width: 300, + render: (_: any, record: any) => { + const editableDesc = record.editableProperties?.description; + const originalDesc = record.description; + + return {editableDesc || originalDesc || '-'}; + }, + }, + ]; return ( - <> - - Models} - renderItem={(item) => ( - - {entityRegistry.renderPreview(EntityType.Mlmodel, PreviewType.PREVIEW, item)} - - )} - /> - - + + Model Group Details + + + + {modelGroup?.properties?.created?.time + ? moment(modelGroup.properties.created.time).format('YYYY-MM-DD HH:mm:ss') + : '-'} + + + + + {modelGroup?.properties?.lastModified?.time + ? moment(modelGroup.properties.lastModified.time).format('YYYY-MM-DD HH:mm:ss') + : '-'} + + + {modelGroup?.properties?.created?.actor && ( + + {modelGroup.properties.created?.actor} + + )} + + Models + , + }} + /> + ); } diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchSection.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchSection.tsx index 9648aaf852bbe3..9da7b5d0ffb0c9 100644 --- a/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchSection.tsx +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchSection.tsx @@ -2,7 +2,7 @@ import React from 'react'; import * as QueryString from 'query-string'; import { useHistory, useLocation } from 'react-router'; import { ApolloError } from '@apollo/client'; -import { FacetFilterInput } from '../../../../../../types.generated'; +import { EntityType, FacetFilterInput } from '../../../../../../types.generated'; import useFilters from '../../../../../search/utils/useFilters'; import { navigateToEntitySearchUrl } from './navigateToEntitySearchUrl'; import { FilterSet, GetSearchResultsParams, SearchResultsInterface } from './types'; @@ -16,6 +16,30 @@ import { } from '../../../../../search/utils/types'; const FILTER = 'filter'; +const SEARCH_ENTITY_TYPES = [ + EntityType.Dataset, + EntityType.Dashboard, + EntityType.Chart, + EntityType.Mlmodel, + EntityType.MlmodelGroup, + EntityType.MlfeatureTable, + EntityType.Mlfeature, + EntityType.MlprimaryKey, + EntityType.DataFlow, + EntityType.DataJob, + EntityType.GlossaryTerm, + EntityType.GlossaryNode, + EntityType.Tag, + EntityType.Role, + EntityType.CorpUser, + EntityType.CorpGroup, + EntityType.Container, + EntityType.Domain, + EntityType.DataProduct, + EntityType.Notebook, + EntityType.BusinessAttribute, + EntityType.DataProcessInstance, +]; function getParamsWithoutFilters(params: QueryString.ParsedQuery) { const paramsCopy = { ...params }; @@ -137,6 +161,7 @@ export const EmbeddedListSearchSection = ({ return ( ; + duration: Maybe; + status: Maybe; +} + +export default function DataProcessInstanceRightColumn({ startTime, duration, status }: Props) { + const statusPillColor = status === DataProcessInstanceRunResultType.Success ? 'green' : 'red'; + + return ( + <> + {startTime && ( + {toLocalDateTimeString(startTime)}} + title={Start Time} + trigger="hover" + overlayInnerStyle={popoverStyles.overlayInnerStyle} + overlayStyle={popoverStyles.overlayStyle} + > + {toRelativeTimeString(startTime)} + + )} + {duration && ( + {formatDetailedDuration(duration)}} + title={Duration} + trigger="hover" + overlayInnerStyle={popoverStyles.overlayInnerStyle} + overlayStyle={popoverStyles.overlayStyle} + > + {formatDuration(duration)} + + )} + {status && ( + <> + + + + + )} + + ); +} diff --git a/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx b/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx index a19862e83ae510..42a32a5a1951ff 100644 --- a/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx +++ b/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx @@ -1,8 +1,8 @@ +import DataProcessInstanceRightColumn from '@app/preview/DataProcessInstanceRightColumn'; import React, { ReactNode, useState } from 'react'; import { Divider, Tooltip, Typography } from 'antd'; import { Link } from 'react-router-dom'; import styled from 'styled-components'; - import { GlobalTags, Owner, @@ -200,6 +200,9 @@ interface Props { paths?: EntityPath[]; health?: Health[]; parentDataset?: Dataset; + startTime?: number | null; + duration?: number | null; + status?: string | null; } export default function DefaultPreviewCard({ @@ -243,6 +246,9 @@ export default function DefaultPreviewCard({ paths, health, parentDataset, + startTime, + duration, + status, }: Props) { // sometimes these lists will be rendered inside an entity container (for example, in the case of impact analysis) // in those cases, we may want to enrich the preview w/ context about the container entity @@ -270,7 +276,8 @@ export default function DefaultPreviewCard({ event.stopPropagation(); }; - const shouldShowRightColumn = (topUsers && topUsers.length > 0) || (owners && owners.length > 0); + const shouldShowRightColumn = + (topUsers && topUsers.length > 0) || (owners && owners.length > 0) || startTime || duration || status; const uniqueOwners = getUniqueOwners(owners); return ( @@ -380,6 +387,7 @@ export default function DefaultPreviewCard({ {shouldShowRightColumn && ( + {topUsers && topUsers?.length > 0 && ( <> diff --git a/datahub-web-react/src/app/shared/time/timeUtils.tsx b/datahub-web-react/src/app/shared/time/timeUtils.tsx index 26d768a204be6f..4ff6ffedf65337 100644 --- a/datahub-web-react/src/app/shared/time/timeUtils.tsx +++ b/datahub-web-react/src/app/shared/time/timeUtils.tsx @@ -206,3 +206,41 @@ export function getTimeRangeDescription(startDate: moment.Moment | null, endDate return 'Unknown time range'; } + +export function formatDuration(durationMs: number): string { + const duration = moment.duration(durationMs); + const hours = Math.floor(duration.asHours()); + const minutes = duration.minutes(); + const seconds = duration.seconds(); + + if (hours === 0 && minutes === 0) { + return `${seconds} secs`; + } + + if (hours === 0) { + return minutes === 1 ? `${minutes} min` : `${minutes} mins`; + } + + const minuteStr = minutes > 0 ? ` ${minutes} mins` : ''; + return hours === 1 ? `${hours} hr${minuteStr}` : `${hours} hrs${minuteStr}`; +} + +export function formatDetailedDuration(durationMs: number): string { + const duration = moment.duration(durationMs); + const hours = Math.floor(duration.asHours()); + const minutes = duration.minutes(); + const seconds = duration.seconds(); + + const parts: string[] = []; + + if (hours > 0) { + parts.push(hours === 1 ? `${hours} hr` : `${hours} hrs`); + } + if (minutes > 0) { + parts.push(minutes === 1 ? `${minutes} min` : `${minutes} mins`); + } + if (seconds > 0) { + parts.push(`${seconds} secs`); + } + return parts.join(' '); +} diff --git a/datahub-web-react/src/graphql/fragments.graphql b/datahub-web-react/src/graphql/fragments.graphql index ecac2997489354..e94fc207fefd97 100644 --- a/datahub-web-react/src/graphql/fragments.graphql +++ b/datahub-web-react/src/graphql/fragments.graphql @@ -897,6 +897,10 @@ fragment nonRecursiveMLModel on MLModel { key value } + mlModelLineageInfo { + trainingJobs + downstreamJobs + } } globalTags { ...globalTagsFields @@ -971,6 +975,14 @@ fragment nonRecursiveMLModelGroupFields on MLModelGroup { time actor } + lastModified { + time + actor + } + mlModelLineageInfo { + trainingJobs + downstreamJobs + } } browsePathV2 { ...browsePathV2Fields diff --git a/datahub-web-react/src/graphql/lineage.graphql b/datahub-web-react/src/graphql/lineage.graphql index 457936ed62cd2e..f387c0c050668f 100644 --- a/datahub-web-react/src/graphql/lineage.graphql +++ b/datahub-web-react/src/graphql/lineage.graphql @@ -272,6 +272,7 @@ fragment lineageNodeProperties on EntityWithRelationships { removed } properties { + propertiesName: name createdTS: created { time actor @@ -296,6 +297,9 @@ fragment lineageNodeProperties on EntityWithRelationships { name description origin + tags { + ...globalTagsFields + } platform { ...platformFields } @@ -305,6 +309,34 @@ fragment lineageNodeProperties on EntityWithRelationships { status { removed } + versionProperties { + versionSet { + urn + type + } + version { + versionTag + } + aliases { + versionTag + } + comment + } + properties { + propertiesName: name + createdTS: created { + time + actor + } + tags + customProperties { + key + value + } + } + editableProperties { + description + } structuredProperties { properties { ...structuredPropertiesFields diff --git a/datahub-web-react/src/graphql/mlModel.graphql b/datahub-web-react/src/graphql/mlModel.graphql index ad97c7c6f530a1..ba10a243e6f9b3 100644 --- a/datahub-web-react/src/graphql/mlModel.graphql +++ b/datahub-web-react/src/graphql/mlModel.graphql @@ -20,6 +20,23 @@ query getMLModel($urn: String!) { } } } + trainedBy: relationships(input: { types: ["TrainedBy"], direction: OUTGOING, start: 0, count: 100 }) { + start + count + total + relationships { + type + direction + entity { + ... on DataProcessInstance { + urn + name + type + ...dataProcessInstanceFields + } + } + } + } privileges { ...entityPrivileges } diff --git a/datahub-web-react/src/graphql/search.graphql b/datahub-web-react/src/graphql/search.graphql index d12193b471d469..be72ff31a4f264 100644 --- a/datahub-web-react/src/graphql/search.graphql +++ b/datahub-web-react/src/graphql/search.graphql @@ -886,6 +886,9 @@ fragment searchResultsWithoutSchemaField on Entity { ...structuredPropertiesFields } } + properties { + propertiesName: name + } } ... on MLModelGroup { name @@ -908,6 +911,9 @@ fragment searchResultsWithoutSchemaField on Entity { ...structuredPropertiesFields } } + properties { + propertiesName: name + } } ... on Tag { name @@ -954,6 +960,9 @@ fragment searchResultsWithoutSchemaField on Entity { ...versionProperties } } + ... on DataProcessInstance { + ...dataProcessInstanceFields + } ... on DataPlatformInstance { ...dataPlatformInstanceFields }