Skip to content

Commit

Permalink
NEOS-1618: Fixes IP Address Form, FK Col Rendering, JM Table Perf (#2960
Browse files Browse the repository at this point in the history 
)
  • Loading branch information
@nickzelei
nickzelei authored Nov 20, 2024
1 parent f686279 commit 33376df
Show file tree
Hide file tree
Showing 33 changed files with 2,407 additions and 1,861 deletions.
92 changes: 88 additions & 4 deletions frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/DataGenConnectionCard.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import {
getAllFormErrors,
} from '@/components/jobs/SchemaTable/SchemaTable';
import { getSchemaConstraintHandler } from '@/components/jobs/SchemaTable/schema-constraint-handler';
import { TransformerResult } from '@/components/jobs/SchemaTable/transformer-handler';
import { getTransformerFilter } from '@/components/jobs/SchemaTable/util';
import { useAccount } from '@/components/providers/account-provider';
import { Alert, AlertTitle } from '@/components/ui/alert';
import { Button } from '@/components/ui/button';
Expand All @@ -25,8 +27,10 @@ import {
SelectTrigger,
SelectValue,
} from '@/components/ui/select';
import { getErrorMessage } from '@/util/util';
import { useGetTransformersHandler } from '@/libs/hooks/useGetTransformersHandler';
import { getErrorMessage, getTransformerFromField } from '@/util/util';
import {
JobMappingTransformerForm,
convertJobMappingTransformerFormToJobMappingTransformer,
convertJobMappingTransformerToForm,
} from '@/yup-validations/jobs';
Expand Down Expand Up @@ -64,8 +68,12 @@ import {
validateJobMapping,
} from '../../../util';
import SchemaPageSkeleton from './SchemaPageSkeleton';
import { useOnApplyDefaultClick } from './useOnApplyDefaultClick';
import { useOnImportMappings } from './useOnImportMappings';
import { getOnSelectedTableToggle } from './util';
import {
getFilteredTransformersForBulkSet,
getOnSelectedTableToggle,
} from './util';

interface Props {
jobId: string;
Expand Down Expand Up @@ -138,7 +146,7 @@ export default function DataGenConnectionCard({ jobId }: Props): ReactElement {
[isSchemaMapValidating, isTableConstraintsValidating]
);
const [selectedTables, setSelectedTables] = useState<Set<string>>(new Set());
const { append, remove, fields } =
const { append, remove, fields, update } =
useFieldArray<SingleTableEditSourceFormValues>({
control: form.control,
name: 'mappings',
Expand Down Expand Up @@ -220,6 +228,22 @@ export default function DataGenConnectionCard({ jobId }: Props): ReactElement {
const { mutateAsync: validateJobMappingsAsync } =
useMutation(validateJobMappings);

const { handler, isLoading: isGetTransformersLoading } =
useGetTransformersHandler(account?.id ?? '');

function onTransformerUpdate(
index: number,
transformer: JobMappingTransformerForm
): void {
const val = form.getValues(`mappings.${index}`);
update(index, {
schema: val.schema,
table: val.table,
column: val.column,
transformer,
});
}

const { onClick: onImportMappingsClick } = useOnImportMappings({
setMappings(mappings) {
form.setValue('mappings', mappings, {
Expand Down Expand Up @@ -247,7 +271,45 @@ export default function DataGenConnectionCard({ jobId }: Props): ReactElement {
setSelectedTables: setSelectedTables,
});

if (isJobLoading || isSchemaDataMapLoading) {
function getAvailableTransformers(idx: number): TransformerResult {
const row = fields[idx];
return handler.getFilteredTransformers(
getTransformerFilter(
schemaConstraintHandler,
{
schema: row.schema,
table: row.table,
column: row.column,
},
'sync'
)
);
}

const { onClick: onApplyDefaultClick } = useOnApplyDefaultClick({
getMappings() {
return form.getValues('mappings');
},
setTransformer: onTransformerUpdate,
constraintHandler: schemaConstraintHandler,
triggerUpdate() {
form.trigger('mappings');
},
});

function onTransformerBulkUpdate(
indices: number[],
config: JobMappingTransformerForm
): void {
indices.forEach((idx) => {
onTransformerUpdate(idx, config);
});
setTimeout(() => {
form.trigger('mappings');
}, 0);
}

if (isJobLoading || isSchemaDataMapLoading || isGetTransformersLoading) {
return <SchemaPageSkeleton />;
}

Expand Down Expand Up @@ -466,6 +528,28 @@ export default function DataGenConnectionCard({ jobId }: Props): ReactElement {
isJobMappingsValidating={isValidatingMappings}
onValidate={validateMappings}
onImportMappingsClick={onImportMappingsClick}
onTransformerUpdate={(idx, cfg) => {
onTransformerUpdate(idx, cfg);
}}
getAvailableTransformers={getAvailableTransformers}
getTransformerFromField={(idx) => {
const row = fields[idx];
return getTransformerFromField(handler, row.transformer);
}}
getAvailableTransformersForBulk={(rows) => {
return getFilteredTransformersForBulkSet(
rows,
handler,
schemaConstraintHandler,
'generate',
'relational'
);
}}
getTransformerFromFieldValue={(fvalue) => {
return getTransformerFromField(handler, fvalue);
}}
onApplyDefaultClick={onApplyDefaultClick}
onTransformerBulkUpdate={onTransformerBulkUpdate}
/>

{form.formState.errors.mappings && (
Expand Down
154 changes: 122 additions & 32 deletions ...tend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/DataSyncConnectionCard.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,11 @@ import {
getAllFormErrors,
} from '@/components/jobs/SchemaTable/SchemaTable';
import { getSchemaConstraintHandler } from '@/components/jobs/SchemaTable/schema-constraint-handler';
import { TransformerResult } from '@/components/jobs/SchemaTable/transformer-handler';
import {
getTransformerFilter,
splitCollection,
} from '@/components/jobs/SchemaTable/util';
import { useAccount } from '@/components/providers/account-provider';
import { Button } from '@/components/ui/button';
import {
Expand All @@ -25,12 +30,14 @@ import {
SelectTrigger,
SelectValue,
} from '@/components/ui/select';
import { useGetTransformersHandler } from '@/libs/hooks/useGetTransformersHandler';
import { splitConnections } from '@/libs/utils';
import { getErrorMessage } from '@/util/util';
import { getErrorMessage, getTransformerFromField } from '@/util/util';
import {
DataSyncSourceFormValues,
EditDestinationOptionsFormValues,
JobMappingFormValues,
JobMappingTransformerForm,
VirtualForeignConstraintFormValues,
convertJobMappingTransformerFormToJobMappingTransformer,
convertJobMappingTransformerToForm,
Expand Down Expand Up @@ -87,11 +94,13 @@ import {
validateJobMapping,
} from '../../../util';
import SchemaPageSkeleton from './SchemaPageSkeleton';
import { useOnApplyDefaultClick } from './useOnApplyDefaultClick';
import { useOnImportMappings } from './useOnImportMappings';
import {
getConnectionIdFromSource,
getDestinationDetailsRecord,
getDynamoDbDestinations,
getFilteredTransformersForBulkSet,
getOnSelectedTableToggle,
isDynamoDBConnection,
isNosqlSource,
Expand Down Expand Up @@ -466,6 +475,22 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement {
[]
);

const { handler, isLoading: isGetTransformersLoading } =
useGetTransformersHandler(account?.id ?? '');

function onTransformerUpdate(
index: number,
transformer: JobMappingTransformerForm
): void {
const val = form.getValues(`mappings.${index}`);
update(index, {
schema: val.schema,
table: val.table,
column: val.column,
transformer,
});
}

const { onClick: onImportMappingsClick } = useOnImportMappings({
setMappings(mappings) {
form.setValue('mappings', mappings, {
Expand All @@ -480,13 +505,7 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement {
appendNewMappings(mappings) {
append(mappings);
},
setTransformer(idx, transformer) {
form.setValue(`mappings.${idx}.transformer`, transformer, {
shouldDirty: true,
shouldTouch: true,
shouldValidate: false,
});
},
setTransformer: onTransformerUpdate,
async triggerUpdate() {
await form.trigger('mappings');
setTimeout(() => {
Expand All @@ -497,7 +516,23 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement {
setSelectedTables: setSelectedTables,
});

if (isConnectionsLoading || isSchemaDataMapLoading || isJobDataLoading) {
const { onClick: onApplyDefaultClick } = useOnApplyDefaultClick({
getMappings() {
return form.getValues('mappings');
},
setTransformer: onTransformerUpdate,
constraintHandler: schemaConstraintHandler,
triggerUpdate() {
form.trigger('mappings');
},
});

if (
isConnectionsLoading ||
isSchemaDataMapLoading ||
isJobDataLoading ||
isGetTransformersLoading
) {
return <SchemaPageSkeleton />;
}

Expand All @@ -518,6 +553,34 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement {
c.connectionConfig?.config.case !== 'gcpCloudstorageConfig'
)
);

function getAvailableTransformers(idx: number): TransformerResult {
const row = formMappings[idx];
return handler.getFilteredTransformers(
getTransformerFilter(
schemaConstraintHandler,
{
schema: row.schema,
table: row.table,
column: row.column,
},
'sync'
)
);
}

function onTransformerBulkUpdate(
indices: number[],
config: JobMappingTransformerForm
): void {
indices.forEach((idx) => {
onTransformerUpdate(idx, config);
});
setTimeout(() => {
form.trigger('mappings');
}, 0);
}

return (
<Form {...form}>
<form onSubmit={form.handleSubmit(onSubmit)}>
Expand Down Expand Up @@ -588,34 +651,20 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement {
validateMappingsResponse
)}
onValidate={validateMappings}
constraintHandler={schemaConstraintHandler}
onRemoveMappings={(values) => {
const valueSet = new Set(
values.map((v) => `${v.schema}.${v.table}.${v.column}`)
);
const toRemove: number[] = [];
formMappings.forEach((mapping, idx) => {
if (
valueSet.has(
`${mapping.schema}.${mapping.table}.${mapping.column}`
)
) {
toRemove.push(idx);
}
});
if (toRemove.length > 0) {
remove(toRemove);
onRemoveMappings={(indices) => {
const indexSet = new Set(indices);
const remainingTables = formMappings
.filter((_, idx) => !indexSet.has(idx))
.map((fm) => fm.table);

if (indices.length > 0) {
remove(indices);
}

if (!source || isDynamoDBConnection(source)) {
return;
}

const toRemoveSet = new Set(toRemove);
const remainingTables = formMappings
.filter((_, idx) => !toRemoveSet.has(idx))
.map((fm) => fm.table);

// Check and update destinationOptions if needed
const destOpts = form.getValues('destinationOptions');
const updatedDestOpts = destOpts
Expand Down Expand Up @@ -653,7 +702,7 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement {
onAddMappings={(values) => {
append(
values.map((v) => {
const [schema, table] = v.collection.split('.');
const [schema, table] = splitCollection(v.collection);
return {
schema,
table,
Expand Down Expand Up @@ -734,6 +783,25 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement {
dynamoDBDestinations.length > 0
)}
onImportMappingsClick={onImportMappingsClick}
getAvailableTransformers={getAvailableTransformers}
getTransformerFromField={(idx) => {
const row = formMappings[idx];
return getTransformerFromField(handler, row.transformer);
}}
getAvailableTransformersForBulk={(rows) => {
return getFilteredTransformersForBulkSet(
rows,
handler,
schemaConstraintHandler,
'sync',
'nosql'
);
}}
getTransformerFromFieldValue={(fvalue) => {
return getTransformerFromField(handler, fvalue);
}}
onApplyDefaultClick={onApplyDefaultClick}
onTransformerBulkUpdate={onTransformerBulkUpdate}
/>
)}

Expand All @@ -757,6 +825,28 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement {
addVirtualForeignKey={addVirtualForeignKey}
removeVirtualForeignKey={removeVirtualForeignKey}
onImportMappingsClick={onImportMappingsClick}
onTransformerUpdate={(idx, cfg) => {
onTransformerUpdate(idx, cfg);
}}
getAvailableTransformers={getAvailableTransformers}
getTransformerFromField={(idx) => {
const row = formMappings[idx];
return getTransformerFromField(handler, row.transformer);
}}
getAvailableTransformersForBulk={(rows) => {
return getFilteredTransformersForBulkSet(
rows,
handler,
schemaConstraintHandler,
'sync',
'relational'
);
}}
getTransformerFromFieldValue={(fvalue) => {
return getTransformerFromField(handler, fvalue);
}}
onApplyDefaultClick={onApplyDefaultClick}
onTransformerBulkUpdate={onTransformerBulkUpdate}
/>
)}
<div className="flex flex-row items-center justify-end w-full mt-4">
Expand Down
Loading

0 comments on commit 33376df

Please sign in to comment.