Skip to content

Commit

Permalink
console: Custom Images for connectors feature
Browse files Browse the repository at this point in the history
  • Loading branch information
absorbb committed Jan 16, 2025
1 parent 5d1d287 commit 2d52e2f
Show file tree
Hide file tree
Showing 9 changed files with 204 additions and 64 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ export type ConfigEditorProps<T extends { id: string } = { id: string }, M = {}>
//for providing custom editor component
editorComponent?: EditorComponentFactory;
testConnectionEnabled?: (o: any) => boolean;
testButtonLabel?: string;
onTest?: (o: T) => Promise<ConfigTestResult>;
backTo?: string;
};
Expand Down Expand Up @@ -326,6 +327,7 @@ const EditorComponent: React.FC<EditorComponentProps> = props => {
isNew={isNew}
isTouched={isTouched}
hasErrors={hasErrors}
testButtonLabel={props.testButtonLabel}
onTest={
onTest && testConnectionEnabled && testConnectionEnabled(formState?.formData || object)
? async () => {
Expand Down
10 changes: 7 additions & 3 deletions webapps/console/components/ConfigObjectEditor/EditorButtons.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ export type EditorButtonProps<T extends { id: string } = { id: string }> = {
isTouched?: boolean;
hasErrors?: boolean;
testStatus?: string;
testButtonLabel?: string;
};

export const EditorButtons: React.FC<EditorButtonProps> = ({
Expand All @@ -28,6 +29,7 @@ export const EditorButtons: React.FC<EditorButtonProps> = ({
onSave,
isTouched,
hasErrors,
testButtonLabel = "Test Connection",
}) => {
const buttonDivRef = useRef<HTMLDivElement>(null);
const appConfig = useAppConfig();
Expand Down Expand Up @@ -93,17 +95,19 @@ export const EditorButtons: React.FC<EditorButtonProps> = ({
(testStatus === "success" ? (
<Popover content={"Connection test passed"} color={"lime"} trigger={"hover"}>
<Button type="link" disabled={loading} size="large" onClick={doTest}>
<CheckOutlined /> Test Connection
<CheckOutlined />
{testButtonLabel}
</Button>
</Popover>
) : (
<Button type="link" disabled={loading} size="large" onClick={doTest}>
{testStatus === "pending" ? (
<>
<LoadingOutlined /> Test Connection
<LoadingOutlined />
{testButtonLabel}
</>
) : (
"Test Connection"
testButtonLabel
)}
</Button>
))}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import React, { PropsWithChildren, ReactNode, useEffect, useState } from "react";
import { branding } from "../../lib/branding";
import { HiSelector } from "react-icons/hi";
import { FaSignOutAlt, FaUserCircle } from "react-icons/fa";
import { FaDocker, FaSignOutAlt, FaUserCircle } from "react-icons/fa";
import { FiSettings } from "react-icons/fi";
import { Button, Drawer, Dropdown, Menu, MenuProps } from "antd";
import MenuItem from "antd/lib/menu/MenuItem";
Expand Down Expand Up @@ -582,6 +582,7 @@ function PageHeader() {
{ title: "Service Connections", path: "/services", icon: <ServerCog className="w-full h-full" /> },
{ title: "Syncs", path: "/syncs", icon: <Share2 className="w-full h-full" /> },
{ title: "All Logs", path: "/syncs/tasks", icon: <ScrollText className="w-full h-full" /> },
{ title: "Custom Images", path: "/custom-images", icon: <FaDocker className="w-full h-full" /> },
],
},
appConfig.ee?.available && {
Expand Down
79 changes: 26 additions & 53 deletions webapps/console/components/ServicesCatalog/ServicesCatalog.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,14 @@ import capitalize from "lodash/capitalize";
import { LoadingAnimation } from "../GlobalLoader/GlobalLoader";
import React from "react";
import { ErrorCard } from "../GlobalError/GlobalError";
import { Button, Input, Popover } from "antd";
import { Input } from "antd";
import { useAppConfig, useWorkspace } from "../../lib/context";
import { useConfigObjectList } from "../../lib/store";

function groupByType(sources: SourceType[]): Record<string, SourceType[]> {
const groups: Record<string, SourceType[]> = {};
const otherGroup = "other";
const sortOrder = ["Datawarehouse", "Product Analytics", "CRM", "Block Storage"];
const sortOrder = ["api", "database", "file", "custom image"];

sources.forEach(s => {
if (s.packageId.endsWith("strict-encrypt") || s.packageId === "airbyte/source-file-secure") {
Expand Down Expand Up @@ -58,11 +59,10 @@ export const ServicesCatalog: React.FC<{ onClick: (packageType, packageId: strin
onClick,
}) => {
const { data, isLoading, error } = useApi<{ sources: SourceType[] }>(`/api/sources?mode=meta`);
const customImages = useConfigObjectList("custom-image");
const sourcesIconsLoader = useApi<{ sources: SourceType[] }>(`/api/sources?mode=icons-only`);
const workspace = useWorkspace();
const [filter, setFilter] = React.useState("");
const [customImage, setCustomImage] = React.useState("");
const [customPopupOpen, setCustomPopupOpen] = React.useState(false);
const appconfig = useAppConfig();
const sourcesIcons: Record<string, string> = sourcesIconsLoader.data
? sourcesIconsLoader.data.sources.reduce(
Expand All @@ -79,7 +79,17 @@ export const ServicesCatalog: React.FC<{ onClick: (packageType, packageId: strin
} else if (error) {
return <ErrorCard error={error} />;
}
const groups = groupByType(data.sources);
const sources = [
...data.sources,
...customImages.map(c => ({
id: c.package,
packageId: c.package,
packageType: "airbyte",
meta: { name: c.name, connectorSubtype: "custom image", dockerImageTag: c.version },
})),
] as SourceType[];

const groups = groupByType(sources);
return (
<div className="p-6 flex flex-col flex-shrink w-full h-full overflow-y-auto">
<div key={"filter"} className={"m-4"}>
Expand Down Expand Up @@ -123,14 +133,23 @@ export const ServicesCatalog: React.FC<{ onClick: (packageType, packageId: strin
<div
key={source.id}
className={`flex items-center cursor-pointer relative w-72 border border-textDisabled ${"hover:scale-105 hover:border-primary"} transition ease-in-out rounded-lg px-4 py-4 space-x-4 m-4`}
onClick={() => onClick(source.packageType, source.packageId)}
onClick={() =>
onClick(
source.packageType,
source.packageId,
source.meta?.connectorSubtype === "custom image" ? source.meta?.dockerImageTag : undefined
)
}
>
<div className={`${styles.icon} flex`}>{getServiceIcon(source, sourcesIcons)}</div>
<div>
<div>
<div className={`text-xl`}>{source.meta.name}</div>
</div>
<div className="text-xs text-textLight">{source.packageId}</div>
<div className="text-xs text-textLight">
{source.packageId}
{source.meta?.connectorSubtype === "custom image" ? ":" + source.meta?.dockerImageTag : ""}
</div>
</div>
</div>
);
Expand All @@ -139,52 +158,6 @@ export const ServicesCatalog: React.FC<{ onClick: (packageType, packageId: strin
</div>
);
})}
<div key={"custom-connector"} className="">
<div className="text-3xl text-textLight px-4 pb-0 pt-3">Advanced</div>
<div className="flex flex-wrap">
<Popover
content={
<div className={"flex flex-row gap-1.5"}>
<Input onChange={e => setCustomImage(e.target.value)} />
<Button
type={"primary"}
onClick={() => {
const [packageId, packageVersion] = (customImage || "").trim().split(":");
if (!packageId) {
return;
}
onClick("airbyte", packageId, packageVersion);
setCustomPopupOpen(false);
setCustomImage("");
}}
>
Add
</Button>
</div>
}
onOpenChange={setCustomPopupOpen}
open={customPopupOpen}
title="Enter docker image name"
placement={"right"}
trigger="click"
>
<div
key="custom-connector"
className={`flex items-center cursor-pointer relative w-72 border border-textDisabled ${"hover:scale-105 hover:border-primary"} transition ease-in-out rounded-lg px-4 py-4 space-x-4 m-4`}
>
<div className={`${styles.icon} flex`}>
<FaDocker />
</div>
<div>
<div>
<div className={`text-xl`}>Custom connector</div>
</div>
<div className="text-xs text-textLight">Custom docker image</div>
</div>
</div>
</Popover>
</div>
</div>
</div>
</div>
);
Expand Down
13 changes: 12 additions & 1 deletion webapps/console/lib/schema/config-objects.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,15 @@
import { coreDestinationsMap } from "./destinations";
import { safeParseWithDate } from "../zod";
import { ApiError } from "../shared/errors";
import { ApiKey, ConfigObjectType, DestinationConfig, FunctionConfig, ServiceConfig, StreamConfig } from "./index";
import {
ApiKey,
ConfigObjectType,
ConnectorImageConfig,
DestinationConfig,
FunctionConfig,
ServiceConfig,
StreamConfig,
} from "./index";
import { assertDefined, createHash, requireDefined } from "juava";
import { checkOrAddToIngress, isDomainAvailable } from "../server/custom-domains";
import { ZodType, ZodTypeDef } from "zod";
Expand Down Expand Up @@ -162,4 +170,7 @@ const configObjectTypes: Record<string, ConfigObjectType> = {
service: {
schema: ServiceConfig,
},
"custom-image": {
schema: ConnectorImageConfig,
},
} as const;
9 changes: 9 additions & 0 deletions webapps/console/lib/schema/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,15 @@ export const ServiceConfig = ConfigEntityBase.merge(
);
export type ServiceConfig = z.infer<typeof ServiceConfig>;

export const ConnectorImageConfig = ConfigEntityBase.merge(
z.object({
name: z.string(),
package: z.string(),
version: z.string(),
})
);
export type ConnectorImageConfig = z.infer<typeof ConnectorImageConfig>;

/**
* What happens to an object before it is saved to DB.
*
Expand Down
5 changes: 3 additions & 2 deletions webapps/console/lib/store/index.tsx
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { DestinationConfig, FunctionConfig, ServiceConfig, StreamConfig } from "../schema";
import type { ConnectorImageConfig, DestinationConfig, FunctionConfig, ServiceConfig, StreamConfig } from "../schema";
import { useCallback, useEffect, useMemo, useState } from "react";
import { getLog, requireDefined, rpc } from "juava";
import { useWorkspace } from "../context";
Expand All @@ -7,7 +7,7 @@ import { z } from "zod";
import { ConfigurationObjectLinkDbModel, ProfileBuilderDbModel, WorkspaceDbModel } from "../../prisma/schema";
import { UseMutationResult } from "@tanstack/react-query/src/types";

export const allConfigTypes = ["stream", "service", "function", "destination"] as const;
export const allConfigTypes = ["stream", "service", "function", "destination", "custom-image"] as const;

export type ConfigType = (typeof allConfigTypes)[number];

Expand All @@ -16,6 +16,7 @@ export type ConfigTypes = {
service: ServiceConfig;
function: FunctionConfig;
destination: DestinationConfig;
"custom-image": ConnectorImageConfig;
};

export function asConfigType(type: string): ConfigType {
Expand Down
133 changes: 133 additions & 0 deletions webapps/console/pages/[workspaceId]/custom-images.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
import { WorkspacePageLayout } from "../../components/PageLayout/WorkspacePageLayout";
import { ConfigEditor, ConfigEditorProps } from "../../components/ConfigObjectEditor/ConfigEditor";
import { ConnectorImageConfig } from "../../lib/schema";
import { useAppConfig, useWorkspace } from "../../lib/context";
import React from "react";
import { SourceType } from "../api/sources";
import { ErrorCard } from "../../components/GlobalError/GlobalError";
import { ServerCog } from "lucide-react";
import { FaDocker } from "react-icons/fa";
import { Htmlizer } from "../../components/Htmlizer/Htmlizer";
import { rpc } from "juava";
import { UpgradeDialog } from "../../components/Billing/UpgradeDialog";
import { useBilling } from "../../components/Billing/BillingProvider";
import { LoadingAnimation } from "../../components/GlobalLoader/GlobalLoader";

const CustomImages: React.FC<any> = () => {
return (
<WorkspacePageLayout>
<CustomImagesList />
</WorkspacePageLayout>
);
};

const CustomImagesList: React.FC<{}> = () => {
const workspace = useWorkspace();
const appconfig = useAppConfig();
const billing = useBilling();

if (billing.loading) {
return <LoadingAnimation />;
}
if (billing.enabled && billing.settings?.planId === "free") {
return <UpgradeDialog featureDescription={"Custom Images"} />;
}

if (!(appconfig.syncs.enabled || workspace.featuresEnabled.includes("syncs"))) {
return (
<ErrorCard
title={"Feature is not enabled"}
error={{ message: "'Sources Sync' feature is not enabled for current project." }}
hideActions={true}
/>
);
}

const config: ConfigEditorProps<ConnectorImageConfig, SourceType> = {
listColumns: [
{
title: "Package",
render: (s: ConnectorImageConfig) => <span className={"font-semibold"}>{`${s.package}:${s.version}`}</span>,
},
],
objectType: ConnectorImageConfig,
fields: {
type: { constant: "custom-image" },
workspaceId: { constant: workspace.id },
package: {
documentation: (
<Htmlizer>
{
"Docker image name. Images can also include a registry hostname, e.g.: <code>fictional.registry.example/imagename</code>, and possibly a port number as well."
}
</Htmlizer>
),
},
version: {
documentation: "Docker image tag",
},
},
noun: "custom image",
type: "custom-image",
explanation: "Custom connector images that can be used to setup Service connector",
icon: () => <FaDocker className="w-full h-full" />,
testConnectionEnabled: () => true,
testButtonLabel: "Check Image",
onTest: async obj => {
try {
const firstRes = await rpc(
`/api/${workspace.id}/sources/spec?package=${obj.package}&version=${obj.version}&force=true`
);
if (firstRes.ok) {
return { ok: true };
} else if (firstRes.error) {
return { ok: false, error: `Cannot load specs for ${obj.package}:${obj.version}: ${firstRes.error}` };
} else {
for (let i = 0; i < 60; i++) {
await new Promise(resolve => setTimeout(resolve, 2000));
const resp = await rpc(`/api/${workspace.id}/sources/spec?package=${obj.package}&version=${obj.version}`);
if (!resp.pending) {
if (resp.error) {
return { ok: false, error: `Cannot load specs for ${obj.package}:${obj.version}: ${resp.error}` };
} else {
return { ok: true };
}
}
}
return { ok: false, error: `Cannot load specs for ${obj.package}:${obj.version}: Timeout` };
}
} catch (error: any) {
return { ok: false, error: `Cannot load specs for ${obj.package}:${obj.version}: ${error.message}` };
}
},
editorTitle: (_: ConnectorImageConfig, isNew: boolean) => {
const verb = isNew ? "New" : "Edit";
return (
<div className="flex items-center">
<div className="h-12 w-12 mr-4">
<FaDocker className="w-full h-full" />
</div>
{verb} custom image
</div>
);
},
actions: [
{
icon: <ServerCog className="w-full h-full" />,
title: "Setup Connector",
collapsed: false,
link: s =>
`/services?id=new&packageType=airbyte&packageId=${encodeURIComponent(s.package)}&version=${encodeURIComponent(
s.version
)}`,
},
],
};
return (
<>
<ConfigEditor {...(config as any)} />
</>
);
};

export default CustomImages;
Loading

0 comments on commit 2d52e2f

Please sign in to comment.