diff --git a/app/cdap/components/SourceControlManagement/SyncStatusFilters.tsx b/app/cdap/components/SourceControlManagement/SyncStatusFilters.tsx
new file mode 100644
index 00000000000..fe897d7a24a
--- /dev/null
+++ b/app/cdap/components/SourceControlManagement/SyncStatusFilters.tsx
@@ -0,0 +1,75 @@
+/*
+ * Copyright © 2023 Cask Data, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+import React from 'react';
+import styled from 'styled-components';
+import T from 'i18n-react';
+import { TSyncStatusFilter } from './types';
+import { Button, ButtonGroup } from '@material-ui/core';
+
+const PREFIX = 'features.SourceControlManagement.table';
+
+const StyledDiv = styled.div`
+ margin: 12px 0px;
+ display: flex;
+ justify-content: flex-end;
+ align-items: center;
+ gap: 12px;
+`;
+
+interface ISyncStatusFiltersProps {
+ syncStatusFilter: TSyncStatusFilter;
+ setSyncStatusFilter: (syncStatusFilter: TSyncStatusFilter) => void;
+}
+
+export const SyncStatusFilters = ({
+ syncStatusFilter,
+ setSyncStatusFilter,
+}: ISyncStatusFiltersProps) => {
+ const setFilter = (filterVal: TSyncStatusFilter) => () => setSyncStatusFilter(filterVal);
+
+ return (
+
+ {T.translate(`${PREFIX}.filtersLabel`)}:
+
+
+
+
+
+
+ );
+};
diff --git a/app/cdap/components/SourceControlManagement/SyncTabs.tsx b/app/cdap/components/SourceControlManagement/SyncTabs.tsx
new file mode 100644
index 00000000000..82c3afd0fe9
--- /dev/null
+++ b/app/cdap/components/SourceControlManagement/SyncTabs.tsx
@@ -0,0 +1,96 @@
+/*
+ * Copyright © 2023 Cask Data, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+import { Tab, Tabs } from '@material-ui/core';
+import React, { useEffect, useState } from 'react';
+import { useSelector } from 'react-redux';
+import { LocalPipelineListView } from './LocalPipelineListView';
+import styled from 'styled-components';
+import T from 'i18n-react';
+import { RemotePipelineListView } from './RemotePipelineListView';
+import { FeatureProvider } from 'services/react/providers/featureFlagProvider';
+import {
+ getNamespacePipelineList,
+ getRemotePipelineList,
+ setSyncStatusOfLocalPipelines,
+ setSyncStatusOfRemotePipelines,
+} from './store/ActionCreator';
+import { getCurrentNamespace } from 'services/NamespaceStore';
+
+const PREFIX = 'features.SourceControlManagement';
+
+const StyledDiv = styled.div`
+ padding: 10px;
+ margin-top: 10px;
+`;
+
+const ScmSyncTabs = () => {
+ const [tabIndex, setTabIndex] = useState(0);
+
+ const { ready: pushStateReady, nameFilter } = useSelector(({ push }) => push);
+ useEffect(() => {
+ if (!pushStateReady) {
+ getNamespacePipelineList(getCurrentNamespace(), nameFilter);
+ }
+ }, [pushStateReady]);
+
+ const { ready: pullStateReady } = useSelector(({ pull }) => pull);
+ useEffect(() => {
+ if (!pullStateReady) {
+ getRemotePipelineList(getCurrentNamespace());
+ }
+ }, [pullStateReady]);
+
+ useEffect(() => {
+ if (pushStateReady && pullStateReady) {
+ setSyncStatusOfLocalPipelines();
+ setSyncStatusOfRemotePipelines();
+ }
+ }, [pushStateReady, pullStateReady]);
+
+ const handleTabChange = (e, newValue) => {
+ setTabIndex(newValue);
+ // refetch latest pipeline data, while displaying possibly stale data
+ if (newValue === 0) {
+ getNamespacePipelineList(getCurrentNamespace(), nameFilter);
+ } else {
+ getRemotePipelineList(getCurrentNamespace());
+ }
+ };
+
+ return (
+ <>
+
+
+
+
+
+
+
+
+ {tabIndex === 0 ? : }
+
+
+
+ );
+};
+
+export default ScmSyncTabs;
diff --git a/app/cdap/components/SourceControlManagement/helpers.ts b/app/cdap/components/SourceControlManagement/helpers.ts
index c8657bb58c2..1e1e1003bbf 100644
--- a/app/cdap/components/SourceControlManagement/helpers.ts
+++ b/app/cdap/components/SourceControlManagement/helpers.ts
@@ -17,7 +17,17 @@
import moment from 'moment';
import { OperationStatus } from './OperationStatus';
import { OperationType } from './OperationType';
-import { IResource, IOperationResource, IOperationRun, ITimeInstant } from './types';
+import {
+ IResource,
+ IOperationResource,
+ IOperationRun,
+ ITimeInstant,
+ IOperationError,
+ IOperationResourceScopedErrorMessage,
+ IRepositoryPipeline,
+ TSyncStatusFilter,
+ TSyncStatus,
+} from './types';
import T from 'i18n-react';
const PREFIX = 'features.SourceControlManagement';
@@ -34,6 +44,20 @@ export const parseOperationResource = (resource: IOperationResource): IResource
};
};
+export const parseErrorMessage = (errorMessage: string): IOperationResourceScopedErrorMessage => {
+ const firstColonIndex = errorMessage.indexOf(':');
+ if (firstColonIndex === -1) {
+ return {
+ message: errorMessage,
+ };
+ }
+
+ return {
+ type: errorMessage.substring(0, firstColonIndex).trim(),
+ message: errorMessage.substring(firstColonIndex + 1).trim(),
+ };
+};
+
export const getOperationRunMessage = (operation: IOperationRun) => {
const n = operation.metadata?.resources?.length || '';
@@ -123,3 +147,62 @@ export const compareTimeInstant = (t1: ITimeInstant, t2: ITimeInstant): number =
export const getOperationStartTime = (operation: IOperationRun): string => {
return moment(operation.metadata?.createTime.seconds * 1000).format('DD-MM-YYYY HH:mm:ss A');
};
+
+export const getOperationRunTime = (operation: IOperationRun): string => {
+ if (operation.metadata?.createTime && operation.metadata?.endTime) {
+ return moment
+ .duration(
+ (operation.metadata?.endTime.seconds - operation.metadata?.createTime.seconds) * 1000
+ )
+ .humanize();
+ }
+ return null;
+};
+
+const getSyncStatusWeight = (syncStatus?: TSyncStatus): number => {
+ if (syncStatus === undefined) {
+ return 0;
+ }
+ if (syncStatus === 'not_available') {
+ return 0;
+ }
+ if (syncStatus === 'not_connected') {
+ return 0;
+ }
+ if (syncStatus === 'out_of_sync') {
+ return 1;
+ }
+ return 2;
+};
+
+export const compareSyncStatus = (a: IRepositoryPipeline, b: IRepositoryPipeline): number => {
+ return getSyncStatusWeight(a.syncStatus) - getSyncStatusWeight(b.syncStatus);
+};
+
+export const stableSort = (array, comparator) => {
+ const stabilizedArray = array.map((el, index) => [el, index]);
+ stabilizedArray.sort((a, b) => {
+ const order = comparator(a[0], b[0]);
+ if (order !== 0) {
+ return order;
+ }
+ return a[1] - b[1];
+ });
+ return stabilizedArray.map((el) => el[0]);
+};
+
+export const filterOnSyncStatus = (
+ array: IRepositoryPipeline[],
+ syncStatusFilter: TSyncStatusFilter
+): IRepositoryPipeline[] => {
+ if (syncStatusFilter === 'all') {
+ return array;
+ }
+
+ return array.filter((pipeline) => {
+ if (syncStatusFilter === 'out_of_sync') {
+ return ['not_connected', 'out_of_sync', 'not_available'].includes(pipeline.syncStatus);
+ }
+ return pipeline.syncStatus === syncStatusFilter;
+ });
+};
diff --git a/app/cdap/components/SourceControlManagement/index.tsx b/app/cdap/components/SourceControlManagement/index.tsx
index af6dad7d893..0b7191f1b78 100644
--- a/app/cdap/components/SourceControlManagement/index.tsx
+++ b/app/cdap/components/SourceControlManagement/index.tsx
@@ -14,31 +14,23 @@
* the License.
*/
-import { Tab, Tabs } from '@material-ui/core';
import { EntityTopPanel } from 'components/EntityTopPanel';
-import React, { useState } from 'react';
-import { LocalPipelineListView } from './LocalPipelineListView';
+import React from 'react';
import { Provider } from 'react-redux';
import SourceControlManagementSyncStore from './store';
-import styled from 'styled-components';
import T from 'i18n-react';
-import { RemotePipelineListView } from './RemotePipelineListView';
import { getCurrentNamespace } from 'services/NamespaceStore';
-import { FeatureProvider } from 'services/react/providers/featureFlagProvider';
+import { resetRemote, reset } from './store/ActionCreator';
+import ScmSyncTabs from './SyncTabs';
+import { useOnUnmount } from 'services/react/customHooks/useOnUnmount';
const PREFIX = 'features.SourceControlManagement';
-const StyledDiv = styled.div`
- padding: 10px;
- margin-top: 10px;
-`;
-
const SourceControlManagementSyncView = () => {
- const [tabIndex, setTabIndex] = useState(0);
-
- const handleTabChange = (e, newValue) => {
- setTabIndex(newValue);
- };
+ useOnUnmount(() => {
+ resetRemote();
+ reset();
+ });
const closeAndBackLink = `/ns/${getCurrentNamespace()}/details/scm`;
@@ -54,22 +46,7 @@ const SourceControlManagementSyncView = () => {
window.location.href = closeAndBackLink;
}}
/>
-
-
-
-
-
-
-
-
- {tabIndex === 0 ? : }
-
-
+
);
};
diff --git a/app/cdap/components/SourceControlManagement/store/ActionCreator.ts b/app/cdap/components/SourceControlManagement/store/ActionCreator.ts
index 999ae6828ee..c65d00a3493 100644
--- a/app/cdap/components/SourceControlManagement/store/ActionCreator.ts
+++ b/app/cdap/components/SourceControlManagement/store/ActionCreator.ts
@@ -27,7 +27,14 @@ import SourceControlManagementSyncStore, {
} from '.';
import { SourceControlApi } from 'api/sourcecontrol';
import { LongRunningOperationApi } from 'api/longRunningOperation';
-import { IPipeline, IPushResponse, IRepositoryPipeline, IOperationRun } from '../types';
+import {
+ IPipeline,
+ IPushResponse,
+ IRepositoryPipeline,
+ IOperationRun,
+ TSyncStatusFilter,
+ TSyncStatus,
+} from '../types';
import { SUPPORT } from 'components/StatusButton/constants';
import { compareTimeInstant } from '../helpers';
@@ -35,6 +42,7 @@ const PREFIX = 'features.SourceControlManagement';
// push actions
export const getNamespacePipelineList = (namespace, nameFilter = null) => {
+ const shouldCalculateSyncStatus = SourceControlManagementSyncStore.getState().pull.ready;
MyPipelineApi.list({
namespace,
artifactName: BATCH_PIPELINE_TYPE,
@@ -43,12 +51,17 @@ export const getNamespacePipelineList = (namespace, nameFilter = null) => {
(res: IPipeline[] | { applications: IPipeline[] }) => {
const pipelines = Array.isArray(res) ? res : res?.applications;
const nsPipelines = pipelines.map((pipeline) => {
- return {
+ const localPipeline: IRepositoryPipeline = {
name: pipeline.name,
fileHash: pipeline.sourceControlMeta?.fileHash,
error: null,
status: null,
+ syncStatus: 'not_available',
};
+ if (shouldCalculateSyncStatus) {
+ localPipeline.syncStatus = getSyncStatus(localPipeline);
+ }
+ return localPipeline;
});
setLocalPipelines(nsPipelines);
},
@@ -58,6 +71,50 @@ export const getNamespacePipelineList = (namespace, nameFilter = null) => {
);
};
+const getSyncStatus = (pipeline: IRepositoryPipeline, withLocal?: boolean): TSyncStatus => {
+ const listOfPipelines = withLocal
+ ? SourceControlManagementSyncStore.getState().push.localPipelines
+ : SourceControlManagementSyncStore.getState().pull.remotePipelines;
+
+ const pipelineInList = listOfPipelines.find((p) => p.name === pipeline.name);
+ if (!pipeline.fileHash || !pipelineInList) {
+ return 'not_connected';
+ }
+ return pipeline.fileHash === pipelineInList.fileHash ? 'in_sync' : 'out_of_sync';
+};
+
+export const setSyncStatusOfLocalPipelines = () => {
+ const localPipelines = SourceControlManagementSyncStore.getState().push.localPipelines;
+ const remotePipelines = SourceControlManagementSyncStore.getState().pull.remotePipelines;
+ if (!remotePipelines.length) {
+ return;
+ }
+
+ setLocalPipelines(
+ localPipelines.map((pipeline) => {
+ const p = { ...pipeline };
+ p.syncStatus = getSyncStatus(pipeline);
+ return p;
+ })
+ );
+};
+
+export const setSyncStatusOfRemotePipelines = () => {
+ const localPipelines = SourceControlManagementSyncStore.getState().push.localPipelines;
+ const remotePipelines = SourceControlManagementSyncStore.getState().pull.remotePipelines;
+ if (!localPipelines.length) {
+ return;
+ }
+
+ setRemotePipelines(
+ remotePipelines.map((pipeline) => {
+ const p = { ...pipeline };
+ p.syncStatus = getSyncStatus(pipeline, true);
+ return p;
+ })
+ );
+};
+
export const setLocalPipelines = (pipelines: IRepositoryPipeline[]) => {
SourceControlManagementSyncStore.dispatch({
type: PushToGitActions.setLocalPipelines,
@@ -84,6 +141,15 @@ export const setNameFilter = (nameFilter: string) => {
debouncedApplySearch();
};
+export const setSyncStatusFilter = (syncStatusFilter: TSyncStatusFilter) => {
+ SourceControlManagementSyncStore.dispatch({
+ type: PushToGitActions.setSyncStatusFilter,
+ payload: {
+ syncStatusFilter,
+ },
+ });
+};
+
export const setSelectedPipelines = (selectedPipelines: any[]) => {
SourceControlManagementSyncStore.dispatch({
type: PushToGitActions.setSelectedPipelines,
@@ -197,18 +263,24 @@ export const reset = () => {
// pull actions
export const getRemotePipelineList = (namespace) => {
+ const shouldCalculateSyncStatus = SourceControlManagementSyncStore.getState().push.ready;
SourceControlApi.list({
namespace,
}).subscribe(
(res: IRepositoryPipeline[] | { apps: IRepositoryPipeline[] }) => {
const pipelines = Array.isArray(res) ? res : res?.apps;
const remotePipelines = pipelines.map((pipeline) => {
- return {
+ const remotePipeline: IRepositoryPipeline = {
name: pipeline.name,
fileHash: pipeline.fileHash,
error: null,
status: null,
+ syncStatus: 'not_available',
};
+ if (shouldCalculateSyncStatus) {
+ remotePipeline.syncStatus = getSyncStatus(remotePipeline, true);
+ }
+ return remotePipeline;
});
setRemotePipelines(remotePipelines);
},
@@ -246,6 +318,15 @@ export const setRemoteNameFilter = (nameFilter: string) => {
});
};
+export const setRemoteSyncStatusFilter = (syncStatusFilter: TSyncStatusFilter) => {
+ SourceControlManagementSyncStore.dispatch({
+ type: PullFromGitActions.setSyncStatusFilter,
+ payload: {
+ syncStatusFilter,
+ },
+ });
+};
+
export const setSelectedRemotePipelines = (selectedPipelines: string[]) => {
SourceControlManagementSyncStore.dispatch({
type: PullFromGitActions.setSelectedPipelines,
diff --git a/app/cdap/components/SourceControlManagement/store/index.ts b/app/cdap/components/SourceControlManagement/store/index.ts
index 9591d223ac5..34ba8558e68 100644
--- a/app/cdap/components/SourceControlManagement/store/index.ts
+++ b/app/cdap/components/SourceControlManagement/store/index.ts
@@ -17,13 +17,13 @@
import { combineReducers, createStore, Store as StoreInterface } from 'redux';
import { composeEnhancers } from 'services/helpers';
import { IAction } from 'services/redux-helpers';
-import { IOperationRun, IRepositoryPipeline } from '../types';
-import { act } from 'react-dom/test-utils';
+import { IOperationRun, IRepositoryPipeline, TSyncStatusFilter } from '../types';
interface IPushViewState {
ready: boolean;
localPipelines: IRepositoryPipeline[];
nameFilter: string;
+ syncStatusFilter: TSyncStatusFilter;
selectedPipelines: string[];
commitModalOpen: boolean;
loadingMessage: string;
@@ -34,6 +34,7 @@ interface IPullViewState {
ready: boolean;
remotePipelines: IRepositoryPipeline[];
nameFilter: string;
+ syncStatusFilter: TSyncStatusFilter;
selectedPipelines: string[];
loadingMessage: string;
showFailedOnly: boolean;
@@ -55,6 +56,7 @@ export const PushToGitActions = {
setLocalPipelines: 'LOCAL_PIPELINES_SET',
reset: 'LOCAL_PIPELINES_RESET',
setNameFilter: 'LOCAL_PIPELINES_SET_NAME_FILTER',
+ setSyncStatusFilter: 'LOCAL_PIPELINES_SET_SYNC_STATUS_FILTER',
applySearch: 'LOCAL_PIPELINES_APPLY_SERACH',
setSelectedPipelines: 'LOCAL_PIPELINES_SET_SELECTED_PIPELINES',
toggleCommitModal: 'LOCAL_PIPELINES_TOGGLE_COMMIT_MODAL',
@@ -66,6 +68,7 @@ export const PullFromGitActions = {
setRemotePipelines: 'REMOTE_PIPELINES_SET',
reset: 'REMOTE_PIPELINES_RESET',
setNameFilter: 'REMOTE_PIPELINES_SET_NAME_FILTER',
+ setSyncStatusFilter: 'REMOTE_PIPELINES_SET_SYNC_STATUS_FILTER',
applySearch: 'REMOTE_PIPELINES_APPLY_SERACH',
setSelectedPipelines: 'REMOTE_PIPELINES_SET_SELECTED_PIPELINES',
setLoadingMessage: 'REMOTE_PIPELINES_SET_LOADING_MESSAGE',
@@ -82,6 +85,7 @@ const defaultPushViewState: IPushViewState = {
ready: false,
localPipelines: [],
nameFilter: '',
+ syncStatusFilter: 'all',
selectedPipelines: [],
commitModalOpen: false,
loadingMessage: null,
@@ -92,6 +96,7 @@ const defaultPullViewState: IPullViewState = {
ready: false,
remotePipelines: [],
nameFilter: '',
+ syncStatusFilter: 'all',
selectedPipelines: [],
loadingMessage: null,
showFailedOnly: false,
@@ -115,6 +120,11 @@ const push = (state = defaultPushViewState, action: IAction) => {
...state,
nameFilter: action.payload.nameFilter,
};
+ case PushToGitActions.setSyncStatusFilter:
+ return {
+ ...state,
+ syncStatusFilter: action.payload.syncStatusFilter,
+ };
case PushToGitActions.applySearch:
return {
...defaultPushViewState,
@@ -161,6 +171,11 @@ const pull = (state = defaultPullViewState, action: IAction) => {
...state,
nameFilter: action.payload.nameFilter,
};
+ case PullFromGitActions.setSyncStatusFilter:
+ return {
+ ...state,
+ syncStatusFilter: action.payload.syncStatusFilter,
+ };
case PullFromGitActions.setPullViewErrorMsg:
return {
...state,
diff --git a/app/cdap/components/SourceControlManagement/types.ts b/app/cdap/components/SourceControlManagement/types.ts
index 952e80bdcbe..9201ffebecd 100644
--- a/app/cdap/components/SourceControlManagement/types.ts
+++ b/app/cdap/components/SourceControlManagement/types.ts
@@ -19,11 +19,15 @@ import { SUPPORT } from 'components/StatusButton/constants';
import { OperationType } from './OperationType';
import { OperationStatus } from './OperationStatus';
+export type TSyncStatusFilter = 'all' | 'in_sync' | 'out_of_sync';
+export type TSyncStatus = 'not_available' | 'in_sync' | 'out_of_sync' | 'not_connected';
+
export interface IRepositoryPipeline {
name: string;
fileHash: string;
error: string;
status: SUPPORT;
+ syncStatus?: TSyncStatus;
}
export interface IOperationResource {
@@ -51,6 +55,11 @@ export interface IOperationResourceScopedError {
message?: string;
}
+export interface IOperationResourceScopedErrorMessage {
+ type?: string;
+ message: string;
+}
+
export interface IOperationRun {
id: string;
type: OperationType;
diff --git a/app/cdap/text/text-en.yaml b/app/cdap/text/text-en.yaml
index d2db1f1b5a9..e8a6b2fd963 100644
--- a/app/cdap/text/text-en.yaml
+++ b/app/cdap/text/text-en.yaml
@@ -3236,6 +3236,10 @@ features:
pipelineSyncMessage: Syncing 1 pipeline with the remote repository
pipelineSyncedSuccess: Successfully synced 1 pipeline with the remote repository
pipelineSyncedFail: Failed to sync 1 pipeline with the remote repository
+ stopOperation: STOP
+ operationStartedAt: Started at
+ operationRanFor: Completed in
+ operationRunningFor: Running for
pull:
emptyPipelineListMessage: There are no pipelines in the remote repository or no pipelines matching the search query "{query}"
modalTitle: Pull pipeline from remote repository
@@ -3276,9 +3280,15 @@ features:
tab: Local pipelines
stopOperation: STOP
table:
+ filtersLabel: Filters
connected: Connected
pipelineName: Pipeline name
gitStatus: Connected to Git
+ gitSyncStatus: Sync Status
+ gitSyncStatusAll: All
+ gitSyncStatusSynced: In sync
+ gitSyncStatusUnsynced: Out of sync
+ gitSyncStatusNotConnected: Not connected
gitStatusHelperText: This status indicates that the pipeline has been pushed to or pulled from the git repository in the past. It does not necessarily mean the content is up to date.
pullFail: Failed to pull this pipeline from remote.
pushFail: Failed to push this pipeline to remote.