diff --git a/deploy/docker/cp-search/config/application.properties b/deploy/docker/cp-search/config/application.properties index d9b2228303..c1aa08a2ba 100644 --- a/deploy/docker/cp-search/config/application.properties +++ b/deploy/docker/cp-search/config/application.properties @@ -56,6 +56,8 @@ sync.s3-file.bulk.load.tags.size=100 sync.s3-file.tag.value.delimiter=${CP_SEARCH_S3_FILE_TAG_DELIMITER:;} sync.s3-file.storage.ids=${CP_SEARCH_S3_INDEX_STORAGE_IDS:} sync.s3-file.storage.skip.ids=${CP_SEARCH_S3_INDEX_SKIP_STORAGE_IDS:} +sync.s3-file.storage.exclude.metadata.key=${CP_SEARCH_S3_FILE_STORAGE_EXCLUDE_METADATA_KEY:Billing status} +sync.s3-file.storage.exclude.metadata.value=${CP_SEARCH_S3_FILE_STORAGE_EXCLUDE_METADATA_VALUE:Exclude} #NFS Files Settings sync.nfs-file.disable=${CP_SEARCH_DISABLE_NFS_FILE:false} @@ -73,6 +75,8 @@ sync.gs-file.index.mapping=file://${CP_SEARCH_MAPPINGS_LOCATION}/storage_file.js sync.gs-file.bulk.insert.size=1000 sync.gs-file.bulk.load.tags.size=100 sync.gs-file.tag.value.delimiter=${CP_SEARCH_GS_FILE_TAG_DELIMITER:;} +sync.gs-file.storage.exclude.metadata.key=${CP_SEARCH_GS_FILE_STORAGE_EXCLUDE_METADATA_KEY:Billing status} +sync.gs-file.storage.exclude.metadata.value=${CP_SEARCH_GS_FILE_STORAGE_EXCLUDE_METADATA_VALUE:Exclude} #GS Storage Settings sync.gs-storage.disable=${CP_SEARCH_DISABLE_GS_STORAGE:false} @@ -105,6 +109,8 @@ sync.az-blob.index.name=az-blob sync.az-blob.bulk.insert.size=1000 sync.az-blob.bulk.load.tags.size=100 sync.az-file.tag.value.delimiter=${CP_SEARCH_AZ_FILE_TAG_DELIMITER:;} +sync.az-file.storage.exclude.metadata.key=${CP_SEARCH_AZ_FILE_STORAGE_EXCLUDE_METADATA_KEY:Billing status} +sync.az-file.storage.exclude.metadata.value=${CP_SEARCH_AZ_FILE_STORAGE_EXCLUDE_METADATA_VALUE:Exclude} #Tool Settings sync.tool.disable=${CP_SEARCH_DISABLE_TOOL:false} diff --git a/elasticsearch-agent/src/main/java/com/epam/pipeline/elasticsearchagent/app/AzureFileSyncConfiguration.java b/elasticsearch-agent/src/main/java/com/epam/pipeline/elasticsearchagent/app/AzureFileSyncConfiguration.java index 27afcd21de..7d5d9de65b 100644 --- a/elasticsearch-agent/src/main/java/com/epam/pipeline/elasticsearchagent/app/AzureFileSyncConfiguration.java +++ b/elasticsearch-agent/src/main/java/com/epam/pipeline/elasticsearchagent/app/AzureFileSyncConfiguration.java @@ -52,6 +52,12 @@ public class AzureFileSyncConfiguration { @Value("${sync.az-file.tag.value.delimiter:;}") private String tagDelimiter; + @Value("${sync.az-file.storage.exclude.metadata.key:Billing status}") + private String storageExcludeKey; + + @Value("${sync.az-file.storage.exclude.metadata.value:Exclude}") + private String storageExcludeValue; + @Bean public ObjectStorageFileManager azFileManager() { return new AzureBlobManager(); @@ -69,6 +75,7 @@ public ObjectStorageIndex azFileSynchronizer( indexSettingsPath, bulkInsertSize, bulkLoadTagsSize, DataStorageType.AZ, SearchDocumentType.AZ_BLOB_FILE, - tagDelimiter, false); + tagDelimiter, false, + storageExcludeKey, storageExcludeValue); } } diff --git a/elasticsearch-agent/src/main/java/com/epam/pipeline/elasticsearchagent/app/GSFileSyncConfiguration.java b/elasticsearch-agent/src/main/java/com/epam/pipeline/elasticsearchagent/app/GSFileSyncConfiguration.java index 2dbe5834ba..616e6fb447 100644 --- a/elasticsearch-agent/src/main/java/com/epam/pipeline/elasticsearchagent/app/GSFileSyncConfiguration.java +++ b/elasticsearch-agent/src/main/java/com/epam/pipeline/elasticsearchagent/app/GSFileSyncConfiguration.java @@ -48,6 +48,12 @@ public class GSFileSyncConfiguration { @Value("${sync.gs-file.tag.value.delimiter:;}") private String tagDelimiter; + @Value("${sync.gs-file.storage.exclude.metadata.key:Billing status}") + private String storageExcludeKey; + + @Value("${sync.gs-file.storage.exclude.metadata.value:Exclude}") + private String storageExcludeValue; + @Bean public ObjectStorageFileManager gsFileManager() { return new GsBucketFileManager(); @@ -65,7 +71,8 @@ public ObjectStorageIndex gsFileSynchronizer( indexSettingsPath, bulkInsertSize, bulkLoadTagsSize, DataStorageType.GS, SearchDocumentType.GS_FILE, - tagDelimiter, false); + tagDelimiter, false, + storageExcludeKey, storageExcludeValue); } } diff --git a/elasticsearch-agent/src/main/java/com/epam/pipeline/elasticsearchagent/app/S3FileSyncConfiguration.java b/elasticsearch-agent/src/main/java/com/epam/pipeline/elasticsearchagent/app/S3FileSyncConfiguration.java index c415ba0461..7a56b63321 100644 --- a/elasticsearch-agent/src/main/java/com/epam/pipeline/elasticsearchagent/app/S3FileSyncConfiguration.java +++ b/elasticsearch-agent/src/main/java/com/epam/pipeline/elasticsearchagent/app/S3FileSyncConfiguration.java @@ -57,6 +57,10 @@ public class S3FileSyncConfiguration { private String storageIds; @Value("${sync.s3-file.storage.skip.ids:}") private String skipStorageIds; + @Value("${sync.s3-file.storage.exclude.metadata.key:Billing status}") + private String storageExcludeKey; + @Value("${sync.s3-file.storage.exclude.metadata.value:Exclude}") + private String storageExcludeValue; @Bean public ObjectStorageFileManager s3FileManager() { @@ -76,7 +80,8 @@ public ObjectStorageIndex s3FileSynchronizer( DataStorageType.S3, SearchDocumentType.S3_FILE, tagDelimiter, - includeVersions); + includeVersions, + storageExcludeKey, storageExcludeValue); if (StringUtils.isNotBlank(storageIds)) { service.setStorageIds(parseIds(storageIds)); } diff --git a/elasticsearch-agent/src/main/java/com/epam/pipeline/elasticsearchagent/service/impl/CloudPipelineAPIClient.java b/elasticsearch-agent/src/main/java/com/epam/pipeline/elasticsearchagent/service/impl/CloudPipelineAPIClient.java index 2cf0f87e25..1c89ccc489 100644 --- a/elasticsearch-agent/src/main/java/com/epam/pipeline/elasticsearchagent/service/impl/CloudPipelineAPIClient.java +++ b/elasticsearch-agent/src/main/java/com/epam/pipeline/elasticsearchagent/service/impl/CloudPipelineAPIClient.java @@ -241,4 +241,8 @@ public List getStorageSearchMasks() { public FileShareMount loadFileShareMount(final Long id) { return executor.execute(cloudPipelineAPI.loadShareMount(id)); } + + public List searchEntriesByMetadata(final AclClass entityClass, final String key, final String value) { + return executor.execute(cloudPipelineAPI.searchMetadata(key, value, entityClass)); + } } diff --git a/elasticsearch-agent/src/main/java/com/epam/pipeline/elasticsearchagent/service/impl/ObjectStorageIndexImpl.java b/elasticsearch-agent/src/main/java/com/epam/pipeline/elasticsearchagent/service/impl/ObjectStorageIndexImpl.java index 264026dc44..492e6179d9 100644 --- a/elasticsearch-agent/src/main/java/com/epam/pipeline/elasticsearchagent/service/impl/ObjectStorageIndexImpl.java +++ b/elasticsearch-agent/src/main/java/com/epam/pipeline/elasticsearchagent/service/impl/ObjectStorageIndexImpl.java @@ -32,10 +32,12 @@ import com.epam.pipeline.entity.datastorage.lifecycle.restore.StorageRestoreAction; import com.epam.pipeline.entity.datastorage.lifecycle.restore.StorageRestorePathType; import com.epam.pipeline.entity.datastorage.lifecycle.restore.StorageRestoreStatus; +import com.epam.pipeline.entity.security.acl.AclClass; import com.epam.pipeline.entity.utils.DateUtils; import com.epam.pipeline.utils.StreamUtils; import com.epam.pipeline.entity.search.SearchDocumentType; import com.epam.pipeline.vo.EntityPermissionVO; +import com.epam.pipeline.vo.EntityVO; import com.epam.pipeline.vo.data.storage.DataStorageTagLoadBatchRequest; import com.epam.pipeline.vo.data.storage.DataStorageTagLoadRequest; import lombok.Getter; @@ -95,6 +97,8 @@ public class ObjectStorageIndexImpl implements ObjectStorageIndex { private final SearchDocumentType documentType; private final String tagDelimiter; private final boolean includeVersions; + private final String storageExcludeKey; + private final String storageExcludeValue; private Set storageIds; private Set skipStorageIds; @@ -105,17 +109,29 @@ public class ObjectStorageIndexImpl implements ObjectStorageIndex { public void synchronize(final LocalDateTime lastSyncTime, final LocalDateTime syncStart) { log.debug("Started {} files synchronization", getStorageType()); fileMapper.updateSearchMasks(cloudPipelineAPIClient, log); + final Set excludeStorageIds = loadExcludedStorageIds(); final List allStorages = cloudPipelineAPIClient.loadAllDataStorages(); allStorages .stream() + .filter(dataStorage -> CollectionUtils.isEmpty(excludeStorageIds) + || !excludeStorageIds.contains(dataStorage.getId())) .filter(dataStorage -> CollectionUtils.isEmpty(skipStorageIds) || !skipStorageIds.contains(dataStorage.getId())) - .filter(dataStorage -> CollectionUtils.isEmpty(storageIds) || storageIds.contains(dataStorage.getId())) + .filter(dataStorage -> CollectionUtils.isEmpty(storageIds) + || storageIds.contains(dataStorage.getId())) .filter(dataStorage -> dataStorage.getType() == getStorageType()) .filter(dataStorage -> isNotSharedOrChild(dataStorage, allStorages)) .forEach(this::indexStorage); } + private Set loadExcludedStorageIds() { + return ListUtils.emptyIfNull(cloudPipelineAPIClient.searchEntriesByMetadata(AclClass.DATA_STORAGE, + storageExcludeKey, storageExcludeValue)) + .stream() + .map(EntityVO::getEntityId) + .collect(Collectors.toSet()); + } + @Override @SuppressWarnings("PMD.AvoidCatchingGenericException") public void indexStorage(final AbstractDataStorage dataStorage) { diff --git a/elasticsearch-agent/src/main/resources/application.properties b/elasticsearch-agent/src/main/resources/application.properties index d1fc073252..87db7faa88 100644 --- a/elasticsearch-agent/src/main/resources/application.properties +++ b/elasticsearch-agent/src/main/resources/application.properties @@ -60,6 +60,8 @@ sync.az-blob.index.mapping=classpath:/templates/storage_file.json sync.az-blob.index.name=az-blob sync.az-blob.bulk.insert.size=1000 sync.az-blob.bulk.load.tags.size=100 +sync.az-file.storage.exclude.metadata.key=${CP_SEARCH_AZ_FILE_STORAGE_EXCLUDE_METADATA_KEY:Billing status} +sync.az-file.storage.exclude.metadata.value=${CP_SEARCH_AZ_FILE_STORAGE_EXCLUDE_METADATA_VALUE:Exclude} #S3 Files Settings #sync.s3-file.disable=true @@ -69,6 +71,8 @@ sync.s3-file.index.include.versions=false sync.s3-file.enable.tags=false sync.s3-file.bulk.insert.size=1000 sync.s3-file.bulk.load.tags.size=100 +sync.s3-file.storage.exclude.metadata.key=${CP_SEARCH_S3_FILE_STORAGE_EXCLUDE_METADATA_KEY:Billing status} +sync.s3-file.storage.exclude.metadata.value=${CP_SEARCH_S3_FILE_STORAGE_EXCLUDE_METADATA_VALUE:Exclude} #GS Files Settings #sync.gs-file.disable=true @@ -76,6 +80,8 @@ sync.gs-file.index.name=gs-file sync.gs-file.index.mapping=classpath:/templates/storage_file.json sync.gs-file.bulk.insert.size=1000 sync.gs-file.bulk.load.tags.size=100 +sync.gs-file.storage.exclude.metadata.key=${CP_SEARCH_GS_FILE_STORAGE_EXCLUDE_METADATA_KEY:Billing status} +sync.gs-file.storage.exclude.metadata.value=${CP_SEARCH_GS_FILE_STORAGE_EXCLUDE_METADATA_VALUE:Exclude} #GS Storage Settings #sync.gs-storage.disable=true diff --git a/elasticsearch-agent/src/test/java/com/epam/pipeline/elasticsearchagent/service/impl/ObjectStorageIndexTest.java b/elasticsearch-agent/src/test/java/com/epam/pipeline/elasticsearchagent/service/impl/ObjectStorageIndexTest.java index 58e40d0c4a..5ee3bb54b5 100644 --- a/elasticsearch-agent/src/test/java/com/epam/pipeline/elasticsearchagent/service/impl/ObjectStorageIndexTest.java +++ b/elasticsearch-agent/src/test/java/com/epam/pipeline/elasticsearchagent/service/impl/ObjectStorageIndexTest.java @@ -44,7 +44,9 @@ public class ObjectStorageIndexTest { private static final String TEST_BLOB_NAME_1 = "1"; private static final String TEST_BLOB_NAME_2 = "2"; - public static final int BULK_SIZE = 1000; + private static final int BULK_SIZE = 1000; + private static final String EXCLUDE_KEY = "key"; + private static final String EXCLUDE_VALUE = "value"; private final AbstractDataStorage dataStorage = new GSBucketStorage( 1L, "storage", "storage", new StoragePolicy(), null @@ -79,7 +81,8 @@ public void init() { BULK_SIZE, DataStorageType.GS, SearchDocumentType.GS_FILE, - ";", false) + ";", false, + EXCLUDE_KEY, EXCLUDE_VALUE) ); } diff --git a/elasticsearch-agent/src/test/java/com/epam/pipeline/elasticsearchagent/service/impl/ObjectStorageIndexVersionsTest.java b/elasticsearch-agent/src/test/java/com/epam/pipeline/elasticsearchagent/service/impl/ObjectStorageIndexVersionsTest.java index dceb440099..6ac3d33ba5 100644 --- a/elasticsearch-agent/src/test/java/com/epam/pipeline/elasticsearchagent/service/impl/ObjectStorageIndexVersionsTest.java +++ b/elasticsearch-agent/src/test/java/com/epam/pipeline/elasticsearchagent/service/impl/ObjectStorageIndexVersionsTest.java @@ -45,7 +45,9 @@ public class ObjectStorageIndexVersionsTest { private static final String TEST_BLOB_NAME_1 = "1"; private static final String TEST_BLOB_NAME_2 = "2"; - public static final int BULK_SIZE = 1000; + private static final int BULK_SIZE = 1000; + private static final String EXCLUDE_KEY = "key"; + private static final String EXCLUDE_VALUE = "value"; private final Supplier temporaryCredentials = () -> TemporaryCredentials.builder().region("").build(); @@ -77,7 +79,8 @@ public void init() { BULK_SIZE, DataStorageType.S3, SearchDocumentType.S3_FILE, - ";", true) + ";", true, + EXCLUDE_KEY, EXCLUDE_VALUE) ); }