Merge pull request #4524 from gchq/gh-4470-feed-import-vol-grp

PR for #4470 - Importing a Feed should not set the Volume group if it does not exist on the destination

settings.gradle

@@ -292,3 +292,4 @@ include 'stroom-view:stroom-view-impl'
 
 include 'stroom-ssl-test'
 include 'stroom-view:stroom-view-api'
+include 'stroom-index:stroom-index-api'

stroom-app/build.gradle

@@ -116,6 +116,7 @@ dependencies {
     implementation project(':stroom-feed:stroom-feed-impl')
     implementation project(':stroom-importexport:stroom-importexport-api')
     implementation project(':stroom-importexport:stroom-importexport-impl')
+    implementation project(':stroom-index:stroom-index-api')
     implementation project(':stroom-index:stroom-index-impl')
     implementation project(':stroom-index:stroom-index-lucene553')
     implementation project(':stroom-index:stroom-index-lucene980')

stroom-app/src/test/java/stroom/index/TestIndexShardServiceImpl.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2017 Crown Copyright
+ * Copyright 2017-2024 Crown Copyright
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -12,15 +12,14 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- *
  */
 
 package stroom.index;
 
 
+import stroom.index.api.IndexVolumeGroupService;
 import stroom.index.impl.IndexShardService;
 import stroom.index.impl.IndexStore;
-import stroom.index.impl.IndexVolumeGroupService;
 import stroom.index.impl.IndexVolumeService;
 import stroom.index.shared.IndexException;
 import stroom.index.shared.IndexVolume;

stroom-app/src/test/java/stroom/index/TestIndexVolumeGroupServiceImpl.java

@@ -1,6 +1,22 @@
+/*
+ * Copyright 2024 Crown Copyright
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package stroom.index;
 
-import stroom.index.impl.IndexVolumeGroupService;
+import stroom.index.api.IndexVolumeGroupService;
 import stroom.index.impl.selection.VolumeConfig;
 import stroom.index.shared.IndexVolumeGroup;
 import stroom.test.AbstractCoreIntegrationTest;

stroom-app/src/test/java/stroom/index/TestIndexVolumeServiceImpl.java

@@ -1,7 +1,23 @@
+/*
+ * Copyright 2024 Crown Copyright
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package stroom.index;
 
 import stroom.entity.shared.ExpressionCriteria;
-import stroom.index.impl.IndexVolumeGroupService;
+import stroom.index.api.IndexVolumeGroupService;
 import stroom.index.impl.IndexVolumeService;
 import stroom.index.impl.selection.VolumeConfig;
 import stroom.index.shared.IndexVolume;

stroom-app/src/test/java/stroom/index/VolumeCreatorForTesting.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2017 Crown Copyright
+ * Copyright 2017-2024 Crown Copyright
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -12,13 +12,12 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- *
  */
 
 package stroom.index;
 
 import stroom.entity.shared.ExpressionCriteria;
-import stroom.index.impl.IndexVolumeGroupService;
+import stroom.index.api.IndexVolumeGroupService;
 import stroom.index.impl.IndexVolumeService;
 import stroom.index.shared.IndexVolume;
 import stroom.index.shared.IndexVolumeGroup;

stroom-app/src/test/java/stroom/receive/TestReceiveDataServlet.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2017 Crown Copyright
+ * Copyright 2017-2024 Crown Copyright
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -12,7 +12,6 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- *
  */
 
 package stroom.receive;
@@ -72,7 +71,7 @@ void init() {
         response.resetMock();
         store.clear();
 
-        if (feedStore.list().size() == 0) {
+        if (feedStore.list().isEmpty()) {
             feedStore.createDocument("TEST-FEED");
         }
     }

stroom-app/src/test/java/stroom/test/SetupSampleDataBean.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2017 Crown Copyright
+ * Copyright 2017-2024 Crown Copyright
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -12,7 +12,6 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- *
  */
 
 package stroom.test;
@@ -28,8 +27,8 @@
 import stroom.feed.api.FeedStore;
 import stroom.importexport.impl.ImportExportSerializer;
 import stroom.importexport.shared.ImportSettings;
+import stroom.index.api.IndexVolumeGroupService;
 import stroom.index.impl.IndexStore;
-import stroom.index.impl.IndexVolumeGroupService;
 import stroom.index.impl.IndexVolumeService;
 import stroom.index.shared.IndexVolume;
 import stroom.meta.shared.MetaFields;
@@ -347,7 +346,7 @@ public void loadDirectory(final boolean shutdown, final Path importRootDir) {
                     dataLoader.loadInputStream(
                             feedName,
                             "Gen data",
-                        null,
+                            null,
                             StreamUtil.stringToStream(randomData),
                             false,
                             startTime);
@@ -459,7 +458,7 @@ private void generateSampleStatisticsData() {
                 dataLoader.loadInputStream(
                         STATS_COUNT_API_FEED_NAME,
                         "Sample statistics count data for export to API",
-                    null,
+                        null,
                         StreamUtil.stringToStream(sampleData),
                         false,
                         startTime);

stroom-data/stroom-data-store-api/build.gradle

@@ -2,6 +2,7 @@ ext.moduleName = 'stroom.data.store.api'
 
 dependencies {
     implementation project(':stroom-core-shared')
+    implementation project(':stroom-docref')
     implementation project(':stroom-meta:stroom-meta-api')
     implementation project(':stroom-util-shared')
     implementation project(':stroom-util')

stroom-data/stroom-data-store-api/src/main/java/stroom/data/store/api/FsVolumeGroupService.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2024 Crown Copyright
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package stroom.data.store.api;
+
+import stroom.data.store.impl.fs.shared.FsVolumeGroup;
+import stroom.docref.DocRef;
+
+import java.util.List;
+import java.util.Optional;
+
+public interface FsVolumeGroupService {
+
+    String ENTITY_TYPE = "FS_VOLUME_GROUP";
+    DocRef EVENT_DOCREF = new DocRef(ENTITY_TYPE, null, null);
+
+    List<String> getNames();
+
+    List<FsVolumeGroup> getAll();
+
+    FsVolumeGroup create();
+
+    FsVolumeGroup getOrCreate(String name);
+
+    FsVolumeGroup update(FsVolumeGroup indexVolumeGroup);
+
+    FsVolumeGroup get(String name);
+
+    FsVolumeGroup get(int id);
+
+    void delete(int id);
+
+    void ensureDefaultVolumes();
+
+    Optional<String> getDefaultVolumeGroup();
+}

stroom-data/stroom-data-store-impl-fs-db/build.gradle

@@ -4,6 +4,7 @@ dependencies {
     implementation project(':stroom-cache:stroom-cache-api')
     implementation project(':stroom-config:stroom-config-common')
     implementation project(':stroom-core-shared')
+    implementation project(':stroom-data:stroom-data-store-api')
     implementation project(':stroom-data:stroom-data-store-impl-fs')
     implementation project(':stroom-data:stroom-data-store-impl-fs-db-jooq')
     implementation project(':stroom-db-util')

stroom-data/stroom-data-store-impl-fs-db/src/test/java/stroom/data/store/impl/fs/db/TestFileSystemVolumeServiceImpl.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2016 Crown Copyright
+ * Copyright 2016-2024 Crown Copyright
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,11 +17,11 @@
 package stroom.data.store.impl.fs.db;
 
 
+import stroom.data.store.api.FsVolumeGroupService;
 import stroom.data.store.impl.fs.DataStoreServiceConfig.DataStoreServiceDbConfig;
 import stroom.data.store.impl.fs.FsVolumeConfig;
 import stroom.data.store.impl.fs.FsVolumeDao;
 import stroom.data.store.impl.fs.FsVolumeGroupDao;
-import stroom.data.store.impl.fs.FsVolumeGroupService;
 import stroom.data.store.impl.fs.FsVolumeGroupServiceImpl;
 import stroom.data.store.impl.fs.FsVolumeService;
 import stroom.data.store.impl.fs.FsVolumeStateDao;

stroom-data/stroom-data-store-impl-fs/src/main/java/stroom/data/store/impl/fs/FsDataStoreModule.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2018 Crown Copyright
+ * Copyright 2018-2024 Crown Copyright
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,6 +16,7 @@
 
 package stroom.data.store.impl.fs;
 
+import stroom.data.store.api.FsVolumeGroupService;
 import stroom.data.store.api.Store;
 import stroom.data.store.impl.AttributeMapFactory;
 import stroom.data.store.impl.fs.shared.FsVolume;

stroom-data/stroom-data-store-impl-fs/src/main/java/stroom/data/store/impl/fs/FsVolumeGroupResourceImpl.java

@@ -1,5 +1,22 @@
+/*
+ * Copyright 2024 Crown Copyright
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package stroom.data.store.impl.fs;
 
+import stroom.data.store.api.FsVolumeGroupService;
 import stroom.data.store.impl.fs.shared.FsVolumeGroup;
 import stroom.data.store.impl.fs.shared.FsVolumeGroupResource;
 import stroom.entity.shared.ExpressionCriteria;

stroom-data/stroom-data-store-impl-fs/src/main/java/stroom/data/store/impl/fs/FsVolumeGroupServiceImpl.java

@@ -1,5 +1,22 @@
+/*
+ * Copyright 2024 Crown Copyright
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package stroom.data.store.impl.fs;
 
+import stroom.data.store.api.FsVolumeGroupService;
 import stroom.data.store.impl.fs.shared.FsVolumeGroup;
 import stroom.security.api.SecurityContext;
 import stroom.security.shared.PermissionNames;
@@ -18,6 +35,7 @@
 import jakarta.inject.Singleton;
 
 import java.util.List;
+import java.util.Optional;
 
 @Singleton
 @EntityEventHandler(type = FsVolumeGroupServiceImpl.ENTITY_TYPE, action = {
@@ -130,6 +148,11 @@ public void ensureDefaultVolumes() {
         }
     }
 
+    @Override
+    public Optional<String> getDefaultVolumeGroup() {
+        return Optional.ofNullable(volumeConfigProvider.get().getDefaultStreamVolumeGroupName());
+    }
+
     private synchronized void createDefaultVolumes() {
         if (!createdDefaultVolumes && !creatingDefaultVolumes) {
             try {

stroom-data/stroom-data-store-impl-fs/src/main/java/stroom/data/store/impl/fs/FsVolumeService.java

@@ -1,6 +1,23 @@
+/*
+ * Copyright 2024 Crown Copyright
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package stroom.data.store.impl.fs;
 
 import stroom.cluster.lock.api.ClusterLockService;
+import stroom.data.store.api.FsVolumeGroupService;
 import stroom.data.store.impl.fs.shared.FindFsVolumeCriteria;
 import stroom.data.store.impl.fs.shared.FsVolume;
 import stroom.data.store.impl.fs.shared.FsVolume.VolumeUseStatus;