Skip to content

Commit

Permalink
Merge pull request #4524 from gchq/gh-4470-feed-import-vol-grp
Browse files Browse the repository at this point in the history
PR for #4470 - Importing a Feed should not set the Volume group if it does not exist on the destination
  • Loading branch information
at055612 authored Oct 9, 2024
2 parents 70822c6 + 51abda9 commit 399efd0
Show file tree
Hide file tree
Showing 48 changed files with 2,427 additions and 102 deletions.
1 change: 1 addition & 0 deletions settings.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -292,3 +292,4 @@ include 'stroom-view:stroom-view-impl'

include 'stroom-ssl-test'
include 'stroom-view:stroom-view-api'
include 'stroom-index:stroom-index-api'
1 change: 1 addition & 0 deletions stroom-app/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ dependencies {
implementation project(':stroom-feed:stroom-feed-impl')
implementation project(':stroom-importexport:stroom-importexport-api')
implementation project(':stroom-importexport:stroom-importexport-impl')
implementation project(':stroom-index:stroom-index-api')
implementation project(':stroom-index:stroom-index-impl')
implementation project(':stroom-index:stroom-index-lucene553')
implementation project(':stroom-index:stroom-index-lucene980')
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2017 Crown Copyright
* Copyright 2017-2024 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -12,15 +12,14 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

package stroom.index;


import stroom.index.api.IndexVolumeGroupService;
import stroom.index.impl.IndexShardService;
import stroom.index.impl.IndexStore;
import stroom.index.impl.IndexVolumeGroupService;
import stroom.index.impl.IndexVolumeService;
import stroom.index.shared.IndexException;
import stroom.index.shared.IndexVolume;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,22 @@
/*
* Copyright 2024 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package stroom.index;

import stroom.index.impl.IndexVolumeGroupService;
import stroom.index.api.IndexVolumeGroupService;
import stroom.index.impl.selection.VolumeConfig;
import stroom.index.shared.IndexVolumeGroup;
import stroom.test.AbstractCoreIntegrationTest;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,23 @@
/*
* Copyright 2024 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package stroom.index;

import stroom.entity.shared.ExpressionCriteria;
import stroom.index.impl.IndexVolumeGroupService;
import stroom.index.api.IndexVolumeGroupService;
import stroom.index.impl.IndexVolumeService;
import stroom.index.impl.selection.VolumeConfig;
import stroom.index.shared.IndexVolume;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2017 Crown Copyright
* Copyright 2017-2024 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -12,13 +12,12 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

package stroom.index;

import stroom.entity.shared.ExpressionCriteria;
import stroom.index.impl.IndexVolumeGroupService;
import stroom.index.api.IndexVolumeGroupService;
import stroom.index.impl.IndexVolumeService;
import stroom.index.shared.IndexVolume;
import stroom.index.shared.IndexVolumeGroup;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2017 Crown Copyright
* Copyright 2017-2024 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -12,7 +12,6 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

package stroom.receive;
Expand Down Expand Up @@ -72,7 +71,7 @@ void init() {
response.resetMock();
store.clear();

if (feedStore.list().size() == 0) {
if (feedStore.list().isEmpty()) {
feedStore.createDocument("TEST-FEED");
}
}
Expand Down
9 changes: 4 additions & 5 deletions stroom-app/src/test/java/stroom/test/SetupSampleDataBean.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2017 Crown Copyright
* Copyright 2017-2024 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -12,7 +12,6 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

package stroom.test;
Expand All @@ -28,8 +27,8 @@
import stroom.feed.api.FeedStore;
import stroom.importexport.impl.ImportExportSerializer;
import stroom.importexport.shared.ImportSettings;
import stroom.index.api.IndexVolumeGroupService;
import stroom.index.impl.IndexStore;
import stroom.index.impl.IndexVolumeGroupService;
import stroom.index.impl.IndexVolumeService;
import stroom.index.shared.IndexVolume;
import stroom.meta.shared.MetaFields;
Expand Down Expand Up @@ -347,7 +346,7 @@ public void loadDirectory(final boolean shutdown, final Path importRootDir) {
dataLoader.loadInputStream(
feedName,
"Gen data",
null,
null,
StreamUtil.stringToStream(randomData),
false,
startTime);
Expand Down Expand Up @@ -459,7 +458,7 @@ private void generateSampleStatisticsData() {
dataLoader.loadInputStream(
STATS_COUNT_API_FEED_NAME,
"Sample statistics count data for export to API",
null,
null,
StreamUtil.stringToStream(sampleData),
false,
startTime);
Expand Down
1 change: 1 addition & 0 deletions stroom-data/stroom-data-store-api/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ ext.moduleName = 'stroom.data.store.api'

dependencies {
implementation project(':stroom-core-shared')
implementation project(':stroom-docref')
implementation project(':stroom-meta:stroom-meta-api')
implementation project(':stroom-util-shared')
implementation project(':stroom-util')
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
/*
* Copyright 2024 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package stroom.data.store.api;

import stroom.data.store.impl.fs.shared.FsVolumeGroup;
import stroom.docref.DocRef;

import java.util.List;
import java.util.Optional;

public interface FsVolumeGroupService {

String ENTITY_TYPE = "FS_VOLUME_GROUP";
DocRef EVENT_DOCREF = new DocRef(ENTITY_TYPE, null, null);

List<String> getNames();

List<FsVolumeGroup> getAll();

FsVolumeGroup create();

FsVolumeGroup getOrCreate(String name);

FsVolumeGroup update(FsVolumeGroup indexVolumeGroup);

FsVolumeGroup get(String name);

FsVolumeGroup get(int id);

void delete(int id);

void ensureDefaultVolumes();

Optional<String> getDefaultVolumeGroup();
}
1 change: 1 addition & 0 deletions stroom-data/stroom-data-store-impl-fs-db/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ dependencies {
implementation project(':stroom-cache:stroom-cache-api')
implementation project(':stroom-config:stroom-config-common')
implementation project(':stroom-core-shared')
implementation project(':stroom-data:stroom-data-store-api')
implementation project(':stroom-data:stroom-data-store-impl-fs')
implementation project(':stroom-data:stroom-data-store-impl-fs-db-jooq')
implementation project(':stroom-db-util')
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2016 Crown Copyright
* Copyright 2016-2024 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -17,11 +17,11 @@
package stroom.data.store.impl.fs.db;


import stroom.data.store.api.FsVolumeGroupService;
import stroom.data.store.impl.fs.DataStoreServiceConfig.DataStoreServiceDbConfig;
import stroom.data.store.impl.fs.FsVolumeConfig;
import stroom.data.store.impl.fs.FsVolumeDao;
import stroom.data.store.impl.fs.FsVolumeGroupDao;
import stroom.data.store.impl.fs.FsVolumeGroupService;
import stroom.data.store.impl.fs.FsVolumeGroupServiceImpl;
import stroom.data.store.impl.fs.FsVolumeService;
import stroom.data.store.impl.fs.FsVolumeStateDao;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2018 Crown Copyright
* Copyright 2018-2024 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -16,6 +16,7 @@

package stroom.data.store.impl.fs;

import stroom.data.store.api.FsVolumeGroupService;
import stroom.data.store.api.Store;
import stroom.data.store.impl.AttributeMapFactory;
import stroom.data.store.impl.fs.shared.FsVolume;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,22 @@
/*
* Copyright 2024 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package stroom.data.store.impl.fs;

import stroom.data.store.api.FsVolumeGroupService;
import stroom.data.store.impl.fs.shared.FsVolumeGroup;
import stroom.data.store.impl.fs.shared.FsVolumeGroupResource;
import stroom.entity.shared.ExpressionCriteria;
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,5 +1,22 @@
/*
* Copyright 2024 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package stroom.data.store.impl.fs;

import stroom.data.store.api.FsVolumeGroupService;
import stroom.data.store.impl.fs.shared.FsVolumeGroup;
import stroom.security.api.SecurityContext;
import stroom.security.shared.PermissionNames;
Expand All @@ -18,6 +35,7 @@
import jakarta.inject.Singleton;

import java.util.List;
import java.util.Optional;

@Singleton
@EntityEventHandler(type = FsVolumeGroupServiceImpl.ENTITY_TYPE, action = {
Expand Down Expand Up @@ -130,6 +148,11 @@ public void ensureDefaultVolumes() {
}
}

@Override
public Optional<String> getDefaultVolumeGroup() {
return Optional.ofNullable(volumeConfigProvider.get().getDefaultStreamVolumeGroupName());
}

private synchronized void createDefaultVolumes() {
if (!createdDefaultVolumes && !creatingDefaultVolumes) {
try {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,23 @@
/*
* Copyright 2024 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package stroom.data.store.impl.fs;

import stroom.cluster.lock.api.ClusterLockService;
import stroom.data.store.api.FsVolumeGroupService;
import stroom.data.store.impl.fs.shared.FindFsVolumeCriteria;
import stroom.data.store.impl.fs.shared.FsVolume;
import stroom.data.store.impl.fs.shared.FsVolume.VolumeUseStatus;
Expand Down
Loading

0 comments on commit 399efd0

Please sign in to comment.