From 097daee2e66b3cdbe757cdb22152b016816236e4 Mon Sep 17 00:00:00 2001 From: Marcus Talbott Date: Wed, 24 Jul 2024 09:59:38 -0400 Subject: [PATCH 01/25] liquibase changeset to create workspace settings table --- .../dsde/rawls/liquibase/changelog.xml | 1 + .../20240723_workspace_settings.xml | 34 +++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changesets/20240723_workspace_settings.xml diff --git a/core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changelog.xml b/core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changelog.xml index f7405381f8..2abf6f7038 100644 --- a/core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changelog.xml +++ b/core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changelog.xml @@ -127,4 +127,5 @@ + diff --git a/core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changesets/20240723_workspace_settings.xml b/core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changesets/20240723_workspace_settings.xml new file mode 100644 index 0000000000..778107c966 --- /dev/null +++ b/core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changesets/20240723_workspace_settings.xml @@ -0,0 +1,34 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 1c2287ad4f6af7bf5ed30760744fe5fa5bc5bd1e Mon Sep 17 00:00:00 2001 From: Marcus Talbott Date: Mon, 29 Jul 2024 09:47:25 -0400 Subject: [PATCH 02/25] wip, compiles but untested --- .../20240723_workspace_settings.xml | 4 +- .../rawls/dataaccess/GoogleServicesDAO.scala | 3 + .../dataaccess/HttpGoogleServicesDAO.scala | 22 ++--- .../rawls/dataaccess/slick/DataAccess.scala | 3 +- .../slick/WorkspaceSettingComponent.scala | 97 +++++++++++++++++++ .../webservice/WorkspaceApiServiceV2.scala | 20 ++++ .../rawls/workspace/WorkspaceRepository.scala | 32 ++++-- .../rawls/workspace/WorkspaceService.scala | 46 +++++++++ .../dsde/rawls/model/WorkspaceModel.scala | 68 +++++++++++++ 9 files changed, 272 insertions(+), 23 deletions(-) create mode 100644 core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala diff --git a/core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changesets/20240723_workspace_settings.xml b/core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changesets/20240723_workspace_settings.xml index 778107c966..c89a892b77 100644 --- a/core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changesets/20240723_workspace_settings.xml +++ b/core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changesets/20240723_workspace_settings.xml @@ -28,7 +28,7 @@ + columnNames="WORKSPACE_ID, TYPE, STATUS" + constraintName="unique_applied_setting_type"/> diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/GoogleServicesDAO.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/GoogleServicesDAO.scala index 44f031d24e..3ec30f6c6b 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/GoogleServicesDAO.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/GoogleServicesDAO.scala @@ -6,6 +6,7 @@ import com.google.api.services.cloudbilling.model.ProjectBillingInfo import com.google.api.services.cloudresourcemanager.model.Project import com.google.api.services.directory.model.Group import com.google.api.services.storage.model.{Bucket, BucketAccessControl, StorageObject} +import com.google.cloud.storage.BucketInfo.LifecycleRule import org.broadinstitute.dsde.rawls.google.AccessContextManagerDAO import org.broadinstitute.dsde.rawls.model.WorkspaceAccessLevels._ import org.broadinstitute.dsde.rawls.model._ @@ -63,6 +64,8 @@ trait GoogleServicesDAO extends ErrorReportable { */ def deleteBucket(bucketName: String): Future[Boolean] + def setBucketLifecycle(bucketName: String, lifecycle: List[LifecycleRule]): Future[Unit] + def isAdmin(userEmail: String): Future[Boolean] def isLibraryCurator(userEmail: String): Future[Boolean] diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpGoogleServicesDAO.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpGoogleServicesDAO.scala index 1622a9c93a..f651f6619c 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpGoogleServicesDAO.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpGoogleServicesDAO.scala @@ -18,12 +18,7 @@ import com.google.api.client.googleapis.json.GoogleJsonResponseException import com.google.api.client.http.{HttpRequest, HttpRequestInitializer, HttpResponseException, InputStreamContent} import com.google.api.client.json.gson.GsonFactory import com.google.api.services.cloudbilling.Cloudbilling -import com.google.api.services.cloudbilling.model.{ - BillingAccount, - ListBillingAccountsResponse, - ProjectBillingInfo, - TestIamPermissionsRequest -} +import com.google.api.services.cloudbilling.model.{BillingAccount, ListBillingAccountsResponse, ProjectBillingInfo, TestIamPermissionsRequest} import com.google.api.services.cloudresourcemanager.CloudResourceManager import com.google.api.services.cloudresourcemanager.model._ import com.google.api.services.compute.{Compute, ComputeScopes} @@ -41,7 +36,7 @@ import com.google.api.services.storage.{Storage, StorageScopes} import com.google.auth.oauth2.ServiceAccountCredentials import com.google.cloud.Identity import com.google.cloud.storage.Storage.BucketSourceOption -import com.google.cloud.storage.{Cors, HttpMethod, StorageClass, StorageException} +import com.google.cloud.storage.{BucketInfo, Cors, HttpMethod, StorageClass, StorageException} import io.opentelemetry.api.common.AttributeKey import org.apache.commons.lang3.StringUtils import org.broadinstitute.dsde.rawls.dataaccess.CloudResourceManagerV2Model.{Folder, FolderSearchResponse} @@ -52,11 +47,7 @@ import org.broadinstitute.dsde.rawls.metrics.GoogleInstrumentedService import org.broadinstitute.dsde.rawls.model.UserAuthJsonSupport._ import org.broadinstitute.dsde.rawls.model.WorkspaceAccessLevels._ import org.broadinstitute.dsde.rawls.model._ -import org.broadinstitute.dsde.rawls.util.TracingUtils.{ - setTraceSpanAttribute, - traceFutureWithParent, - traceNakedWithParent -} +import org.broadinstitute.dsde.rawls.util.TracingUtils.{setTraceSpanAttribute, traceFutureWithParent, traceNakedWithParent} import org.broadinstitute.dsde.rawls.util.{FutureSupport, HttpClientUtilsStandard} import org.broadinstitute.dsde.rawls.{RawlsException, RawlsExceptionWithErrorReport} import org.broadinstitute.dsde.workbench.google.{GoogleCredentialModes, HttpGoogleIamDAO} @@ -331,6 +322,13 @@ class HttpGoogleServicesDAO(val clientSecrets: GoogleClientSecrets, } } + override def setBucketLifecycle(bucketName: String, lifecycle: List[BucketInfo.LifecycleRule]): Future[Unit] = + googleStorageService + .setBucketLifecycle(GcsBucketName(bucketName), lifecycle) + .compile + .drain + .unsafeToFuture() + override def isAdmin(userEmail: String): Future[Boolean] = hasGoogleRole(adminGroupName, userEmail) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/DataAccess.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/DataAccess.scala index 4285ba4e6e..264f446d5d 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/DataAccess.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/DataAccess.scala @@ -27,7 +27,8 @@ trait DataAccess with WorkspaceFeatureFlagComponent with WorkspaceManagerResourceMonitorRecordComponent with FastPassGrantComponent - with MultiregionalBucketMigrationHistory { + with MultiregionalBucketMigrationHistory + with WorkspaceSettingComponent { this: DriverComponent => diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala new file mode 100644 index 0000000000..b2987d00e8 --- /dev/null +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala @@ -0,0 +1,97 @@ +package org.broadinstitute.dsde.rawls.dataaccess.slick + +import org.broadinstitute.dsde.rawls.model.WorkspaceJsonSupport.GcpBucketLifecycleConfigFormat +import org.broadinstitute.dsde.rawls.model.WorkspaceSettingConfig.GcpBucketLifecycleConfig +import org.broadinstitute.dsde.rawls.model.WorkspaceSettingTypes.WorkspaceSettingType +import org.broadinstitute.dsde.rawls.model._ + +import java.sql.Timestamp +import java.util.{Date, UUID} + +case class WorkspaceSettingRecord(`type`: String, + workspaceId: UUID, + config: Option[String], + status: String, + createdTime: Timestamp, + lastUpdated: Timestamp +) + +object WorkspaceSettingRecord { + object SettingStatus extends SlickEnum { + type SettingStatus = Value + val Applying: Value = Value("Applying") + val Applied: Value = Value("Applied") + } + + def toWorkspaceSettingRecord(workspaceId: UUID, workspaceSettings: WorkspaceSettings): WorkspaceSettingRecord = { + import spray.json._ + import DefaultJsonProtocol._ + import WorkspaceJsonSupport._ + + val currentTime = new Timestamp(new Date().getTime) + val configString = workspaceSettings.config.toJson.compactPrint + WorkspaceSettingRecord(workspaceSettings.`type`.toString, + workspaceId, + Option(configString), + WorkspaceSettingRecord.SettingStatus.Applying.toString, + currentTime, + currentTime + ) + } + + def toWorkspaceSettings(workspaceSettingRecord: WorkspaceSettingRecord): WorkspaceSettings = { + import spray.json._ + + val settingType = WorkspaceSettingTypes.withName(workspaceSettingRecord.`type`) + val settingConfig = workspaceSettingRecord.config.map { configuration => + settingType match { + case WorkspaceSettingTypes.GcpBucketLifecycle => configuration.parseJson.convertTo[GcpBucketLifecycleConfig] + } + } + WorkspaceSettings(settingType, settingConfig) + } +} + +trait WorkspaceSettingComponent { + this: DriverComponent with WorkspaceComponent => + + import driver.api._ + class WorkspaceSettingTable(tag: Tag) extends Table[WorkspaceSettingRecord](tag, "WORKSPACE_SETTINGS") { + def `type` = column[String]("type", O.Length(254)) + def workspaceId = column[UUID]("workspace_id") + def config = column[Option[String]]("config") + def status = column[String]("status", O.Length(254)) + def createdTime = column[Timestamp]("created_time") + def lastUpdated = column[Timestamp]("last_updated") + + def * = (`type`, workspaceId, config, status, createdTime, lastUpdated) <> (WorkspaceSettingRecord.tupled, + WorkspaceSettingRecord.unapply + ) + } + + object workspaceSettingQuery extends TableQuery(new WorkspaceSettingTable(_)) { + def saveAll(workspaceId: UUID, workspaceSettings: List[WorkspaceSettings]): ReadWriteAction[List[WorkspaceSettings]] = { + val records = workspaceSettings.map(WorkspaceSettingRecord.toWorkspaceSettingRecord(workspaceId, _)) + (workspaceSettingQuery ++= records).map(_ => workspaceSettings) + } + + def updateStatuses(workspaceId: UUID, + workspaceSettingTypes: List[WorkspaceSettingType], + status: WorkspaceSettingRecord.SettingStatus.SettingStatus + ): ReadWriteAction[Int] = + workspaceSettingQuery + .filter(record => + record.workspaceId === workspaceId && record.`type`.inSetBind(workspaceSettingTypes.map(_.toString)) + ) + .map(_.status) + .update(status.toString) + + def deleteSettingsForWorkspace(workspaceId: UUID, + status: WorkspaceSettingRecord.SettingStatus.SettingStatus + ): ReadWriteAction[Int] = + filter(record => record.workspaceId === workspaceId && record.status === status.toString).delete + + def listAllForWorkspace(workspaceId: UUID): ReadAction[List[WorkspaceSettings]] = + filter(_.workspaceId === workspaceId).result.map(_.map(WorkspaceSettingRecord.toWorkspaceSettings).toList) + } +} diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/WorkspaceApiServiceV2.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/WorkspaceApiServiceV2.scala index 7e56d784bf..c89305614f 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/WorkspaceApiServiceV2.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/WorkspaceApiServiceV2.scala @@ -86,6 +86,26 @@ trait WorkspaceApiServiceV2 extends UserInfoDirectives { } } } + } ~ + pathPrefix("settings") { + pathEndOrSingleSlash { + get { + complete { + workspaceServiceConstructor(ctx) + .getWorkspaceSettings(workspaceName) + .map(StatusCodes.OK -> _) + } + } ~ + put { + entity(as[List[WorkspaceSettings]]) { settings => + complete { + workspaceServiceConstructor(ctx) + .setWorkspaceSettings(workspaceName, settings) + .map(StatusCodes.OK -> _) + } + } + } + } } } ~ pathPrefix("bucketMigration") { diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala index f0bbd5815a..cd51723409 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala @@ -3,15 +3,9 @@ package org.broadinstitute.dsde.rawls.workspace import akka.http.scaladsl.model.StatusCodes import org.broadinstitute.dsde.rawls.RawlsExceptionWithErrorReport import org.broadinstitute.dsde.rawls.dataaccess.SlickDataSource +import org.broadinstitute.dsde.rawls.dataaccess.slick.WorkspaceSettingRecord import org.broadinstitute.dsde.rawls.model.Attributable.AttributeMap -import org.broadinstitute.dsde.rawls.model.{ - ErrorReport, - RawlsRequestContext, - Workspace, - WorkspaceAttributeSpecs, - WorkspaceName, - WorkspaceState -} +import org.broadinstitute.dsde.rawls.model.{ErrorReport, RawlsRequestContext, Workspace, WorkspaceAttributeSpecs, WorkspaceName, WorkspaceSettings, WorkspaceState} import org.broadinstitute.dsde.rawls.model.WorkspaceState.WorkspaceState import org.broadinstitute.dsde.rawls.util.TracingUtils.traceDBIOWithParent import org.joda.time.DateTime @@ -99,4 +93,26 @@ class WorkspaceRepository(dataSource: SlickDataSource) { } yield newWorkspace } + def getWorkspaceSettings(workspaceId: UUID): Future[List[WorkspaceSettings]] = + dataSource.inTransaction { access => + access.workspaceSettingQuery.listAllForWorkspace(workspaceId) + } + + def overwriteWorkspaceSettings(workspaceId: UUID, workspaceSettings: List[WorkspaceSettings]): Future[List[WorkspaceSettings]] = + dataSource.inTransaction { access => + access.workspaceSettingQuery.saveAll(workspaceId, workspaceSettings) + } + + def markWorkspaceSettingsApplied(workspaceId: UUID, workspaceSettings: List[WorkspaceSettings])(implicit ex: ExecutionContext): Future[Int] = + dataSource.inTransaction { access => + for { + _ <- access.workspaceSettingQuery.deleteSettingsForWorkspace(workspaceId, WorkspaceSettingRecord.SettingStatus.Applied) + res <- access.workspaceSettingQuery.updateStatuses(workspaceId, workspaceSettings.map(_.`type`), WorkspaceSettingRecord.SettingStatus.Applied) + } yield res + } + + def removeUnappliedSettings(workspaceId: UUID): Future[Int] = + dataSource.inTransaction { access => + access.workspaceSettingQuery.deleteSettingsForWorkspace(workspaceId, WorkspaceSettingRecord.SettingStatus.Applying) + } } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala index 23e477bb93..cee1eb8ffe 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala @@ -47,7 +47,10 @@ import spray.json.DefaultJsonProtocol._ import spray.json._ import org.broadinstitute.dsde.rawls.metrics.MetricsHelper import cats.effect.unsafe.implicits.global +import com.google.cloud.storage.BucketInfo.LifecycleRule +import com.google.cloud.storage.BucketInfo.LifecycleRule.{LifecycleCondition, LifecycleAction} import org.broadinstitute.dsde.rawls.billing.BillingRepository +import org.broadinstitute.dsde.rawls.model.WorkspaceSettingConfig.GcpBucketLifecycleConfig import java.io.IOException import java.util.UUID @@ -2001,6 +2004,49 @@ class WorkspaceService( } } yield {} + def getWorkspaceSettings(workspaceName: WorkspaceName): Future[List[WorkspaceSettings]] = { + getV2WorkspaceContextAndPermissions(workspaceName, SamWorkspaceActions.read).flatMap { workspace => + workspaceRepository.getWorkspaceSettings(workspace.workspaceIdAsUUID) + } + } + + def setWorkspaceSettings(workspaceName: WorkspaceName, workspaceSettings: List[WorkspaceSettings]): Future[List[WorkspaceSettings]] = { + def applySettings(workspace: Workspace, settings: WorkspaceSettings): Future[Unit] = { + settings match { + case WorkspaceSettings(WorkspaceSettingTypes.GcpBucketLifecycle, Some(GcpBucketLifecycleConfig(rules))) => { + val googleRules = rules.map { rule => + val conditionBuilder = LifecycleCondition.newBuilder().setMatchesPrefix(rule.conditions.matchesPrefix.toList.asJava) + rule.conditions.age.map(age => conditionBuilder.setAge(age)) + + val action = rule.action.`type` match { + case actionType if actionType.equals("Delete") => LifecycleAction.newDeleteAction() + case _ => throw new RawlsException("unsupported lifecycle action") + } + + new LifecycleRule(action, conditionBuilder.build()) + } + + gcsDAO.setBucketLifecycle(workspace.bucketName, googleRules) + } + case _ => throw new RawlsException("unsupported workspace setting") + } + } + + getV2WorkspaceContextAndPermissions(workspaceName, SamWorkspaceActions.own).flatMap { workspace => + (for { + _ <- workspaceRepository.overwriteWorkspaceSettings(workspace.workspaceIdAsUUID, workspaceSettings) + _ <- workspaceSettings.traverse(s => applySettings(workspace, s)) // todo: maybe this should catch exceptions and keep track of which settings went through and which didn't. then it can roll back anything that failed. + _ <- workspaceRepository.markWorkspaceSettingsApplied(workspace.workspaceIdAsUUID, workspaceSettings) + } yield { + workspaceSettings + }).recoverWith { case e => + workspaceRepository.removeUnappliedSettings(workspace.workspaceIdAsUUID).flatMap { _ => + Future.failed(new RawlsException("Failed to apply workspace settings")) + } + } + } + } + // helper methods private def createWorkflowCollectionForWorkspace(workspaceId: String, diff --git a/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala b/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala index cbd0aa78db..7e1afdc235 100644 --- a/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala +++ b/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala @@ -11,6 +11,8 @@ import org.broadinstitute.dsde.rawls.model.SortDirections.SortDirection import org.broadinstitute.dsde.rawls.model.UserModelJsonSupport.ManagedGroupRefFormat import org.broadinstitute.dsde.rawls.model.WorkspaceAccessLevels.WorkspaceAccessLevel import org.broadinstitute.dsde.rawls.model.WorkspaceCloudPlatform.WorkspaceCloudPlatform +import org.broadinstitute.dsde.rawls.model.WorkspaceSettingConfig.{GcpBucketLifecycleConfig, GcpBucketLifecycleRule, GcpLifecycleAction, GcpLifecycleCondition} +import org.broadinstitute.dsde.rawls.model.WorkspaceSettingTypes.{GcpBucketLifecycle, WorkspaceSettingType} import org.broadinstitute.dsde.rawls.model.WorkspaceState.WorkspaceState import org.broadinstitute.dsde.rawls.model.WorkspaceType.WorkspaceType import org.broadinstitute.dsde.rawls.model.WorkspaceVersions.WorkspaceVersion @@ -561,6 +563,31 @@ object WorkspaceState { case object DeleteFailed extends WorkspaceState } +case class WorkspaceSettings(`type`: WorkspaceSettingType, config: Option[WorkspaceSettingConfig]) + +object WorkspaceSettingTypes { + sealed trait WorkspaceSettingType extends RawlsEnumeration[WorkspaceSettingType] { + override def toString: String = getClass.getSimpleName.stripSuffix("$") + override def withName(name: String): WorkspaceSettingType = WorkspaceSettingTypes.withName(name) + } + + def withName(name: String): WorkspaceSettingType = name.toLowerCase match { + case "gcp_bucket_lifecycle" => GcpBucketLifecycle + case _ => throw new RawlsException(s"invalid WorkspaceSetting [$name]") + } + + case object GcpBucketLifecycle extends WorkspaceSettingType +} + +sealed trait WorkspaceSettingConfig +object WorkspaceSettingConfig { + case class GcpBucketLifecycleConfig(rules: List[GcpBucketLifecycleRule]) extends WorkspaceSettingConfig + + case class GcpBucketLifecycleRule(action: GcpLifecycleAction, conditions: GcpLifecycleCondition) + case class GcpLifecycleAction(`type`: String) + case class GcpLifecycleCondition(matchesPrefix: Set[String], age: Option[Int]) +} + sealed trait MethodRepoMethod { def methodUri: String @@ -1165,6 +1192,47 @@ class WorkspaceJsonSupport extends JsonSupport { } } + implicit val GcpLifecycleConditionFormat: RootJsonFormat[GcpLifecycleCondition] = jsonFormat2(GcpLifecycleCondition.apply) + implicit val GcpLifecycleActionFormat: RootJsonFormat[GcpLifecycleAction] = jsonFormat1(GcpLifecycleAction.apply) + implicit val GcpBucketLifecycleRuleFormat: RootJsonFormat[GcpBucketLifecycleRule] = jsonFormat2(GcpBucketLifecycleRule.apply) + implicit val GcpBucketLifecycleConfigFormat: RootJsonFormat[GcpBucketLifecycleConfig] = jsonFormat1(GcpBucketLifecycleConfig.apply) + + implicit object WorkspaceSettingTypeFormat extends RootJsonFormat[WorkspaceSettingType] { + override def write(obj: WorkspaceSettingType): JsValue = JsString(obj.toString) + + override def read(json: JsValue): WorkspaceSettingType = json match { + case JsString(name) => WorkspaceSettingTypes.withName(name) + case _ => throw DeserializationException("unexpected json type") + } + } + + implicit object WorkspaceSettingsConfigurationFormat extends RootJsonFormat[WorkspaceSettingConfig] { + def write(obj: WorkspaceSettingConfig): JsValue = obj match { + case config: GcpBucketLifecycleConfig => config.toJson + } + + def read(json: JsValue): WorkspaceSettingConfig = { + throw DeserializationException("WorkspaceSettingsConfiguration cannot be read directly") + } + } + + implicit object WorkspaceSettingsFormat extends RootJsonFormat[WorkspaceSettings] { + def write(ws: WorkspaceSettings): JsValue = JsObject( + "type" -> ws.`type`.toJson, + "config" -> ws.config.toJson + ) + + def read(json: JsValue): WorkspaceSettings = { + val fields = json.asJsObject.fields + val settingType = fields("type").convertTo[WorkspaceSettingType] + val configuration = settingType match { + case GcpBucketLifecycle => fields.get("config").map(_.convertTo[GcpBucketLifecycleConfig]) + case _ => throw DeserializationException(s"unexpected setting type $settingType") + } + WorkspaceSettings(settingType, configuration) + } + } + implicit val WorkspaceNameFormat: RootJsonFormat[WorkspaceName] = jsonFormat2(WorkspaceName) implicit val EntityFormat: RootJsonFormat[Entity] = jsonFormat3(Entity) From a4536df48a72f70ea27267df525bb78e80bf1d75 Mon Sep 17 00:00:00 2001 From: Marcus Talbott Date: Mon, 29 Jul 2024 17:14:09 -0400 Subject: [PATCH 03/25] swagger --- core/src/main/resources/swagger/api-docs.yaml | 128 ++++++++++++++++++ .../dsde/rawls/model/WorkspaceModel.scala | 2 +- 2 files changed, 129 insertions(+), 1 deletion(-) diff --git a/core/src/main/resources/swagger/api-docs.yaml b/core/src/main/resources/swagger/api-docs.yaml index b25a19ef14..073a249ce1 100644 --- a/core/src/main/resources/swagger/api-docs.yaml +++ b/core/src/main/resources/swagger/api-docs.yaml @@ -4423,6 +4423,78 @@ paths: $ref: '#/components/schemas/ErrorReport' 500: $ref: '#/components/responses/RawlsInternalError' + /api/workspaces/v2/{workspaceNamespace}/{workspaceName}/settings: + get: + tags: + - workspaces_v2 + summary: Get workspace settings + description: Get the settings for a workspace + operationId: getWorkspaceSettings + parameters: + - $ref: '#/components/parameters/workspaceNamespacePathParam' + - $ref: '#/components/parameters/workspaceNamePathParam' + responses: + 200: + description: Success + content: + 'application/json': + schema: + type: array + items: + $ref: '#/components/schemas/WorkspaceSettings' + 403: + description: User does not have access to requested workspace + content: + 'application/json': + schema: + $ref: '#/components/schemas/ErrorReport' + 404: + description: Workspace not found + content: + 'application/json': + schema: + $ref: '#/components/schemas/ErrorReport' + 500: + $ref: '#/components/responses/RawlsInternalError' + put: + tags: + - workspaces_v2 + summary: Overwrite workspace settings + description: Overwrite the settings for a workspace + operationId: overwriteWorkspaceSettings + parameters: + - $ref: '#/components/parameters/workspaceNamespacePathParam' + - $ref: '#/components/parameters/workspaceNamePathParam' + requestBody: + description: New settings for the workspace + content: + 'application/json': + schema: + type: array + items: + $ref: '#/components/schemas/WorkspaceSettings' + required: true + responses: + 200: + description: Success + content: + 'application/json': + schema: + $ref: '#/components/schemas/WorkspaceSettings' + 403: + description: User must be an owner of the workspace to overwrite settings + content: + 'application/json': + schema: + $ref: '#/components/schemas/ErrorReport' + 404: + description: Workspace not found + content: + 'application/json': + schema: + $ref: '#/components/schemas/ErrorReport' + 500: + $ref: '#/components/responses/RawlsInternalError' components: schemas: BillingAccount: @@ -5818,6 +5890,62 @@ components: - Deleting - DeleteFailed description: "" + WorkspaceSettings: + required: + - type + - config + type: object + properties: + 'type': + type: string + description: The type of the workspace setting + enum: + - GcpBucketLifecycle + config: + description: The configuration of the workspace setting + oneOf: + - $ref: '#/components/schemas/WorkspaceSettingGcpBucketLifecycleConfig' + WorkspaceSettingGcpBucketLifecycleConfig: + required: + - rules + type: object + properties: + rules: + type: array + description: The lifecycle rules for the workspace bucket + items: + $ref: '#/components/schemas/GcpBucketLifecycleRule' + GcpBucketLifecycleRule: + required: + - action + - conditions + type: object + properties: + action: + $ref: '#/components/schemas/GcpBucketLifecycleAction' + conditions: + $ref: '#/components/schemas/GcpBucketLifecycleCondition' + GcpBucketLifecycleAction: + required: + - type + type: object + properties: + "type": + type: string + description: The type of the lifecycle action + enum: + - Delete + GcpBucketLifecycleCondition: + type: object + properties: + age: + type: integer + description: The age of the object in days + matchesPrefix: + type: array + description: Object name prefixes that this rule applies to + items: + type: string WorkspaceSubmissionStats: required: - runningSubmissionsCount diff --git a/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala b/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala index 7e1afdc235..15a122dce0 100644 --- a/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala +++ b/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala @@ -572,7 +572,7 @@ object WorkspaceSettingTypes { } def withName(name: String): WorkspaceSettingType = name.toLowerCase match { - case "gcp_bucket_lifecycle" => GcpBucketLifecycle + case "gcpbucketlifecycle" => GcpBucketLifecycle case _ => throw new RawlsException(s"invalid WorkspaceSetting [$name]") } From 01e0d272ac8552b65011715bab28e4a0dc42850c Mon Sep 17 00:00:00 2001 From: Marcus Talbott Date: Thu, 1 Aug 2024 17:10:19 -0400 Subject: [PATCH 04/25] only return applied settings --- .../dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala index b2987d00e8..c51e78095b 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala @@ -92,6 +92,6 @@ trait WorkspaceSettingComponent { filter(record => record.workspaceId === workspaceId && record.status === status.toString).delete def listAllForWorkspace(workspaceId: UUID): ReadAction[List[WorkspaceSettings]] = - filter(_.workspaceId === workspaceId).result.map(_.map(WorkspaceSettingRecord.toWorkspaceSettings).toList) + filter(rec => rec.workspaceId === workspaceId && rec.status === WorkspaceSettingRecord.SettingStatus.Applied.toString).result.map(_.map(WorkspaceSettingRecord.toWorkspaceSettings).toList) } } From e541f612da2c5d3505b2ff09251e581951d6062e Mon Sep 17 00:00:00 2001 From: Marcus Talbott Date: Tue, 6 Aug 2024 14:00:26 -0400 Subject: [PATCH 05/25] rename WorkspaceSetting, handle empty settings, simplify rollback --- .../slick/WorkspaceSettingComponent.scala | 16 ++++-- .../webservice/WorkspaceApiServiceV2.scala | 2 +- .../rawls/workspace/WorkspaceRepository.scala | 17 +++--- .../rawls/workspace/WorkspaceService.scala | 56 +++++++++++++------ .../dataaccess/MockGoogleServicesDAO.scala | 3 + .../dsde/rawls/model/WorkspaceModel.scala | 23 ++++---- 6 files changed, 76 insertions(+), 41 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala index c51e78095b..ac651a4ec1 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala @@ -23,7 +23,7 @@ object WorkspaceSettingRecord { val Applied: Value = Value("Applied") } - def toWorkspaceSettingRecord(workspaceId: UUID, workspaceSettings: WorkspaceSettings): WorkspaceSettingRecord = { + def toWorkspaceSettingRecord(workspaceId: UUID, workspaceSettings: WorkspaceSetting): WorkspaceSettingRecord = { import spray.json._ import DefaultJsonProtocol._ import WorkspaceJsonSupport._ @@ -39,7 +39,7 @@ object WorkspaceSettingRecord { ) } - def toWorkspaceSettings(workspaceSettingRecord: WorkspaceSettingRecord): WorkspaceSettings = { + def toWorkspaceSettings(workspaceSettingRecord: WorkspaceSettingRecord): WorkspaceSetting = { import spray.json._ val settingType = WorkspaceSettingTypes.withName(workspaceSettingRecord.`type`) @@ -48,7 +48,7 @@ object WorkspaceSettingRecord { case WorkspaceSettingTypes.GcpBucketLifecycle => configuration.parseJson.convertTo[GcpBucketLifecycleConfig] } } - WorkspaceSettings(settingType, settingConfig) + WorkspaceSetting(settingType, settingConfig) } } @@ -70,7 +70,7 @@ trait WorkspaceSettingComponent { } object workspaceSettingQuery extends TableQuery(new WorkspaceSettingTable(_)) { - def saveAll(workspaceId: UUID, workspaceSettings: List[WorkspaceSettings]): ReadWriteAction[List[WorkspaceSettings]] = { + def saveAll(workspaceId: UUID, workspaceSettings: List[WorkspaceSetting]): ReadWriteAction[List[WorkspaceSetting]] = { val records = workspaceSettings.map(WorkspaceSettingRecord.toWorkspaceSettingRecord(workspaceId, _)) (workspaceSettingQuery ++= records).map(_ => workspaceSettings) } @@ -91,7 +91,13 @@ trait WorkspaceSettingComponent { ): ReadWriteAction[Int] = filter(record => record.workspaceId === workspaceId && record.status === status.toString).delete - def listAllForWorkspace(workspaceId: UUID): ReadAction[List[WorkspaceSettings]] = + def deleteSettingsForWorkspaceByTypeAndStatus(workspaceId: UUID, + settingType: List[WorkspaceSettingType], + status: WorkspaceSettingRecord.SettingStatus.SettingStatus + ): ReadWriteAction[Int] = + filter(record => record.workspaceId === workspaceId && record.`type` === settingType.toString && record.status === status.toString).delete + + def listAllForWorkspace(workspaceId: UUID): ReadAction[List[WorkspaceSetting]] = filter(rec => rec.workspaceId === workspaceId && rec.status === WorkspaceSettingRecord.SettingStatus.Applied.toString).result.map(_.map(WorkspaceSettingRecord.toWorkspaceSettings).toList) } } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/WorkspaceApiServiceV2.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/WorkspaceApiServiceV2.scala index c89305614f..7f49d10c87 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/WorkspaceApiServiceV2.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/WorkspaceApiServiceV2.scala @@ -97,7 +97,7 @@ trait WorkspaceApiServiceV2 extends UserInfoDirectives { } } ~ put { - entity(as[List[WorkspaceSettings]]) { settings => + entity(as[List[WorkspaceSetting]]) { settings => complete { workspaceServiceConstructor(ctx) .setWorkspaceSettings(workspaceName, settings) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala index cd51723409..5d41e68b45 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala @@ -5,8 +5,9 @@ import org.broadinstitute.dsde.rawls.RawlsExceptionWithErrorReport import org.broadinstitute.dsde.rawls.dataaccess.SlickDataSource import org.broadinstitute.dsde.rawls.dataaccess.slick.WorkspaceSettingRecord import org.broadinstitute.dsde.rawls.model.Attributable.AttributeMap -import org.broadinstitute.dsde.rawls.model.{ErrorReport, RawlsRequestContext, Workspace, WorkspaceAttributeSpecs, WorkspaceName, WorkspaceSettings, WorkspaceState} +import org.broadinstitute.dsde.rawls.model.{ErrorReport, RawlsRequestContext, Workspace, WorkspaceAttributeSpecs, WorkspaceName, WorkspaceSetting, WorkspaceState} import org.broadinstitute.dsde.rawls.model.WorkspaceState.WorkspaceState +import org.broadinstitute.dsde.rawls.model.WorkspaceSettingTypes.WorkspaceSettingType import org.broadinstitute.dsde.rawls.util.TracingUtils.traceDBIOWithParent import org.joda.time.DateTime @@ -93,26 +94,26 @@ class WorkspaceRepository(dataSource: SlickDataSource) { } yield newWorkspace } - def getWorkspaceSettings(workspaceId: UUID): Future[List[WorkspaceSettings]] = + def getWorkspaceSettings(workspaceId: UUID): Future[List[WorkspaceSetting]] = dataSource.inTransaction { access => access.workspaceSettingQuery.listAllForWorkspace(workspaceId) } - def overwriteWorkspaceSettings(workspaceId: UUID, workspaceSettings: List[WorkspaceSettings]): Future[List[WorkspaceSettings]] = + def createWorkspaceSettingsRecords(workspaceId: UUID, workspaceSettings: List[WorkspaceSetting]): Future[List[WorkspaceSetting]] = dataSource.inTransaction { access => access.workspaceSettingQuery.saveAll(workspaceId, workspaceSettings) } - def markWorkspaceSettingsApplied(workspaceId: UUID, workspaceSettings: List[WorkspaceSettings])(implicit ex: ExecutionContext): Future[Int] = + def markWorkspaceSettingApplied(workspaceId: UUID, workspaceSetting: WorkspaceSetting)(implicit ec: ExecutionContext): Future[Int] = dataSource.inTransaction { access => for { - _ <- access.workspaceSettingQuery.deleteSettingsForWorkspace(workspaceId, WorkspaceSettingRecord.SettingStatus.Applied) - res <- access.workspaceSettingQuery.updateStatuses(workspaceId, workspaceSettings.map(_.`type`), WorkspaceSettingRecord.SettingStatus.Applied) + _ <- access.workspaceSettingQuery.deleteSettingsForWorkspaceByTypeAndStatus(workspaceId, List(workspaceSetting.`type`), WorkspaceSettingRecord.SettingStatus.Applied) + res <- access.workspaceSettingQuery.updateStatuses(workspaceId, List(workspaceSetting.`type`), WorkspaceSettingRecord.SettingStatus.Applied) } yield res } - def removeUnappliedSettings(workspaceId: UUID): Future[Int] = + def removeUnappliedSetting(workspaceId: UUID, workspaceSetting: WorkspaceSetting): Future[Int] = dataSource.inTransaction { access => - access.workspaceSettingQuery.deleteSettingsForWorkspace(workspaceId, WorkspaceSettingRecord.SettingStatus.Applying) + access.workspaceSettingQuery.deleteSettingsForWorkspaceByTypeAndStatus(workspaceId, List(workspaceSetting.`type`), WorkspaceSettingRecord.SettingStatus.Applying) } } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala index cee1eb8ffe..405c1937fa 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala @@ -28,6 +28,7 @@ import org.broadinstitute.dsde.rawls.model.WorkspaceJsonSupport._ import org.broadinstitute.dsde.rawls.model.WorkspaceState.WorkspaceState import org.broadinstitute.dsde.rawls.model.WorkspaceType.WorkspaceType import org.broadinstitute.dsde.rawls.model.WorkspaceVersions.WorkspaceVersion +import org.broadinstitute.dsde.rawls.model.WorkspaceSettingTypes.WorkspaceSettingType import org.broadinstitute.dsde.rawls.model._ import org.broadinstitute.dsde.rawls.monitor.migration.MigrationUtils.Implicits.monadThrowDBIOAction import org.broadinstitute.dsde.rawls.resourcebuffer.ResourceBufferService @@ -50,7 +51,7 @@ import cats.effect.unsafe.implicits.global import com.google.cloud.storage.BucketInfo.LifecycleRule import com.google.cloud.storage.BucketInfo.LifecycleRule.{LifecycleCondition, LifecycleAction} import org.broadinstitute.dsde.rawls.billing.BillingRepository -import org.broadinstitute.dsde.rawls.model.WorkspaceSettingConfig.GcpBucketLifecycleConfig +import org.broadinstitute.dsde.rawls.model.WorkspaceSettingConfig._ import java.io.IOException import java.util.UUID @@ -2004,16 +2005,22 @@ class WorkspaceService( } } yield {} - def getWorkspaceSettings(workspaceName: WorkspaceName): Future[List[WorkspaceSettings]] = { + def getWorkspaceSettings(workspaceName: WorkspaceName): Future[List[WorkspaceSetting]] = { getV2WorkspaceContextAndPermissions(workspaceName, SamWorkspaceActions.read).flatMap { workspace => workspaceRepository.getWorkspaceSettings(workspace.workspaceIdAsUUID) } } - def setWorkspaceSettings(workspaceName: WorkspaceName, workspaceSettings: List[WorkspaceSettings]): Future[List[WorkspaceSettings]] = { - def applySettings(workspace: Workspace, settings: WorkspaceSettings): Future[Unit] = { - settings match { - case WorkspaceSettings(WorkspaceSettingTypes.GcpBucketLifecycle, Some(GcpBucketLifecycleConfig(rules))) => { + def setWorkspaceSettings(workspaceName: WorkspaceName, workspaceSettings: List[WorkspaceSetting]): Future[List[WorkspaceSetting]] = { + /** Apply a setting to a workspace. If the setting is successfully applied, update the database + * and return None. If the setting fails to apply, remove the failed setting from the database + * and return the setting type with an error report. If the setting is not supported, throw an + * exception. We make more trips to the database here than necessary, but we support a small + * number of setting types and it's easier to reason about this way. + */ + def applySetting(workspace: Workspace, setting: WorkspaceSetting): Future[Option[(WorkspaceSettingType, ErrorReport)]] = { + (setting match { + case WorkspaceSetting(WorkspaceSettingTypes.GcpBucketLifecycle, Some(GcpBucketLifecycleConfig(rules))) => { val googleRules = rules.map { rule => val conditionBuilder = LifecycleCondition.newBuilder().setMatchesPrefix(rule.conditions.matchesPrefix.toList.asJava) rule.conditions.age.map(age => conditionBuilder.setAge(age)) @@ -2026,22 +2033,37 @@ class WorkspaceService( new LifecycleRule(action, conditionBuilder.build()) } - gcsDAO.setBucketLifecycle(workspace.bucketName, googleRules) + for { + _ <- gcsDAO.setBucketLifecycle(workspace.bucketName, googleRules) + _ <- workspaceRepository.markWorkspaceSettingApplied(workspace.workspaceIdAsUUID, setting) + } yield None } case _ => throw new RawlsException("unsupported workspace setting") + }).recoverWith { case e => // todo: filter out workspace repository exceptions? + workspaceRepository.removeUnappliedSetting(workspace.workspaceIdAsUUID, setting).map(_ => Some((setting.`type`, ErrorReport(StatusCodes.InternalServerError, e)))) } } - getV2WorkspaceContextAndPermissions(workspaceName, SamWorkspaceActions.own).flatMap { workspace => - (for { - _ <- workspaceRepository.overwriteWorkspaceSettings(workspace.workspaceIdAsUUID, workspaceSettings) - _ <- workspaceSettings.traverse(s => applySettings(workspace, s)) // todo: maybe this should catch exceptions and keep track of which settings went through and which didn't. then it can roll back anything that failed. - _ <- workspaceRepository.markWorkspaceSettingsApplied(workspace.workspaceIdAsUUID, workspaceSettings) - } yield { - workspaceSettings - }).recoverWith { case e => - workspaceRepository.removeUnappliedSettings(workspace.workspaceIdAsUUID).flatMap { _ => - Future.failed(new RawlsException("Failed to apply workspace settings")) + /** Iterate over existing settings. If an existing setting type is included in the + * requestedSettings, use the requested setting. If it isn't, use the setting type's default + * config to restore the workspace to the default state. */ + def computeNewSettings(workspace: Workspace, requestedSettings: List[WorkspaceSetting], existingSettings: List[WorkspaceSetting]): List[WorkspaceSetting] = { + val requestedSettingsMap = requestedSettings.map(s => s.`type` -> s.config).toMap + existingSettings.map { case WorkspaceSetting(settingType, _) => + WorkspaceSetting(settingType, requestedSettingsMap.getOrElse(settingType, settingType.defaultConfig())) + } + } + + for { + workspace <- getV2WorkspaceContextAndPermissions(workspaceName, SamWorkspaceActions.own) + currentSettings <- workspaceRepository.getWorkspaceSettings(workspace.workspaceIdAsUUID) + newSettings = computeNewSettings(workspace, workspaceSettings, currentSettings) + _ <- workspaceRepository.createWorkspaceSettingsRecords(workspace.workspaceIdAsUUID, workspaceSettings) + applyFailures <- workspaceSettings.traverse(s => applySetting(workspace, s)) // todo: what do we do with the failed setting errors? return them? + } yield { + workspaceSettings.filterNot { s => + applyFailures.flatten.exists { + case (failedSettingType, _) => failedSettingType == s.`type` } } } diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/MockGoogleServicesDAO.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/MockGoogleServicesDAO.scala index ad6b4f5448..6fba67cf9c 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/MockGoogleServicesDAO.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/MockGoogleServicesDAO.scala @@ -7,6 +7,7 @@ import com.google.api.services.directory.model.Group import com.google.api.services.cloudbilling.model.ProjectBillingInfo import com.google.api.services.cloudresourcemanager.model.Project import com.google.api.services.storage.model.{Bucket, BucketAccessControl, StorageObject} +import com.google.cloud.storage.BucketInfo import io.opencensus.trace.Span import org.broadinstitute.dsde.rawls.RawlsException import org.broadinstitute.dsde.rawls.dataaccess.slick.RawlsBillingProjectOperationRecord @@ -123,6 +124,8 @@ class MockGoogleServicesDAO(groupsPrefix: String, override def deleteBucket(bucketName: String) = Future.successful(true) + override def setBucketLifecycle(bucketName: String, lifecycle: List[BucketInfo.LifecycleRule]): Future[Unit] = ??? + override def getBucket(bucketName: String, userProject: Option[GoogleProjectId])(implicit executionContext: ExecutionContext ): Future[Either[String, Bucket]] = Future.successful(Right(new Bucket)) diff --git a/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala b/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala index 15a122dce0..5c1d6a16ad 100644 --- a/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala +++ b/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala @@ -563,12 +563,13 @@ object WorkspaceState { case object DeleteFailed extends WorkspaceState } -case class WorkspaceSettings(`type`: WorkspaceSettingType, config: Option[WorkspaceSettingConfig]) +case class WorkspaceSetting(`type`: WorkspaceSettingType, config: Option[WorkspaceSettingConfig]) object WorkspaceSettingTypes { sealed trait WorkspaceSettingType extends RawlsEnumeration[WorkspaceSettingType] { - override def toString: String = getClass.getSimpleName.stripSuffix("$") - override def withName(name: String): WorkspaceSettingType = WorkspaceSettingTypes.withName(name) + override def toString: String = getClass.getSimpleName.stripSuffix("$") + override def withName(name: String): WorkspaceSettingType = WorkspaceSettingTypes.withName(name) + def defaultConfig(): Option[WorkspaceSettingConfig] } def withName(name: String): WorkspaceSettingType = name.toLowerCase match { @@ -576,7 +577,9 @@ object WorkspaceSettingTypes { case _ => throw new RawlsException(s"invalid WorkspaceSetting [$name]") } - case object GcpBucketLifecycle extends WorkspaceSettingType + case object GcpBucketLifecycle extends WorkspaceSettingType { + override def defaultConfig(): Option[WorkspaceSettingConfig] = Some(GcpBucketLifecycleConfig(List.empty)) + } } sealed trait WorkspaceSettingConfig @@ -1206,30 +1209,30 @@ class WorkspaceJsonSupport extends JsonSupport { } } - implicit object WorkspaceSettingsConfigurationFormat extends RootJsonFormat[WorkspaceSettingConfig] { + implicit object WorkspaceSettingConfigFormat extends RootJsonFormat[WorkspaceSettingConfig] { def write(obj: WorkspaceSettingConfig): JsValue = obj match { case config: GcpBucketLifecycleConfig => config.toJson } def read(json: JsValue): WorkspaceSettingConfig = { - throw DeserializationException("WorkspaceSettingsConfiguration cannot be read directly") + throw DeserializationException("WorkspaceSettingConfig cannot be read directly") } } - implicit object WorkspaceSettingsFormat extends RootJsonFormat[WorkspaceSettings] { - def write(ws: WorkspaceSettings): JsValue = JsObject( + implicit object WorkspaceSettingFormat extends RootJsonFormat[WorkspaceSetting] { + def write(ws: WorkspaceSetting): JsValue = JsObject( "type" -> ws.`type`.toJson, "config" -> ws.config.toJson ) - def read(json: JsValue): WorkspaceSettings = { + def read(json: JsValue): WorkspaceSetting = { val fields = json.asJsObject.fields val settingType = fields("type").convertTo[WorkspaceSettingType] val configuration = settingType match { case GcpBucketLifecycle => fields.get("config").map(_.convertTo[GcpBucketLifecycleConfig]) case _ => throw DeserializationException(s"unexpected setting type $settingType") } - WorkspaceSettings(settingType, configuration) + WorkspaceSetting(settingType, configuration) } } From 3dc24ec172af8debafefe44a8acaac4d74c85c27 Mon Sep 17 00:00:00 2001 From: Marcus Talbott Date: Tue, 6 Aug 2024 17:33:56 -0400 Subject: [PATCH 06/25] return errors alongside successes, soft delete settings --- .../20240723_workspace_settings.xml | 4 -- .../slick/WorkspaceSettingComponent.scala | 32 +++++++++----- .../rawls/workspace/WorkspaceRepository.scala | 18 +++++--- .../rawls/workspace/WorkspaceService.scala | 43 +++++++++++-------- .../dsde/rawls/model/WorkspaceModel.scala | 4 ++ 5 files changed, 63 insertions(+), 38 deletions(-) diff --git a/core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changesets/20240723_workspace_settings.xml b/core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changesets/20240723_workspace_settings.xml index c89a892b77..770a84013b 100644 --- a/core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changesets/20240723_workspace_settings.xml +++ b/core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changesets/20240723_workspace_settings.xml @@ -26,9 +26,5 @@ - diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala index ac651a4ec1..89fde070b3 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala @@ -19,8 +19,9 @@ case class WorkspaceSettingRecord(`type`: String, object WorkspaceSettingRecord { object SettingStatus extends SlickEnum { type SettingStatus = Value - val Applying: Value = Value("Applying") + val Pending: Value = Value("Pending") val Applied: Value = Value("Applied") + val Deleted: Value = Value("Deleted") } def toWorkspaceSettingRecord(workspaceId: UUID, workspaceSettings: WorkspaceSetting): WorkspaceSettingRecord = { @@ -33,7 +34,7 @@ object WorkspaceSettingRecord { WorkspaceSettingRecord(workspaceSettings.`type`.toString, workspaceId, Option(configString), - WorkspaceSettingRecord.SettingStatus.Applying.toString, + WorkspaceSettingRecord.SettingStatus.Pending.toString, currentTime, currentTime ) @@ -70,7 +71,9 @@ trait WorkspaceSettingComponent { } object workspaceSettingQuery extends TableQuery(new WorkspaceSettingTable(_)) { - def saveAll(workspaceId: UUID, workspaceSettings: List[WorkspaceSetting]): ReadWriteAction[List[WorkspaceSetting]] = { + def saveAll(workspaceId: UUID, + workspaceSettings: List[WorkspaceSetting] + ): ReadWriteAction[List[WorkspaceSetting]] = { val records = workspaceSettings.map(WorkspaceSettingRecord.toWorkspaceSettingRecord(workspaceId, _)) (workspaceSettingQuery ++= records).map(_ => workspaceSettings) } @@ -89,15 +92,22 @@ trait WorkspaceSettingComponent { def deleteSettingsForWorkspace(workspaceId: UUID, status: WorkspaceSettingRecord.SettingStatus.SettingStatus ): ReadWriteAction[Int] = - filter(record => record.workspaceId === workspaceId && record.status === status.toString).delete + filter(record => record.workspaceId === workspaceId && record.status === status.toString) + .map(_.status) + .update(WorkspaceSettingRecord.SettingStatus.Deleted.toString) - def deleteSettingsForWorkspaceByTypeAndStatus(workspaceId: UUID, - settingType: List[WorkspaceSettingType], - status: WorkspaceSettingRecord.SettingStatus.SettingStatus + def deleteSettingTypeForWorkspaceByStatus(workspaceId: UUID, + settingType: WorkspaceSettingType, + status: WorkspaceSettingRecord.SettingStatus.SettingStatus ): ReadWriteAction[Int] = - filter(record => record.workspaceId === workspaceId && record.`type` === settingType.toString && record.status === status.toString).delete - - def listAllForWorkspace(workspaceId: UUID): ReadAction[List[WorkspaceSetting]] = - filter(rec => rec.workspaceId === workspaceId && rec.status === WorkspaceSettingRecord.SettingStatus.Applied.toString).result.map(_.map(WorkspaceSettingRecord.toWorkspaceSettings).toList) + filter(record => + record.workspaceId === workspaceId && record.`type` === settingType.toString && record.status === status.toString + ).map(_.status).update(WorkspaceSettingRecord.SettingStatus.Deleted.toString) + + def listSettingsForWorkspaceByStatus(workspaceId: UUID, + status: WorkspaceSettingRecord.SettingStatus.SettingStatus + ): ReadAction[List[WorkspaceSetting]] = + filter(rec => rec.workspaceId === workspaceId && rec.status === status.toString).result + .map(_.map(WorkspaceSettingRecord.toWorkspaceSettings).toList) } } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala index 5d41e68b45..85ffe2d9e1 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala @@ -96,24 +96,30 @@ class WorkspaceRepository(dataSource: SlickDataSource) { def getWorkspaceSettings(workspaceId: UUID): Future[List[WorkspaceSetting]] = dataSource.inTransaction { access => - access.workspaceSettingQuery.listAllForWorkspace(workspaceId) + access.workspaceSettingQuery.listSettingsForWorkspaceByStatus(workspaceId, WorkspaceSettingRecord.SettingStatus.Applied) } - def createWorkspaceSettingsRecords(workspaceId: UUID, workspaceSettings: List[WorkspaceSetting]): Future[List[WorkspaceSetting]] = + def createWorkspaceSettingsRecords(workspaceId: UUID, workspaceSettings: List[WorkspaceSetting])(implicit ec: ExecutionContext): Future[List[WorkspaceSetting]] = dataSource.inTransaction { access => - access.workspaceSettingQuery.saveAll(workspaceId, workspaceSettings) + for { + pendingSettingsForWorkspace <- access.workspaceSettingQuery.listSettingsForWorkspaceByStatus(workspaceId, WorkspaceSettingRecord.SettingStatus.Pending) + _ = if (pendingSettingsForWorkspace.nonEmpty) { + throw new RawlsExceptionWithErrorReport(ErrorReport(StatusCodes.Conflict, s"Workspace $workspaceId already has pending settings")) + } + _ <- access.workspaceSettingQuery.saveAll(workspaceId, workspaceSettings) + } yield workspaceSettings } def markWorkspaceSettingApplied(workspaceId: UUID, workspaceSetting: WorkspaceSetting)(implicit ec: ExecutionContext): Future[Int] = dataSource.inTransaction { access => for { - _ <- access.workspaceSettingQuery.deleteSettingsForWorkspaceByTypeAndStatus(workspaceId, List(workspaceSetting.`type`), WorkspaceSettingRecord.SettingStatus.Applied) + _ <- access.workspaceSettingQuery.deleteSettingTypeForWorkspaceByStatus(workspaceId, workspaceSetting.`type`, WorkspaceSettingRecord.SettingStatus.Applied) res <- access.workspaceSettingQuery.updateStatuses(workspaceId, List(workspaceSetting.`type`), WorkspaceSettingRecord.SettingStatus.Applied) } yield res } - def removeUnappliedSetting(workspaceId: UUID, workspaceSetting: WorkspaceSetting): Future[Int] = + def removePendingSetting(workspaceId: UUID, workspaceSetting: WorkspaceSetting): Future[Int] = dataSource.inTransaction { access => - access.workspaceSettingQuery.deleteSettingsForWorkspaceByTypeAndStatus(workspaceId, List(workspaceSetting.`type`), WorkspaceSettingRecord.SettingStatus.Applying) + access.workspaceSettingQuery.deleteSettingTypeForWorkspaceByStatus(workspaceId, workspaceSetting.`type`, WorkspaceSettingRecord.SettingStatus.Pending) } } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala index 405c1937fa..dd9bb49da9 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala @@ -49,7 +49,7 @@ import spray.json._ import org.broadinstitute.dsde.rawls.metrics.MetricsHelper import cats.effect.unsafe.implicits.global import com.google.cloud.storage.BucketInfo.LifecycleRule -import com.google.cloud.storage.BucketInfo.LifecycleRule.{LifecycleCondition, LifecycleAction} +import com.google.cloud.storage.BucketInfo.LifecycleRule.{LifecycleAction, LifecycleCondition} import org.broadinstitute.dsde.rawls.billing.BillingRepository import org.broadinstitute.dsde.rawls.model.WorkspaceSettingConfig._ @@ -2005,29 +2005,34 @@ class WorkspaceService( } } yield {} - def getWorkspaceSettings(workspaceName: WorkspaceName): Future[List[WorkspaceSetting]] = { + def getWorkspaceSettings(workspaceName: WorkspaceName): Future[List[WorkspaceSetting]] = getV2WorkspaceContextAndPermissions(workspaceName, SamWorkspaceActions.read).flatMap { workspace => workspaceRepository.getWorkspaceSettings(workspace.workspaceIdAsUUID) } - } - def setWorkspaceSettings(workspaceName: WorkspaceName, workspaceSettings: List[WorkspaceSetting]): Future[List[WorkspaceSetting]] = { + def setWorkspaceSettings(workspaceName: WorkspaceName, + workspaceSettings: List[WorkspaceSetting] + ): Future[WorkspaceSettingResponse] = { + /** Apply a setting to a workspace. If the setting is successfully applied, update the database * and return None. If the setting fails to apply, remove the failed setting from the database * and return the setting type with an error report. If the setting is not supported, throw an * exception. We make more trips to the database here than necessary, but we support a small * number of setting types and it's easier to reason about this way. */ - def applySetting(workspace: Workspace, setting: WorkspaceSetting): Future[Option[(WorkspaceSettingType, ErrorReport)]] = { + def applySetting(workspace: Workspace, + setting: WorkspaceSetting + ): Future[Option[(WorkspaceSettingType, ErrorReport)]] = (setting match { - case WorkspaceSetting(WorkspaceSettingTypes.GcpBucketLifecycle, Some(GcpBucketLifecycleConfig(rules))) => { + case WorkspaceSetting(WorkspaceSettingTypes.GcpBucketLifecycle, Some(GcpBucketLifecycleConfig(rules))) => val googleRules = rules.map { rule => - val conditionBuilder = LifecycleCondition.newBuilder().setMatchesPrefix(rule.conditions.matchesPrefix.toList.asJava) + val conditionBuilder = + LifecycleCondition.newBuilder().setMatchesPrefix(rule.conditions.matchesPrefix.toList.asJava) rule.conditions.age.map(age => conditionBuilder.setAge(age)) val action = rule.action.`type` match { case actionType if actionType.equals("Delete") => LifecycleAction.newDeleteAction() - case _ => throw new RawlsException("unsupported lifecycle action") + case _ => throw new RawlsException("unsupported lifecycle action") } new LifecycleRule(action, conditionBuilder.build()) @@ -2037,17 +2042,20 @@ class WorkspaceService( _ <- gcsDAO.setBucketLifecycle(workspace.bucketName, googleRules) _ <- workspaceRepository.markWorkspaceSettingApplied(workspace.workspaceIdAsUUID, setting) } yield None - } case _ => throw new RawlsException("unsupported workspace setting") - }).recoverWith { case e => // todo: filter out workspace repository exceptions? - workspaceRepository.removeUnappliedSetting(workspace.workspaceIdAsUUID, setting).map(_ => Some((setting.`type`, ErrorReport(StatusCodes.InternalServerError, e)))) + }).recoverWith { case e => + workspaceRepository + .removePendingSetting(workspace.workspaceIdAsUUID, setting) + .map(_ => Some((setting.`type`, ErrorReport(StatusCodes.InternalServerError, e.getMessage)))) } - } /** Iterate over existing settings. If an existing setting type is included in the * requestedSettings, use the requested setting. If it isn't, use the setting type's default * config to restore the workspace to the default state. */ - def computeNewSettings(workspace: Workspace, requestedSettings: List[WorkspaceSetting], existingSettings: List[WorkspaceSetting]): List[WorkspaceSetting] = { + def computeNewSettings(workspace: Workspace, + requestedSettings: List[WorkspaceSetting], + existingSettings: List[WorkspaceSetting] + ): List[WorkspaceSetting] = { val requestedSettingsMap = requestedSettings.map(s => s.`type` -> s.config).toMap existingSettings.map { case WorkspaceSetting(settingType, _) => WorkspaceSetting(settingType, requestedSettingsMap.getOrElse(settingType, settingType.defaultConfig())) @@ -2059,13 +2067,14 @@ class WorkspaceService( currentSettings <- workspaceRepository.getWorkspaceSettings(workspace.workspaceIdAsUUID) newSettings = computeNewSettings(workspace, workspaceSettings, currentSettings) _ <- workspaceRepository.createWorkspaceSettingsRecords(workspace.workspaceIdAsUUID, workspaceSettings) - applyFailures <- workspaceSettings.traverse(s => applySetting(workspace, s)) // todo: what do we do with the failed setting errors? return them? + applyFailures <- workspaceSettings.traverse(s => applySetting(workspace, s)) } yield { - workspaceSettings.filterNot { s => - applyFailures.flatten.exists { - case (failedSettingType, _) => failedSettingType == s.`type` + val successes = workspaceSettings.filterNot { s => + applyFailures.flatten.exists { case (failedSettingType, _) => + failedSettingType == s.`type` } } + WorkspaceSettingResponse(successes, applyFailures.flatten.toMap) } } diff --git a/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala b/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala index 5c1d6a16ad..64f81c649f 100644 --- a/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala +++ b/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala @@ -591,6 +591,8 @@ object WorkspaceSettingConfig { case class GcpLifecycleCondition(matchesPrefix: Set[String], age: Option[Int]) } +case class WorkspaceSettingResponse(successes: List[WorkspaceSetting], failures: Map[WorkspaceSettingType, ErrorReport]) + sealed trait MethodRepoMethod { def methodUri: String @@ -1478,6 +1480,8 @@ class WorkspaceJsonSupport extends JsonSupport { ) ) + implicit val WorkspaceSettingResponseFormat: RootJsonFormat[WorkspaceSettingResponse] = jsonFormat2(WorkspaceSettingResponse) + implicit val ApplicationVersionFormat: RootJsonFormat[ApplicationVersion] = jsonFormat3(ApplicationVersion) } From bbde9b5249af4535272e8b0ade9848484126e732 Mon Sep 17 00:00:00 2001 From: Marcus Talbott Date: Wed, 7 Aug 2024 14:08:37 -0400 Subject: [PATCH 07/25] bug fixes, require config, add validation --- .../slick/WorkspaceSettingComponent.scala | 27 +++----- .../rawls/workspace/WorkspaceRepository.scala | 2 +- .../rawls/workspace/WorkspaceService.scala | 61 +++++++++++++------ .../dsde/rawls/model/WorkspaceModel.scala | 8 +-- 4 files changed, 59 insertions(+), 39 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala index 89fde070b3..05aee176d7 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala @@ -10,7 +10,7 @@ import java.util.{Date, UUID} case class WorkspaceSettingRecord(`type`: String, workspaceId: UUID, - config: Option[String], + config: String, status: String, createdTime: Timestamp, lastUpdated: Timestamp @@ -33,7 +33,7 @@ object WorkspaceSettingRecord { val configString = workspaceSettings.config.toJson.compactPrint WorkspaceSettingRecord(workspaceSettings.`type`.toString, workspaceId, - Option(configString), + configString, WorkspaceSettingRecord.SettingStatus.Pending.toString, currentTime, currentTime @@ -44,11 +44,10 @@ object WorkspaceSettingRecord { import spray.json._ val settingType = WorkspaceSettingTypes.withName(workspaceSettingRecord.`type`) - val settingConfig = workspaceSettingRecord.config.map { configuration => - settingType match { - case WorkspaceSettingTypes.GcpBucketLifecycle => configuration.parseJson.convertTo[GcpBucketLifecycleConfig] - } + val settingConfig = settingType match { + case WorkspaceSettingTypes.GcpBucketLifecycle => workspaceSettingRecord.config.parseJson.convertTo[GcpBucketLifecycleConfig] } + WorkspaceSetting(settingType, settingConfig) } } @@ -60,7 +59,7 @@ trait WorkspaceSettingComponent { class WorkspaceSettingTable(tag: Tag) extends Table[WorkspaceSettingRecord](tag, "WORKSPACE_SETTINGS") { def `type` = column[String]("type", O.Length(254)) def workspaceId = column[UUID]("workspace_id") - def config = column[Option[String]]("config") + def config = column[String]("config") def status = column[String]("status", O.Length(254)) def createdTime = column[Timestamp]("created_time") def lastUpdated = column[Timestamp]("last_updated") @@ -80,21 +79,15 @@ trait WorkspaceSettingComponent { def updateStatuses(workspaceId: UUID, workspaceSettingTypes: List[WorkspaceSettingType], - status: WorkspaceSettingRecord.SettingStatus.SettingStatus + currentStatus: WorkspaceSettingRecord.SettingStatus.SettingStatus, + newStatus: WorkspaceSettingRecord.SettingStatus.SettingStatus ): ReadWriteAction[Int] = workspaceSettingQuery .filter(record => - record.workspaceId === workspaceId && record.`type`.inSetBind(workspaceSettingTypes.map(_.toString)) + record.workspaceId === workspaceId && record.`type`.inSetBind(workspaceSettingTypes.map(_.toString)) && record.status === currentStatus.toString ) .map(_.status) - .update(status.toString) - - def deleteSettingsForWorkspace(workspaceId: UUID, - status: WorkspaceSettingRecord.SettingStatus.SettingStatus - ): ReadWriteAction[Int] = - filter(record => record.workspaceId === workspaceId && record.status === status.toString) - .map(_.status) - .update(WorkspaceSettingRecord.SettingStatus.Deleted.toString) + .update(newStatus.toString) def deleteSettingTypeForWorkspaceByStatus(workspaceId: UUID, settingType: WorkspaceSettingType, diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala index 85ffe2d9e1..65ea40fac6 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala @@ -114,7 +114,7 @@ class WorkspaceRepository(dataSource: SlickDataSource) { dataSource.inTransaction { access => for { _ <- access.workspaceSettingQuery.deleteSettingTypeForWorkspaceByStatus(workspaceId, workspaceSetting.`type`, WorkspaceSettingRecord.SettingStatus.Applied) - res <- access.workspaceSettingQuery.updateStatuses(workspaceId, List(workspaceSetting.`type`), WorkspaceSettingRecord.SettingStatus.Applied) + res <- access.workspaceSettingQuery.updateStatuses(workspaceId, List(workspaceSetting.`type`), WorkspaceSettingRecord.SettingStatus.Pending, WorkspaceSettingRecord.SettingStatus.Applied) } yield res } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala index dd9bb49da9..c978e7274d 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala @@ -2013,8 +2013,47 @@ class WorkspaceService( def setWorkspaceSettings(workspaceName: WorkspaceName, workspaceSettings: List[WorkspaceSetting] ): Future[WorkspaceSettingResponse] = { + /** + * Perform basic validation checks on requested settings. + */ + def validateSettings(requestedSettings: List[WorkspaceSetting]): Unit = { + val validationErrors = requestedSettings.flatMap { + case WorkspaceSetting(settingType@WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig(rules)) => + rules.flatMap { rule => + rule.conditions.age.collect { case age if age < 0 => + ErrorReport(s"invalid $settingType configuration: age must be a non-negative integer.") + } + } + } + + if (validationErrors.nonEmpty) { + throw new RawlsExceptionWithErrorReport( + ErrorReport(StatusCodes.BadRequest, "invalid settings requested", validationErrors) + ) + } + } + + /** + * Iterate over existing settings. If an existing setting type is included in the + * requestedSettings, use the requested setting. If it isn't, use the setting type's default + * config to restore the workspace to the default state. + */ + def computeNewSettings(workspace: Workspace, + requestedSettings: List[WorkspaceSetting], + existingSettings: List[WorkspaceSetting] + ): List[WorkspaceSetting] = { + if (existingSettings.isEmpty) { + requestedSettings + } else { + val requestedSettingsMap = requestedSettings.map(s => s.`type` -> s.config).toMap + existingSettings.map { case WorkspaceSetting(settingType, _) => + WorkspaceSetting(settingType, requestedSettingsMap.getOrElse(settingType, settingType.defaultConfig())) + } + } + } - /** Apply a setting to a workspace. If the setting is successfully applied, update the database + /** + * Apply a setting to a workspace. If the setting is successfully applied, update the database * and return None. If the setting fails to apply, remove the failed setting from the database * and return the setting type with an error report. If the setting is not supported, throw an * exception. We make more trips to the database here than necessary, but we support a small @@ -2024,7 +2063,7 @@ class WorkspaceService( setting: WorkspaceSetting ): Future[Option[(WorkspaceSettingType, ErrorReport)]] = (setting match { - case WorkspaceSetting(WorkspaceSettingTypes.GcpBucketLifecycle, Some(GcpBucketLifecycleConfig(rules))) => + case WorkspaceSetting(WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig(rules)) => val googleRules = rules.map { rule => val conditionBuilder = LifecycleCondition.newBuilder().setMatchesPrefix(rule.conditions.matchesPrefix.toList.asJava) @@ -2049,27 +2088,15 @@ class WorkspaceService( .map(_ => Some((setting.`type`, ErrorReport(StatusCodes.InternalServerError, e.getMessage)))) } - /** Iterate over existing settings. If an existing setting type is included in the - * requestedSettings, use the requested setting. If it isn't, use the setting type's default - * config to restore the workspace to the default state. */ - def computeNewSettings(workspace: Workspace, - requestedSettings: List[WorkspaceSetting], - existingSettings: List[WorkspaceSetting] - ): List[WorkspaceSetting] = { - val requestedSettingsMap = requestedSettings.map(s => s.`type` -> s.config).toMap - existingSettings.map { case WorkspaceSetting(settingType, _) => - WorkspaceSetting(settingType, requestedSettingsMap.getOrElse(settingType, settingType.defaultConfig())) - } - } - + validateSettings(workspaceSettings) for { workspace <- getV2WorkspaceContextAndPermissions(workspaceName, SamWorkspaceActions.own) currentSettings <- workspaceRepository.getWorkspaceSettings(workspace.workspaceIdAsUUID) newSettings = computeNewSettings(workspace, workspaceSettings, currentSettings) _ <- workspaceRepository.createWorkspaceSettingsRecords(workspace.workspaceIdAsUUID, workspaceSettings) - applyFailures <- workspaceSettings.traverse(s => applySetting(workspace, s)) + applyFailures <- newSettings.traverse(s => applySetting(workspace, s)) } yield { - val successes = workspaceSettings.filterNot { s => + val successes = newSettings.filterNot { s => applyFailures.flatten.exists { case (failedSettingType, _) => failedSettingType == s.`type` } diff --git a/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala b/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala index 64f81c649f..a1f8d81d0d 100644 --- a/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala +++ b/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala @@ -563,13 +563,13 @@ object WorkspaceState { case object DeleteFailed extends WorkspaceState } -case class WorkspaceSetting(`type`: WorkspaceSettingType, config: Option[WorkspaceSettingConfig]) +case class WorkspaceSetting(`type`: WorkspaceSettingType, config: WorkspaceSettingConfig) object WorkspaceSettingTypes { sealed trait WorkspaceSettingType extends RawlsEnumeration[WorkspaceSettingType] { override def toString: String = getClass.getSimpleName.stripSuffix("$") override def withName(name: String): WorkspaceSettingType = WorkspaceSettingTypes.withName(name) - def defaultConfig(): Option[WorkspaceSettingConfig] + def defaultConfig(): WorkspaceSettingConfig } def withName(name: String): WorkspaceSettingType = name.toLowerCase match { @@ -578,7 +578,7 @@ object WorkspaceSettingTypes { } case object GcpBucketLifecycle extends WorkspaceSettingType { - override def defaultConfig(): Option[WorkspaceSettingConfig] = Some(GcpBucketLifecycleConfig(List.empty)) + override def defaultConfig(): WorkspaceSettingConfig = GcpBucketLifecycleConfig(List.empty) } } @@ -1231,7 +1231,7 @@ class WorkspaceJsonSupport extends JsonSupport { val fields = json.asJsObject.fields val settingType = fields("type").convertTo[WorkspaceSettingType] val configuration = settingType match { - case GcpBucketLifecycle => fields.get("config").map(_.convertTo[GcpBucketLifecycleConfig]) + case GcpBucketLifecycle => fields("config").convertTo[GcpBucketLifecycleConfig] case _ => throw DeserializationException(s"unexpected setting type $settingType") } WorkspaceSetting(settingType, configuration) From 03d7e31788813f15677815f4fc3dfbc220f37c4b Mon Sep 17 00:00:00 2001 From: Marcus Talbott Date: Wed, 7 Aug 2024 16:15:47 -0400 Subject: [PATCH 08/25] start unit tests --- .../workspace/WorkspaceServiceUnitTests.scala | 52 ++++++++++++++++--- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala index 73b285fc34..642dede741 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala @@ -4,7 +4,7 @@ import akka.http.scaladsl.model.StatusCodes import akka.http.scaladsl.model.headers.OAuth2BearerToken import akka.stream.Materializer import bio.terra.workspace.model.{IamRole, RoleBinding, RoleBindingList} -import org.broadinstitute.dsde.rawls.billing.BillingProfileManagerDAO +import org.broadinstitute.dsde.rawls.billing.{BillingProfileManagerDAO, BillingRepository} import org.broadinstitute.dsde.rawls.config._ import org.broadinstitute.dsde.rawls.dataaccess._ import org.broadinstitute.dsde.rawls.dataaccess.leonardo.LeonardoService @@ -55,7 +55,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki val workspace: Workspace = Workspace( "test-namespace", "test-name", - "aWorkspaceId", + UUID.randomUUID().toString, "aBucket", Some("workflow-collection"), new DateTime(), @@ -87,9 +87,12 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki billingProfileManagerDAO: BillingProfileManagerDAO = mock[BillingProfileManagerDAO](RETURNS_SMART_NULLS), aclManagerDatasource: SlickDataSource = mock[SlickDataSource](RETURNS_SMART_NULLS), fastPassServiceConstructor: (RawlsRequestContext, SlickDataSource) => FastPassServiceImpl = (_, _) => - mock[FastPassServiceImpl](RETURNS_SMART_NULLS) + mock[FastPassServiceImpl](RETURNS_SMART_NULLS), + workspaceRepository: WorkspaceRepository = mock[WorkspaceRepository](RETURNS_SMART_NULLS), + billingRepository: BillingRepository = mock[BillingRepository](RETURNS_SMART_NULLS) ): RawlsRequestContext => WorkspaceService = info => - WorkspaceService.constructor( + new WorkspaceService( + info, datasource, executionServiceCluster, workspaceManagerDAO, @@ -111,8 +114,10 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki terraBucketWriterRole, new RawlsWorkspaceAclManager(samDAO), new MultiCloudWorkspaceAclManager(workspaceManagerDAO, samDAO, billingProfileManagerDAO, aclManagerDatasource), - fastPassServiceConstructor - )(info)(mock[Materializer], scala.concurrent.ExecutionContext.global) + fastPassServiceConstructor, + workspaceRepository, + billingRepository + )(scala.concurrent.ExecutionContext.global) "getWorkspaceById" should "return the workspace returned by getWorkspace(WorkspaceName) on success" in { val datasource = mock[SlickDataSource] @@ -772,4 +777,39 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki Await.result(service.updateACL(WorkspaceName("fake_namespace", "fake_name"), aclUpdate, true), Duration.Inf) } + + "getWorkspaceSettings" should "return the workspace settings" in { + val workspaceId = workspace.workspaceIdAsUUID + val workspaceName = workspace.toWorkspaceName + val workspaceSettings = List( + WorkspaceSetting(WorkspaceSettingTypes.GcpBucketLifecycle, + WorkspaceSettingTypes.GcpBucketLifecycle.defaultConfig() + ) + ) + val workspaceRepository = mock[WorkspaceRepository] + when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) + when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(workspaceSettings)) + + val samDAO = mock[SamDAO] + when(samDAO.getUserStatus(any())) + .thenReturn(Future.successful(Option(SamUserStatusResponse("fake_user_id", "user@example.com", true)))) + when( + samDAO.userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), + ArgumentMatchers.eq(workspaceId.toString), + ArgumentMatchers.eq(SamWorkspaceActions.read), + any() + ) + ).thenReturn(Future.successful(true)) + + val service = + workspaceServiceConstructor(samDAO = samDAO, workspaceRepository = workspaceRepository)(defaultRequestContext) + + val returnedSettings = Await.result(service.getWorkspaceSettings(workspaceName), Duration.Inf) + returnedSettings shouldEqual workspaceSettings + verify(samDAO).userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), + ArgumentMatchers.eq(workspaceId.toString), + ArgumentMatchers.eq(SamWorkspaceActions.read), + any() + ) + } } From 6259e9a01fa56b2d8b9f9e5c73682bac41dfafc1 Mon Sep 17 00:00:00 2001 From: Marcus Talbott Date: Wed, 7 Aug 2024 17:35:00 -0400 Subject: [PATCH 09/25] fix tests --- .../workspace/WorkspaceServiceUnitTests.scala | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala index 642dede741..ee4aad6997 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala @@ -398,16 +398,16 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki samDAO } - def mockDatasourceForAclTests(workspaceType: WorkspaceType, - workspaceId: UUID = UUID.randomUUID() - ): SlickDataSource = { - val datasource = mock[SlickDataSource](RETURNS_SMART_NULLS) + def mockWorkspaceRepositoryForAclTests(workspaceType: WorkspaceType, + workspaceId: UUID = UUID.randomUUID() + ): WorkspaceRepository = { + val workspaceRepository = mock[WorkspaceRepository](RETURNS_SMART_NULLS) val googleProjectId = workspaceType match { case WorkspaceType.McWorkspace => GoogleProjectId("") case WorkspaceType.RawlsWorkspace => GoogleProjectId("fake-project-id") } - when(datasource.inTransaction[Option[Workspace]](any(), any())).thenReturn( + when(workspaceRepository.getWorkspace(any[WorkspaceName](), any())).thenReturn( Future.successful( Option( Workspace("fake_namespace", @@ -423,7 +423,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki ) ) ) - datasource + workspaceRepository } def samWorkspacePoliciesForAclTests(projectOwnerEmail: String, @@ -472,9 +472,9 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki Future.successful(samWorkspacePoliciesForAclTests(projectOwnerEmail, ownerEmail, writerEmail, readerEmail)) ) - val datasource = mockDatasourceForAclTests(WorkspaceType.RawlsWorkspace) + val workspaceRepository = mockWorkspaceRepositoryForAclTests(WorkspaceType.RawlsWorkspace) - val service = workspaceServiceConstructor(datasource, samDAO = samDAO)(defaultRequestContext) + val service = workspaceServiceConstructor(workspaceRepository = workspaceRepository, samDAO = samDAO)(defaultRequestContext) val result = Await.result(service.getACL(WorkspaceName("fake_namespace", "fake_name")), Duration.Inf) val expected = WorkspaceACL( @@ -495,11 +495,11 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki val readerEmail = "reader@example.com" val wsmDAO = mockWsmForAclTests(ownerEmail, writerEmail, readerEmail) - val datasource = mockDatasourceForAclTests(WorkspaceType.McWorkspace) + val workspaceRepository = mockWorkspaceRepositoryForAclTests(WorkspaceType.McWorkspace) val samDAO = mockSamForAclTests() val service = - workspaceServiceConstructor(datasource, samDAO = samDAO, workspaceManagerDAO = wsmDAO)(defaultRequestContext) + workspaceServiceConstructor(workspaceRepository = workspaceRepository, samDAO = samDAO, workspaceManagerDAO = wsmDAO)(defaultRequestContext) val expected = WorkspaceACL( Map( @@ -530,7 +530,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki when(samDAO.removeUserFromPolicy(any(), any(), any(), any(), any())).thenReturn(Future.successful()) val workspaceId = UUID.randomUUID() - val datasource = mockDatasourceForAclTests(WorkspaceType.RawlsWorkspace, workspaceId) + val workspaceRepository = mockWorkspaceRepositoryForAclTests(WorkspaceType.RawlsWorkspace, workspaceId) val requesterPaysSetupService = mock[RequesterPaysSetupServiceImpl](RETURNS_SMART_NULLS) when(requesterPaysSetupService.revokeUserFromWorkspace(any(), any())).thenReturn(Future.successful(Seq.empty)) @@ -540,7 +540,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki .thenReturn(Future.successful()) val service = - workspaceServiceConstructor(datasource, + workspaceServiceConstructor(workspaceRepository = workspaceRepository, samDAO = samDAO, requesterPaysSetupService = requesterPaysSetupService, fastPassServiceConstructor = (_, _) => mockFastPassService @@ -582,7 +582,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki val workspaceId = UUID.randomUUID() val wsmDAO = mockWsmForAclTests(ownerEmail, writerEmail, readerEmail) - val datasource = mockDatasourceForAclTests(WorkspaceType.McWorkspace, workspaceId) + val workspaceRepository = mockWorkspaceRepositoryForAclTests(WorkspaceType.McWorkspace, workspaceId) val samDAO = mockSamForAclTests() val aclManagerDatasource = mock[SlickDataSource] @@ -607,7 +607,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki ) ) val service = - workspaceServiceConstructor(datasource, + workspaceServiceConstructor(workspaceRepository = workspaceRepository, samDAO = samDAO, workspaceManagerDAO = wsmDAO, aclManagerDatasource = aclManagerDatasource, @@ -647,7 +647,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki val workspaceId = UUID.randomUUID() val wsmDAO = mockWsmForAclTests(ownerEmail, writerEmail, readerEmail) - val datasource = mockDatasourceForAclTests(WorkspaceType.McWorkspace, workspaceId) + val workspaceRepository = mockWorkspaceRepositoryForAclTests(WorkspaceType.McWorkspace, workspaceId) val samDAO = mockSamForAclTests() val aclUpdates = Set( @@ -655,7 +655,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki ) val service = - workspaceServiceConstructor(datasource, samDAO = samDAO, workspaceManagerDAO = wsmDAO)(defaultRequestContext) + workspaceServiceConstructor(workspaceRepository = workspaceRepository, samDAO = samDAO, workspaceManagerDAO = wsmDAO)(defaultRequestContext) val exception = intercept[InvalidWorkspaceAclUpdateException] { Await.result(service.updateACL(WorkspaceName("fake_namespace", "fake_name"), aclUpdates, true), Duration.Inf) } @@ -670,7 +670,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki val workspaceId = UUID.randomUUID() val wsmDAO = mockWsmForAclTests(ownerEmail, writerEmail, readerEmail) - val datasource = mockDatasourceForAclTests(WorkspaceType.McWorkspace, workspaceId) + val workspaceRepository = mockWorkspaceRepositoryForAclTests(WorkspaceType.McWorkspace, workspaceId) val samDAO = mockSamForAclTests() val aclUpdates = Set( @@ -678,7 +678,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki ) val service = - workspaceServiceConstructor(datasource, samDAO = samDAO, workspaceManagerDAO = wsmDAO)(defaultRequestContext) + workspaceServiceConstructor(workspaceRepository = workspaceRepository, samDAO = samDAO, workspaceManagerDAO = wsmDAO)(defaultRequestContext) val exception = intercept[InvalidWorkspaceAclUpdateException] { Await.result(service.updateACL(WorkspaceName("fake_namespace", "fake_name"), aclUpdates, true), Duration.Inf) } @@ -693,7 +693,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki val workspaceId = UUID.randomUUID() val wsmDAO = mockWsmForAclTests(ownerEmail, writerEmail, readerEmail) - val datasource = mockDatasourceForAclTests(WorkspaceType.McWorkspace, workspaceId) + val workspaceRepository = mockWorkspaceRepositoryForAclTests(WorkspaceType.McWorkspace, workspaceId) val samDAO = mockSamForAclTests() val aclUpdates = Set( @@ -701,7 +701,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki ) val service = - workspaceServiceConstructor(datasource, samDAO = samDAO, workspaceManagerDAO = wsmDAO)(defaultRequestContext) + workspaceServiceConstructor(workspaceRepository = workspaceRepository, samDAO = samDAO, workspaceManagerDAO = wsmDAO)(defaultRequestContext) val exception = intercept[InvalidWorkspaceAclUpdateException] { Await.result(service.updateACL(WorkspaceName("fake_namespace", "fake_name"), aclUpdates, true), Duration.Inf) } @@ -722,12 +722,12 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki when(samDAO.addUserToPolicy(any(), any(), any(), any(), any())).thenReturn(Future.successful()) val workspaceId = UUID.randomUUID() - val datasource = mockDatasourceForAclTests(WorkspaceType.RawlsWorkspace, workspaceId) + val workspaceRepository = mockWorkspaceRepositoryForAclTests(WorkspaceType.RawlsWorkspace, workspaceId) val mockFastPassService = mock[FastPassServiceImpl] when(mockFastPassService.syncFastPassesForUserInWorkspace(any[Workspace], any[String])) .thenReturn(Future.successful()) - val service = workspaceServiceConstructor(datasource, + val service = workspaceServiceConstructor(workspaceRepository = workspaceRepository, samDAO = samDAO, fastPassServiceConstructor = (_, _) => mockFastPassService )(defaultRequestContext) @@ -760,12 +760,12 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki when(samDAO.addUserToPolicy(any(), any(), any(), any(), any())).thenReturn(Future.successful()) val workspaceId = UUID.randomUUID() - val datasource = mockDatasourceForAclTests(WorkspaceType.RawlsWorkspace, workspaceId) + val workspaceRepository = mockWorkspaceRepositoryForAclTests(WorkspaceType.RawlsWorkspace, workspaceId) val mockFastPassService = mock[FastPassServiceImpl] when(mockFastPassService.syncFastPassesForUserInWorkspace(any[Workspace], any[String])) .thenReturn(Future.successful()) - val service = workspaceServiceConstructor(datasource, + val service = workspaceServiceConstructor(workspaceRepository = workspaceRepository, samDAO = samDAO, fastPassServiceConstructor = (_, _) => mockFastPassService )(defaultRequestContext) From 75301c57d88f0b5cef9fbee582174fa5128b5963 Mon Sep 17 00:00:00 2001 From: Marcus Talbott Date: Thu, 8 Aug 2024 15:06:09 -0400 Subject: [PATCH 10/25] more unit tests, rework some db things --- .../slick/WorkspaceSettingComponent.scala | 15 +- .../rawls/workspace/WorkspaceRepository.scala | 51 ++- .../rawls/workspace/WorkspaceService.scala | 17 +- .../workspace/WorkspaceServiceUnitTests.scala | 372 +++++++++++++++++- 4 files changed, 417 insertions(+), 38 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala index 05aee176d7..f36c897697 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala @@ -45,7 +45,8 @@ object WorkspaceSettingRecord { val settingType = WorkspaceSettingTypes.withName(workspaceSettingRecord.`type`) val settingConfig = settingType match { - case WorkspaceSettingTypes.GcpBucketLifecycle => workspaceSettingRecord.config.parseJson.convertTo[GcpBucketLifecycleConfig] + case WorkspaceSettingTypes.GcpBucketLifecycle => + workspaceSettingRecord.config.parseJson.convertTo[GcpBucketLifecycleConfig] } WorkspaceSetting(settingType, settingConfig) @@ -77,14 +78,14 @@ trait WorkspaceSettingComponent { (workspaceSettingQuery ++= records).map(_ => workspaceSettings) } - def updateStatuses(workspaceId: UUID, - workspaceSettingTypes: List[WorkspaceSettingType], - currentStatus: WorkspaceSettingRecord.SettingStatus.SettingStatus, - newStatus: WorkspaceSettingRecord.SettingStatus.SettingStatus + def updateSettingStatus(workspaceId: UUID, + workspaceSettingType: WorkspaceSettingType, + currentStatus: WorkspaceSettingRecord.SettingStatus.SettingStatus, + newStatus: WorkspaceSettingRecord.SettingStatus.SettingStatus ): ReadWriteAction[Int] = workspaceSettingQuery .filter(record => - record.workspaceId === workspaceId && record.`type`.inSetBind(workspaceSettingTypes.map(_.toString)) && record.status === currentStatus.toString + record.workspaceId === workspaceId && record.`type` === workspaceSettingType.toString && record.status === currentStatus.toString ) .map(_.status) .update(newStatus.toString) @@ -95,7 +96,7 @@ trait WorkspaceSettingComponent { ): ReadWriteAction[Int] = filter(record => record.workspaceId === workspaceId && record.`type` === settingType.toString && record.status === status.toString - ).map(_.status).update(WorkspaceSettingRecord.SettingStatus.Deleted.toString) + ).delete def listSettingsForWorkspaceByStatus(workspaceId: UUID, status: WorkspaceSettingRecord.SettingStatus.SettingStatus diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala index 65ea40fac6..85ca49b562 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala @@ -5,7 +5,15 @@ import org.broadinstitute.dsde.rawls.RawlsExceptionWithErrorReport import org.broadinstitute.dsde.rawls.dataaccess.SlickDataSource import org.broadinstitute.dsde.rawls.dataaccess.slick.WorkspaceSettingRecord import org.broadinstitute.dsde.rawls.model.Attributable.AttributeMap -import org.broadinstitute.dsde.rawls.model.{ErrorReport, RawlsRequestContext, Workspace, WorkspaceAttributeSpecs, WorkspaceName, WorkspaceSetting, WorkspaceState} +import org.broadinstitute.dsde.rawls.model.{ + ErrorReport, + RawlsRequestContext, + Workspace, + WorkspaceAttributeSpecs, + WorkspaceName, + WorkspaceSetting, + WorkspaceState +} import org.broadinstitute.dsde.rawls.model.WorkspaceState.WorkspaceState import org.broadinstitute.dsde.rawls.model.WorkspaceSettingTypes.WorkspaceSettingType import org.broadinstitute.dsde.rawls.util.TracingUtils.traceDBIOWithParent @@ -96,30 +104,53 @@ class WorkspaceRepository(dataSource: SlickDataSource) { def getWorkspaceSettings(workspaceId: UUID): Future[List[WorkspaceSetting]] = dataSource.inTransaction { access => - access.workspaceSettingQuery.listSettingsForWorkspaceByStatus(workspaceId, WorkspaceSettingRecord.SettingStatus.Applied) + access.workspaceSettingQuery.listSettingsForWorkspaceByStatus(workspaceId, + WorkspaceSettingRecord.SettingStatus.Applied + ) } - def createWorkspaceSettingsRecords(workspaceId: UUID, workspaceSettings: List[WorkspaceSetting])(implicit ec: ExecutionContext): Future[List[WorkspaceSetting]] = + def createWorkspaceSettingsRecords(workspaceId: UUID, workspaceSettings: List[WorkspaceSetting])(implicit + ec: ExecutionContext + ): Future[List[WorkspaceSetting]] = dataSource.inTransaction { access => for { - pendingSettingsForWorkspace <- access.workspaceSettingQuery.listSettingsForWorkspaceByStatus(workspaceId, WorkspaceSettingRecord.SettingStatus.Pending) + pendingSettingsForWorkspace <- access.workspaceSettingQuery.listSettingsForWorkspaceByStatus( + workspaceId, + WorkspaceSettingRecord.SettingStatus.Pending + ) _ = if (pendingSettingsForWorkspace.nonEmpty) { - throw new RawlsExceptionWithErrorReport(ErrorReport(StatusCodes.Conflict, s"Workspace $workspaceId already has pending settings")) + throw new RawlsExceptionWithErrorReport( + ErrorReport(StatusCodes.Conflict, s"Workspace $workspaceId already has pending settings") + ) } _ <- access.workspaceSettingQuery.saveAll(workspaceId, workspaceSettings) } yield workspaceSettings } - def markWorkspaceSettingApplied(workspaceId: UUID, workspaceSetting: WorkspaceSetting)(implicit ec: ExecutionContext): Future[Int] = + // Transition old Applied settings to Deleted and Pending settings to Applied + def markWorkspaceSettingApplied(workspaceId: UUID, workspaceSettingType: WorkspaceSettingType)(implicit + ec: ExecutionContext + ): Future[Int] = dataSource.inTransaction { access => for { - _ <- access.workspaceSettingQuery.deleteSettingTypeForWorkspaceByStatus(workspaceId, workspaceSetting.`type`, WorkspaceSettingRecord.SettingStatus.Applied) - res <- access.workspaceSettingQuery.updateStatuses(workspaceId, List(workspaceSetting.`type`), WorkspaceSettingRecord.SettingStatus.Pending, WorkspaceSettingRecord.SettingStatus.Applied) + _ <- access.workspaceSettingQuery.updateSettingStatus(workspaceId, + workspaceSettingType, + WorkspaceSettingRecord.SettingStatus.Applied, + WorkspaceSettingRecord.SettingStatus.Deleted + ) + res <- access.workspaceSettingQuery.updateSettingStatus(workspaceId, + workspaceSettingType, + WorkspaceSettingRecord.SettingStatus.Pending, + WorkspaceSettingRecord.SettingStatus.Applied + ) } yield res } - def removePendingSetting(workspaceId: UUID, workspaceSetting: WorkspaceSetting): Future[Int] = + def removePendingSetting(workspaceId: UUID, workspaceSettingType: WorkspaceSettingType): Future[Int] = dataSource.inTransaction { access => - access.workspaceSettingQuery.deleteSettingTypeForWorkspaceByStatus(workspaceId, workspaceSetting.`type`, WorkspaceSettingRecord.SettingStatus.Pending) + access.workspaceSettingQuery.deleteSettingTypeForWorkspaceByStatus(workspaceId, + workspaceSettingType, + WorkspaceSettingRecord.SettingStatus.Pending + ) } } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala index c978e7274d..5fd9a509fb 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala @@ -2013,15 +2013,19 @@ class WorkspaceService( def setWorkspaceSettings(workspaceName: WorkspaceName, workspaceSettings: List[WorkspaceSetting] ): Future[WorkspaceSettingResponse] = { + /** * Perform basic validation checks on requested settings. */ def validateSettings(requestedSettings: List[WorkspaceSetting]): Unit = { val validationErrors = requestedSettings.flatMap { - case WorkspaceSetting(settingType@WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig(rules)) => + case WorkspaceSetting(settingType @ WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig(rules) + ) => rules.flatMap { rule => - rule.conditions.age.collect { case age if age < 0 => - ErrorReport(s"invalid $settingType configuration: age must be a non-negative integer.") + rule.conditions.age.collect { + case age if age < 0 => + ErrorReport(s"invalid $settingType configuration: age must be a non-negative integer.") } } } @@ -2041,7 +2045,7 @@ class WorkspaceService( def computeNewSettings(workspace: Workspace, requestedSettings: List[WorkspaceSetting], existingSettings: List[WorkspaceSetting] - ): List[WorkspaceSetting] = { + ): List[WorkspaceSetting] = if (existingSettings.isEmpty) { requestedSettings } else { @@ -2050,7 +2054,6 @@ class WorkspaceService( WorkspaceSetting(settingType, requestedSettingsMap.getOrElse(settingType, settingType.defaultConfig())) } } - } /** * Apply a setting to a workspace. If the setting is successfully applied, update the database @@ -2079,12 +2082,12 @@ class WorkspaceService( for { _ <- gcsDAO.setBucketLifecycle(workspace.bucketName, googleRules) - _ <- workspaceRepository.markWorkspaceSettingApplied(workspace.workspaceIdAsUUID, setting) + _ <- workspaceRepository.markWorkspaceSettingApplied(workspace.workspaceIdAsUUID, setting.`type`) } yield None case _ => throw new RawlsException("unsupported workspace setting") }).recoverWith { case e => workspaceRepository - .removePendingSetting(workspace.workspaceIdAsUUID, setting) + .removePendingSetting(workspace.workspaceIdAsUUID, setting.`type`) .map(_ => Some((setting.`type`, ErrorReport(StatusCodes.InternalServerError, e.getMessage)))) } diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala index ee4aad6997..ae3abba378 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala @@ -4,12 +4,22 @@ import akka.http.scaladsl.model.StatusCodes import akka.http.scaladsl.model.headers.OAuth2BearerToken import akka.stream.Materializer import bio.terra.workspace.model.{IamRole, RoleBinding, RoleBindingList} +import com.google.api.client.googleapis.json.GoogleJsonResponseException +import com.google.cloud.storage.BucketInfo.LifecycleRule +import com.google.cloud.storage.BucketInfo.LifecycleRule.{LifecycleAction, LifecycleCondition} +import org.broadinstitute.dsde.rawls import org.broadinstitute.dsde.rawls.billing.{BillingProfileManagerDAO, BillingRepository} import org.broadinstitute.dsde.rawls.config._ import org.broadinstitute.dsde.rawls.dataaccess._ import org.broadinstitute.dsde.rawls.dataaccess.leonardo.LeonardoService import org.broadinstitute.dsde.rawls.dataaccess.workspacemanager.WorkspaceManagerDAO import org.broadinstitute.dsde.rawls.fastpass.FastPassServiceImpl +import org.broadinstitute.dsde.rawls.model.WorkspaceSettingConfig.{ + GcpBucketLifecycleConfig, + GcpBucketLifecycleRule, + GcpLifecycleAction, + GcpLifecycleCondition +} import org.broadinstitute.dsde.rawls.model.WorkspaceType.WorkspaceType import org.broadinstitute.dsde.rawls.model._ import org.broadinstitute.dsde.rawls.resourcebuffer.ResourceBufferServiceImpl @@ -31,6 +41,7 @@ import org.mockito.ArgumentMatchers._ import org.mockito.Mockito._ import org.scalatest.OptionValues import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.must.Matchers.{contain, include} import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper import spray.json.{JsObject, JsString} @@ -474,7 +485,8 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki val workspaceRepository = mockWorkspaceRepositoryForAclTests(WorkspaceType.RawlsWorkspace) - val service = workspaceServiceConstructor(workspaceRepository = workspaceRepository, samDAO = samDAO)(defaultRequestContext) + val service = + workspaceServiceConstructor(workspaceRepository = workspaceRepository, samDAO = samDAO)(defaultRequestContext) val result = Await.result(service.getACL(WorkspaceName("fake_namespace", "fake_name")), Duration.Inf) val expected = WorkspaceACL( @@ -499,7 +511,10 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki val samDAO = mockSamForAclTests() val service = - workspaceServiceConstructor(workspaceRepository = workspaceRepository, samDAO = samDAO, workspaceManagerDAO = wsmDAO)(defaultRequestContext) + workspaceServiceConstructor(workspaceRepository = workspaceRepository, + samDAO = samDAO, + workspaceManagerDAO = wsmDAO + )(defaultRequestContext) val expected = WorkspaceACL( Map( @@ -540,10 +555,11 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki .thenReturn(Future.successful()) val service = - workspaceServiceConstructor(workspaceRepository = workspaceRepository, - samDAO = samDAO, - requesterPaysSetupService = requesterPaysSetupService, - fastPassServiceConstructor = (_, _) => mockFastPassService + workspaceServiceConstructor( + workspaceRepository = workspaceRepository, + samDAO = samDAO, + requesterPaysSetupService = requesterPaysSetupService, + fastPassServiceConstructor = (_, _) => mockFastPassService )( defaultRequestContext ) @@ -607,11 +623,12 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki ) ) val service = - workspaceServiceConstructor(workspaceRepository = workspaceRepository, - samDAO = samDAO, - workspaceManagerDAO = wsmDAO, - aclManagerDatasource = aclManagerDatasource, - fastPassServiceConstructor = (_, _) => mockFastPassService + workspaceServiceConstructor( + workspaceRepository = workspaceRepository, + samDAO = samDAO, + workspaceManagerDAO = wsmDAO, + aclManagerDatasource = aclManagerDatasource, + fastPassServiceConstructor = (_, _) => mockFastPassService )(defaultRequestContext) val aclUpdates = Set( @@ -655,7 +672,10 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki ) val service = - workspaceServiceConstructor(workspaceRepository = workspaceRepository, samDAO = samDAO, workspaceManagerDAO = wsmDAO)(defaultRequestContext) + workspaceServiceConstructor(workspaceRepository = workspaceRepository, + samDAO = samDAO, + workspaceManagerDAO = wsmDAO + )(defaultRequestContext) val exception = intercept[InvalidWorkspaceAclUpdateException] { Await.result(service.updateACL(WorkspaceName("fake_namespace", "fake_name"), aclUpdates, true), Duration.Inf) } @@ -678,7 +698,10 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki ) val service = - workspaceServiceConstructor(workspaceRepository = workspaceRepository, samDAO = samDAO, workspaceManagerDAO = wsmDAO)(defaultRequestContext) + workspaceServiceConstructor(workspaceRepository = workspaceRepository, + samDAO = samDAO, + workspaceManagerDAO = wsmDAO + )(defaultRequestContext) val exception = intercept[InvalidWorkspaceAclUpdateException] { Await.result(service.updateACL(WorkspaceName("fake_namespace", "fake_name"), aclUpdates, true), Duration.Inf) } @@ -701,7 +724,10 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki ) val service = - workspaceServiceConstructor(workspaceRepository = workspaceRepository, samDAO = samDAO, workspaceManagerDAO = wsmDAO)(defaultRequestContext) + workspaceServiceConstructor(workspaceRepository = workspaceRepository, + samDAO = samDAO, + workspaceManagerDAO = wsmDAO + )(defaultRequestContext) val exception = intercept[InvalidWorkspaceAclUpdateException] { Await.result(service.updateACL(WorkspaceName("fake_namespace", "fake_name"), aclUpdates, true), Duration.Inf) } @@ -786,6 +812,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki WorkspaceSettingTypes.GcpBucketLifecycle.defaultConfig() ) ) + val workspaceRepository = mock[WorkspaceRepository] when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(workspaceSettings)) @@ -812,4 +839,321 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki any() ) } + + it should "handle a workspace with no settings" in { + val workspaceId = workspace.workspaceIdAsUUID + val workspaceName = workspace.toWorkspaceName + + val workspaceRepository = mock[WorkspaceRepository] + when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) + when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(List.empty)) + + val samDAO = mock[SamDAO] + when(samDAO.getUserStatus(any())) + .thenReturn(Future.successful(Option(SamUserStatusResponse("fake_user_id", "user@example.com", true)))) + when( + samDAO.userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), + ArgumentMatchers.eq(workspaceId.toString), + ArgumentMatchers.eq(SamWorkspaceActions.read), + any() + ) + ).thenReturn(Future.successful(true)) + + val service = + workspaceServiceConstructor(samDAO = samDAO, workspaceRepository = workspaceRepository)(defaultRequestContext) + + val returnedSettings = Await.result(service.getWorkspaceSettings(workspaceName), Duration.Inf) + returnedSettings shouldEqual List.empty + verify(samDAO).userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), + ArgumentMatchers.eq(workspaceId.toString), + ArgumentMatchers.eq(SamWorkspaceActions.read), + any() + ) + } + + it should "return an error if the user does not have read access to the workspace" in { + val workspaceId = workspace.workspaceIdAsUUID + val workspaceName = workspace.toWorkspaceName + + val workspaceRepository = mock[WorkspaceRepository] + when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) + + val samDAO = mock[SamDAO] + when(samDAO.getUserStatus(any())) + .thenReturn(Future.successful(Option(SamUserStatusResponse("fake_user_id", "user@example.com", true)))) + when( + samDAO.userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), + ArgumentMatchers.eq(workspaceId.toString), + ArgumentMatchers.eq(SamWorkspaceActions.read), + any() + ) + ).thenReturn(Future.successful(false)) + val service = + workspaceServiceConstructor(samDAO = samDAO, workspaceRepository = workspaceRepository)(defaultRequestContext) + + assertThrows[NoSuchWorkspaceException] { + Await.result(service.getWorkspaceSettings(workspaceName), Duration.Inf) + } + } + + "setWorkspaceSettings" should "set the workspace settings if there aren't any set" in { + val workspaceId = workspace.workspaceIdAsUUID + val workspaceName = workspace.toWorkspaceName + val workspaceSetting = WorkspaceSetting(WorkspaceSettingTypes.GcpBucketLifecycle, + WorkspaceSettingTypes.GcpBucketLifecycle.defaultConfig() + ) + + val workspaceRepository = mock[WorkspaceRepository] + when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) + when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(List.empty)) + when(workspaceRepository.createWorkspaceSettingsRecords(workspaceId, List(workspaceSetting))) + .thenReturn(Future.successful(List(workspaceSetting))) + when(workspaceRepository.markWorkspaceSettingApplied(workspaceId, workspaceSetting.`type`)) + .thenReturn(Future.successful(1)) + + val samDAO = mock[SamDAO] + when(samDAO.getUserStatus(any())) + .thenReturn(Future.successful(Option(SamUserStatusResponse("fake_user_id", "user@example.com", true)))) + when( + samDAO.userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), + ArgumentMatchers.eq(workspaceId.toString), + ArgumentMatchers.eq(SamWorkspaceActions.own), + any() + ) + ).thenReturn(Future.successful(true)) + + val gcsDAO = mock[GoogleServicesDAO] + when(gcsDAO.setBucketLifecycle(workspace.bucketName, List())).thenReturn(Future.successful()) + + val service = workspaceServiceConstructor(samDAO = samDAO, + workspaceRepository = workspaceRepository, + gcsDAO = gcsDAO + )(defaultRequestContext) + + val res = Await.result(service.setWorkspaceSettings(workspaceName, List(workspaceSetting)), Duration.Inf) + res.successes should contain theSameElementsAs List(workspaceSetting) + res.failures shouldEqual Map.empty + } + + it should "overwrite existing settings" in { + val workspaceId = workspace.workspaceIdAsUUID + val workspaceName = workspace.toWorkspaceName + val existingSetting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpLifecycleAction("Delete"), GcpLifecycleCondition(Set("prefixToMatch"), Some(30))) + ) + ) + ) + val newSetting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpLifecycleAction("Delete"), GcpLifecycleCondition(Set("muchBetterPrefix"), Some(31))) + ) + ) + ) + + val workspaceRepository = mock[WorkspaceRepository] + when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) + when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(List(existingSetting))) + when(workspaceRepository.createWorkspaceSettingsRecords(workspaceId, List(newSetting))) + .thenReturn(Future.successful(List(newSetting))) + when(workspaceRepository.markWorkspaceSettingApplied(workspaceId, newSetting.`type`)) + .thenReturn(Future.successful(1)) + + val samDAO = mock[SamDAO] + when(samDAO.getUserStatus(any())) + .thenReturn(Future.successful(Option(SamUserStatusResponse("fake_user_id", "user@example.com", true)))) + when( + samDAO.userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), + ArgumentMatchers.eq(workspaceId.toString), + ArgumentMatchers.eq(SamWorkspaceActions.own), + any() + ) + ).thenReturn(Future.successful(true)) + + val gcsDAO = mock[GoogleServicesDAO] + val newSettingGoogleRule = new LifecycleRule( + LifecycleAction.newDeleteAction(), + LifecycleCondition.newBuilder().setMatchesPrefix(List("muchBetterPrefix").asJava).setAge(31).build() + ) + when(gcsDAO.setBucketLifecycle(workspace.bucketName, List(newSettingGoogleRule))).thenReturn(Future.successful()) + + val service = workspaceServiceConstructor(samDAO = samDAO, + workspaceRepository = workspaceRepository, + gcsDAO = gcsDAO + )(defaultRequestContext) + + val res = Await.result(service.setWorkspaceSettings(workspaceName, List(newSetting)), Duration.Inf) + res.successes should contain theSameElementsAs List(newSetting) + res.failures shouldEqual Map.empty + } + + it should "remove existing settings if no settings are specified" in { + val workspaceId = workspace.workspaceIdAsUUID + val workspaceName = workspace.toWorkspaceName + val existingSetting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpLifecycleAction("Delete"), GcpLifecycleCondition(Set("prefixToMatch"), Some(30))) + ) + ) + ) + val defaultSetting = WorkspaceSetting(WorkspaceSettingTypes.GcpBucketLifecycle, + WorkspaceSettingTypes.GcpBucketLifecycle.defaultConfig() + ) + + val workspaceRepository = mock[WorkspaceRepository] + when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) + when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(List(existingSetting))) + when(workspaceRepository.createWorkspaceSettingsRecords(workspaceId, List.empty)) + .thenReturn(Future.successful(List.empty)) + when(workspaceRepository.markWorkspaceSettingApplied(workspaceId, defaultSetting.`type`)) + .thenReturn(Future.successful(1)) + + val samDAO = mock[SamDAO] + when(samDAO.getUserStatus(any())) + .thenReturn(Future.successful(Option(SamUserStatusResponse("fake_user_id", "user@example.com", true)))) + when( + samDAO.userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), + ArgumentMatchers.eq(workspaceId.toString), + ArgumentMatchers.eq(SamWorkspaceActions.own), + any() + ) + ).thenReturn(Future.successful(true)) + + val gcsDAO = mock[GoogleServicesDAO] + when(gcsDAO.setBucketLifecycle(workspace.bucketName, List())).thenReturn(Future.successful()) + + val service = workspaceServiceConstructor(samDAO = samDAO, + workspaceRepository = workspaceRepository, + gcsDAO = gcsDAO + )(defaultRequestContext) + + val res = Await.result(service.setWorkspaceSettings(workspaceName, List.empty), Duration.Inf) + res.successes should contain theSameElementsAs List(defaultSetting) + res.failures shouldEqual Map.empty + } + + it should "report errors while applying settings and remove pending settings" in { + val workspaceId = workspace.workspaceIdAsUUID + val workspaceName = workspace.toWorkspaceName + val existingSetting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpLifecycleAction("Delete"), GcpLifecycleCondition(Set("prefixToMatch"), Some(30))) + ) + ) + ) + val newSetting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpLifecycleAction("Delete"), GcpLifecycleCondition(Set("muchBetterPrefix"), Some(31))) + ) + ) + ) + + val workspaceRepository = mock[WorkspaceRepository] + when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) + when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(List(existingSetting))) + when(workspaceRepository.createWorkspaceSettingsRecords(workspaceId, List(newSetting))) + .thenReturn(Future.successful(List(newSetting))) + when(workspaceRepository.removePendingSetting(workspaceId, newSetting.`type`)).thenReturn(Future.successful(1)) + + val samDAO = mock[SamDAO] + when(samDAO.getUserStatus(any())) + .thenReturn(Future.successful(Option(SamUserStatusResponse("fake_user_id", "user@example.com", true)))) + when( + samDAO.userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), + ArgumentMatchers.eq(workspaceId.toString), + ArgumentMatchers.eq(SamWorkspaceActions.own), + any() + ) + ).thenReturn(Future.successful(true)) + + val gcsDAO = mock[GoogleServicesDAO] + val newSettingGoogleRule = new LifecycleRule( + LifecycleAction.newDeleteAction(), + LifecycleCondition.newBuilder().setMatchesPrefix(List("muchBetterPrefix").asJava).setAge(31).build() + ) + when(gcsDAO.setBucketLifecycle(workspace.bucketName, List(newSettingGoogleRule))) + .thenReturn(Future.failed(new Exception("failed to apply settings"))) + + val service = workspaceServiceConstructor(samDAO = samDAO, + workspaceRepository = workspaceRepository, + gcsDAO = gcsDAO + )(defaultRequestContext) + + val res = Await.result(service.setWorkspaceSettings(workspaceName, List(newSetting)), Duration.Inf) + res.successes shouldEqual List.empty + res.failures(WorkspaceSettingTypes.GcpBucketLifecycle) shouldEqual ErrorReport(StatusCodes.InternalServerError, + "failed to apply settings" + ) + verify(workspaceRepository).removePendingSetting(workspaceId, newSetting.`type`) + } + + it should "be limited to owners" in { + val workspaceId = workspace.workspaceIdAsUUID + val workspaceName = workspace.toWorkspaceName + val newSetting = WorkspaceSetting(WorkspaceSettingTypes.GcpBucketLifecycle, + WorkspaceSettingTypes.GcpBucketLifecycle.defaultConfig() + ) + + val workspaceRepository = mock[WorkspaceRepository] + when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) + + val samDAO = mock[SamDAO] + when(samDAO.getUserStatus(any())) + .thenReturn(Future.successful(Option(SamUserStatusResponse("fake_user_id", "user@example.com", true)))) + when( + samDAO.userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), + ArgumentMatchers.eq(workspaceId.toString), + ArgumentMatchers.eq(SamWorkspaceActions.own), + any() + ) + ).thenReturn(Future.successful(false)) + // Rawls confirms a user has at least read access to the workspace after a failed authz check + // to determine if it should throw a 403 or 404 + when( + samDAO.userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), + ArgumentMatchers.eq(workspaceId.toString), + ArgumentMatchers.eq(SamWorkspaceActions.read), + any() + ) + ).thenReturn(Future.successful(true)) + + val service = + workspaceServiceConstructor(samDAO = samDAO, workspaceRepository = workspaceRepository)(defaultRequestContext) + + val exception = intercept[RawlsExceptionWithErrorReport] { + Await.result(service.setWorkspaceSettings(workspaceName, List(newSetting)), Duration.Inf) + } + exception.errorReport.statusCode shouldBe Some(StatusCodes.Forbidden) + } + + it should "validate requested settings" in { + val workspaceId = workspace.workspaceIdAsUUID + val workspaceName = workspace.toWorkspaceName + val newSetting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpLifecycleAction("Delete"), GcpLifecycleCondition(Set("prefixToMatch"), Some(-1))) + ) + ) + ) + + val service = workspaceServiceConstructor()(defaultRequestContext) + + val exception = intercept[RawlsExceptionWithErrorReport] { + Await.result(service.setWorkspaceSettings(workspaceName, List(newSetting)), Duration.Inf) + } + exception.errorReport.statusCode shouldBe Some(StatusCodes.BadRequest) + exception.errorReport.message should include("invalid settings requested") + } } From 460df0602c6f4c3bcdedc89dd5eebc64ab333531 Mon Sep 17 00:00:00 2001 From: Marcus Talbott Date: Thu, 8 Aug 2024 16:10:26 -0400 Subject: [PATCH 11/25] repository tests --- .../workspace/WorkspaceRepositorySpec.scala | 226 +++++++++++++++++- 1 file changed, 224 insertions(+), 2 deletions(-) diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepositorySpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepositorySpec.scala index 414e7ce308..b143de4a64 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepositorySpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepositorySpec.scala @@ -2,8 +2,20 @@ package org.broadinstitute.dsde.rawls.workspace import akka.http.scaladsl.model.StatusCodes import org.broadinstitute.dsde.rawls.RawlsExceptionWithErrorReport -import org.broadinstitute.dsde.rawls.dataaccess.slick.TestDriverComponent -import org.broadinstitute.dsde.rawls.model.{Workspace, WorkspaceName, WorkspaceState} +import org.broadinstitute.dsde.rawls.dataaccess.slick.{TestDriverComponent, WorkspaceSettingRecord} +import org.broadinstitute.dsde.rawls.model.WorkspaceSettingConfig.{ + GcpBucketLifecycleConfig, + GcpBucketLifecycleRule, + GcpLifecycleAction, + GcpLifecycleCondition +} +import org.broadinstitute.dsde.rawls.model.{ + Workspace, + WorkspaceName, + WorkspaceSetting, + WorkspaceSettingTypes, + WorkspaceState +} import org.joda.time.DateTime import org.scalatest.concurrent.ScalaFutures import org.scalatest.flatspec.AnyFlatSpec @@ -115,4 +127,214 @@ class WorkspaceRepositorySpec assertResult(readback)(None) assertResult(result)(true) } + + behavior of "getWorkspaceSettings" + + it should "return the applied workspace settings" in { + val repo = new WorkspaceRepository(slickDataSource) + val ws: Workspace = makeWorkspace() + Await.result(repo.createWorkspace(ws), Duration.Inf) + val appliedSetting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpLifecycleAction("Delete"), GcpLifecycleCondition(Set("applied"), Some(30))) + ) + ) + ) + val pendingSetting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpLifecycleAction("Delete"), GcpLifecycleCondition(Set("pending"), Some(31))) + ) + ) + ) + + Await.result( + slickDataSource.inTransaction { dataAccess => + for { + _ <- dataAccess.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, List(appliedSetting)) + _ <- dataAccess.workspaceSettingQuery.updateSettingStatus( + ws.workspaceIdAsUUID, + WorkspaceSettingTypes.GcpBucketLifecycle, + WorkspaceSettingRecord.SettingStatus.Pending, + WorkspaceSettingRecord.SettingStatus.Applied + ) + _ <- dataAccess.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, List(pendingSetting)) + } yield () + }, + Duration.Inf + ) + + val result = Await.result(repo.getWorkspaceSettings(ws.workspaceIdAsUUID), Duration.Inf) + + assertResult(result)(List(appliedSetting)) + } + + behavior of "createWorkspaceSettingsRecords" + + it should "create pending workspace settings" in { + val repo = new WorkspaceRepository(slickDataSource) + val ws: Workspace = makeWorkspace() + Await.result(repo.createWorkspace(ws), Duration.Inf) + val setting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpLifecycleAction("Delete"), GcpLifecycleCondition(Set("newSetting"), Some(30))) + ) + ) + ) + + val result = Await.result(repo.createWorkspaceSettingsRecords(ws.workspaceIdAsUUID, List(setting)), Duration.Inf) + + val newSettings = Await.result( + slickDataSource.inTransaction( + _.workspaceSettingQuery.listSettingsForWorkspaceByStatus(ws.workspaceIdAsUUID, + WorkspaceSettingRecord.SettingStatus.Pending + ) + ), + Duration.Inf + ) + assertResult(newSettings)(List(setting)) + } + + it should "throw an exception if there are already pending settings" in { + val repo = new WorkspaceRepository(slickDataSource) + val ws: Workspace = makeWorkspace() + Await.result(repo.createWorkspace(ws), Duration.Inf) + val setting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpLifecycleAction("Delete"), GcpLifecycleCondition(Set("newSetting"), Some(30))) + ) + ) + ) + + Await.result(slickDataSource.inTransaction(_.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, List(setting))), + Duration.Inf + ) + + val thrown = intercept[RawlsExceptionWithErrorReport] { + Await.result(repo.createWorkspaceSettingsRecords(ws.workspaceIdAsUUID, List(setting)), Duration.Inf) + } + thrown.errorReport.statusCode shouldBe Some(StatusCodes.Conflict) + } + + behavior of "markWorkspaceSettingApplied" + + it should "mark Pending settings as Applied and Applied settings as Deleted" in { + val repo = new WorkspaceRepository(slickDataSource) + val ws: Workspace = makeWorkspace() + Await.result(repo.createWorkspace(ws), Duration.Inf) + val appliedSetting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpLifecycleAction("Delete"), GcpLifecycleCondition(Set("applied"), Some(30))) + ) + ) + ) + val pendingSetting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpLifecycleAction("Delete"), GcpLifecycleCondition(Set("pending"), Some(31))) + ) + ) + ) + + Await.result( + slickDataSource.inTransaction { dataAccess => + for { + _ <- dataAccess.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, List(appliedSetting)) + _ <- dataAccess.workspaceSettingQuery.updateSettingStatus( + ws.workspaceIdAsUUID, + WorkspaceSettingTypes.GcpBucketLifecycle, + WorkspaceSettingRecord.SettingStatus.Pending, + WorkspaceSettingRecord.SettingStatus.Applied + ) + _ <- dataAccess.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, List(pendingSetting)) + } yield () + }, + Duration.Inf + ) + + Await.result(repo.markWorkspaceSettingApplied(ws.workspaceIdAsUUID, WorkspaceSettingTypes.GcpBucketLifecycle), + Duration.Inf + ) + + // existing settings should now be deleted + val deletedSettings = Await.result( + slickDataSource.inTransaction( + _.workspaceSettingQuery.listSettingsForWorkspaceByStatus(ws.workspaceIdAsUUID, + WorkspaceSettingRecord.SettingStatus.Deleted + ) + ), + Duration.Inf + ) + assertResult(deletedSettings)(List(appliedSetting)) + + // new settings should now be applied + val appliedSettings = Await.result( + slickDataSource.inTransaction( + _.workspaceSettingQuery.listSettingsForWorkspaceByStatus(ws.workspaceIdAsUUID, + WorkspaceSettingRecord.SettingStatus.Applied + ) + ), + Duration.Inf + ) + assertResult(appliedSettings)(List(pendingSetting)) + } + + behavior of "removePendingSetting" + + it should "delete the pending workspace setting" in { + val repo = new WorkspaceRepository(slickDataSource) + val ws: Workspace = makeWorkspace() + Await.result(repo.createWorkspace(ws), Duration.Inf) + val setting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpLifecycleAction("Delete"), GcpLifecycleCondition(Set("newSetting"), Some(30))) + ) + ) + ) + + Await.result(slickDataSource.inTransaction(_.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, List(setting))), + Duration.Inf + ) + + Await.result(repo.removePendingSetting(ws.workspaceIdAsUUID, WorkspaceSettingTypes.GcpBucketLifecycle), + Duration.Inf + ) + + Await.result( + slickDataSource.inTransaction( + _.workspaceSettingQuery.listSettingsForWorkspaceByStatus(ws.workspaceIdAsUUID, + WorkspaceSettingRecord.SettingStatus.Deleted + ) + ), + Duration.Inf + ) shouldBe empty + Await.result( + slickDataSource.inTransaction( + _.workspaceSettingQuery.listSettingsForWorkspaceByStatus(ws.workspaceIdAsUUID, + WorkspaceSettingRecord.SettingStatus.Applied + ) + ), + Duration.Inf + ) shouldBe empty + Await.result( + slickDataSource.inTransaction( + _.workspaceSettingQuery.listSettingsForWorkspaceByStatus(ws.workspaceIdAsUUID, + WorkspaceSettingRecord.SettingStatus.Pending + ) + ), + Duration.Inf + ) shouldBe empty + } } From 39342b5b67778c45cbe36c09a94bbe4f82ce033a Mon Sep 17 00:00:00 2001 From: Marcus Talbott Date: Thu, 8 Aug 2024 17:02:33 -0400 Subject: [PATCH 12/25] parsing tests and rename lifecycle case classes --- .../workspace/WorkspaceRepositorySpec.scala | 18 +-- .../workspace/WorkspaceServiceUnitTests.scala | 16 +- .../dsde/rawls/model/WorkspaceModel.scala | 14 +- .../dsde/rawls/model/WorkspaceModelSpec.scala | 139 +++++++++++++++++- 4 files changed, 163 insertions(+), 24 deletions(-) diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepositorySpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepositorySpec.scala index b143de4a64..2ae341e29b 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepositorySpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepositorySpec.scala @@ -6,8 +6,8 @@ import org.broadinstitute.dsde.rawls.dataaccess.slick.{TestDriverComponent, Work import org.broadinstitute.dsde.rawls.model.WorkspaceSettingConfig.{ GcpBucketLifecycleConfig, GcpBucketLifecycleRule, - GcpLifecycleAction, - GcpLifecycleCondition + GcpBucketLifecycleAction, + GcpBucketLifecycleCondition } import org.broadinstitute.dsde.rawls.model.{ Workspace, @@ -138,7 +138,7 @@ class WorkspaceRepositorySpec WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpLifecycleAction("Delete"), GcpLifecycleCondition(Set("applied"), Some(30))) + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("applied"), Some(30))) ) ) ) @@ -146,7 +146,7 @@ class WorkspaceRepositorySpec WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpLifecycleAction("Delete"), GcpLifecycleCondition(Set("pending"), Some(31))) + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("pending"), Some(31))) ) ) ) @@ -182,7 +182,7 @@ class WorkspaceRepositorySpec WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpLifecycleAction("Delete"), GcpLifecycleCondition(Set("newSetting"), Some(30))) + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("newSetting"), Some(30))) ) ) ) @@ -208,7 +208,7 @@ class WorkspaceRepositorySpec WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpLifecycleAction("Delete"), GcpLifecycleCondition(Set("newSetting"), Some(30))) + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("newSetting"), Some(30))) ) ) ) @@ -233,7 +233,7 @@ class WorkspaceRepositorySpec WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpLifecycleAction("Delete"), GcpLifecycleCondition(Set("applied"), Some(30))) + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("applied"), Some(30))) ) ) ) @@ -241,7 +241,7 @@ class WorkspaceRepositorySpec WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpLifecycleAction("Delete"), GcpLifecycleCondition(Set("pending"), Some(31))) + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("pending"), Some(31))) ) ) ) @@ -299,7 +299,7 @@ class WorkspaceRepositorySpec WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpLifecycleAction("Delete"), GcpLifecycleCondition(Set("newSetting"), Some(30))) + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("newSetting"), Some(30))) ) ) ) diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala index ae3abba378..6cf8cfe488 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala @@ -17,8 +17,8 @@ import org.broadinstitute.dsde.rawls.fastpass.FastPassServiceImpl import org.broadinstitute.dsde.rawls.model.WorkspaceSettingConfig.{ GcpBucketLifecycleConfig, GcpBucketLifecycleRule, - GcpLifecycleAction, - GcpLifecycleCondition + GcpBucketLifecycleAction, + GcpBucketLifecycleCondition } import org.broadinstitute.dsde.rawls.model.WorkspaceType.WorkspaceType import org.broadinstitute.dsde.rawls.model._ @@ -942,7 +942,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpLifecycleAction("Delete"), GcpLifecycleCondition(Set("prefixToMatch"), Some(30))) + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("prefixToMatch"), Some(30))) ) ) ) @@ -950,7 +950,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpLifecycleAction("Delete"), GcpLifecycleCondition(Set("muchBetterPrefix"), Some(31))) + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("muchBetterPrefix"), Some(31))) ) ) ) @@ -998,7 +998,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpLifecycleAction("Delete"), GcpLifecycleCondition(Set("prefixToMatch"), Some(30))) + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("prefixToMatch"), Some(30))) ) ) ) @@ -1045,7 +1045,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpLifecycleAction("Delete"), GcpLifecycleCondition(Set("prefixToMatch"), Some(30))) + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("prefixToMatch"), Some(30))) ) ) ) @@ -1053,7 +1053,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpLifecycleAction("Delete"), GcpLifecycleCondition(Set("muchBetterPrefix"), Some(31))) + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("muchBetterPrefix"), Some(31))) ) ) ) @@ -1143,7 +1143,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpLifecycleAction("Delete"), GcpLifecycleCondition(Set("prefixToMatch"), Some(-1))) + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("prefixToMatch"), Some(-1))) ) ) ) diff --git a/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala b/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala index a1f8d81d0d..1d5a3871fc 100644 --- a/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala +++ b/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala @@ -11,7 +11,7 @@ import org.broadinstitute.dsde.rawls.model.SortDirections.SortDirection import org.broadinstitute.dsde.rawls.model.UserModelJsonSupport.ManagedGroupRefFormat import org.broadinstitute.dsde.rawls.model.WorkspaceAccessLevels.WorkspaceAccessLevel import org.broadinstitute.dsde.rawls.model.WorkspaceCloudPlatform.WorkspaceCloudPlatform -import org.broadinstitute.dsde.rawls.model.WorkspaceSettingConfig.{GcpBucketLifecycleConfig, GcpBucketLifecycleRule, GcpLifecycleAction, GcpLifecycleCondition} +import org.broadinstitute.dsde.rawls.model.WorkspaceSettingConfig.{GcpBucketLifecycleConfig, GcpBucketLifecycleRule, GcpBucketLifecycleAction, GcpBucketLifecycleCondition} import org.broadinstitute.dsde.rawls.model.WorkspaceSettingTypes.{GcpBucketLifecycle, WorkspaceSettingType} import org.broadinstitute.dsde.rawls.model.WorkspaceState.WorkspaceState import org.broadinstitute.dsde.rawls.model.WorkspaceType.WorkspaceType @@ -586,9 +586,9 @@ sealed trait WorkspaceSettingConfig object WorkspaceSettingConfig { case class GcpBucketLifecycleConfig(rules: List[GcpBucketLifecycleRule]) extends WorkspaceSettingConfig - case class GcpBucketLifecycleRule(action: GcpLifecycleAction, conditions: GcpLifecycleCondition) - case class GcpLifecycleAction(`type`: String) - case class GcpLifecycleCondition(matchesPrefix: Set[String], age: Option[Int]) + case class GcpBucketLifecycleRule(action: GcpBucketLifecycleAction, conditions: GcpBucketLifecycleCondition) + case class GcpBucketLifecycleAction(`type`: String) + case class GcpBucketLifecycleCondition(matchesPrefix: Set[String], age: Option[Int]) } case class WorkspaceSettingResponse(successes: List[WorkspaceSetting], failures: Map[WorkspaceSettingType, ErrorReport]) @@ -1197,8 +1197,8 @@ class WorkspaceJsonSupport extends JsonSupport { } } - implicit val GcpLifecycleConditionFormat: RootJsonFormat[GcpLifecycleCondition] = jsonFormat2(GcpLifecycleCondition.apply) - implicit val GcpLifecycleActionFormat: RootJsonFormat[GcpLifecycleAction] = jsonFormat1(GcpLifecycleAction.apply) + implicit val GcpLifecycleConditionFormat: RootJsonFormat[GcpBucketLifecycleCondition] = jsonFormat2(GcpBucketLifecycleCondition.apply) + implicit val GcpLifecycleActionFormat: RootJsonFormat[GcpBucketLifecycleAction] = jsonFormat1(GcpBucketLifecycleAction.apply) implicit val GcpBucketLifecycleRuleFormat: RootJsonFormat[GcpBucketLifecycleRule] = jsonFormat2(GcpBucketLifecycleRule.apply) implicit val GcpBucketLifecycleConfigFormat: RootJsonFormat[GcpBucketLifecycleConfig] = jsonFormat1(GcpBucketLifecycleConfig.apply) @@ -1216,6 +1216,8 @@ class WorkspaceJsonSupport extends JsonSupport { case config: GcpBucketLifecycleConfig => config.toJson } + // We prevent reading WorkspaceSettingConfig directly because we need + // the corresponding WorkspaceSettingType to know how to read it def read(json: JsValue): WorkspaceSettingConfig = { throw DeserializationException("WorkspaceSettingConfig cannot be read directly") } diff --git a/model/src/test/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModelSpec.scala b/model/src/test/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModelSpec.scala index d6665fd82e..da03f7bd31 100644 --- a/model/src/test/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModelSpec.scala +++ b/model/src/test/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModelSpec.scala @@ -3,7 +3,8 @@ package org.broadinstitute.dsde.rawls.model import akka.http.scaladsl.model.StatusCodes.BadRequest import org.broadinstitute.dsde.rawls.{RawlsException, RawlsExceptionWithErrorReport} import org.broadinstitute.dsde.rawls.model.Attributable.AttributeMap -import org.broadinstitute.dsde.rawls.model.WorkspaceJsonSupport.MethodRepoMethodFormat +import org.broadinstitute.dsde.rawls.model.WorkspaceJsonSupport.{MethodRepoMethodFormat, WorkspaceSettingFormat} +import org.broadinstitute.dsde.rawls.model.WorkspaceSettingConfig.{GcpBucketLifecycleConfig, GcpBucketLifecycleRule, GcpBucketLifecycleAction, GcpBucketLifecycleCondition} import org.joda.time.DateTime import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers @@ -668,4 +669,140 @@ class WorkspaceModelSpec extends AnyFreeSpec with Matchers { e.errorReport.statusCode shouldBe Some(BadRequest) } } + + "WorkspaceSetting" - { + "throws an exception for invalid workspace setting type" in { + val fakeSetting = + """{ + | "type": "FakeWorkspaceSetting", + | "config": { + | "rules": [] + | } + | }""".stripMargin.parseJson + + intercept[RawlsException] { + WorkspaceSettingFormat.read(fakeSetting) + } + } + + "GoogleBucketLifecycleSettings" - { + "parses lifecycle settings with matchesPrefix and age" in { + val lifecycleSetting = + """{ + | "type": "GcpBucketLifecycle", + | "config": { + | "rules": [ + | { + | "action": { + | "type": "Delete" + | }, + | "conditions": { + | "age": 30, + | "matchesPrefix": [ + | "prefix1", + | "prefix2" + | ] + | } + | } + | ] + | } + | }""".stripMargin.parseJson + assertResult { + WorkspaceSetting(WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig(List(GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("prefix1", "prefix2"), Some(30)))))) + } { + WorkspaceSettingFormat.read(lifecycleSetting) + } + } + + "parses lifecycle settings with no prefixes" in { + val lifecycleSettingNoPrefixes = + """{ + | "type": "GcpBucketLifecycle", + | "config": { + | "rules": [ + | { + | "action": { + | "type": "Delete" + | }, + | "conditions": { + | "age": 30, + | "matchesPrefix": [] + | } + | } + | ] + | } + | }""".stripMargin.parseJson + assertResult { + WorkspaceSetting(WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig(List(GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set.empty, Some(30)))))) + } { + WorkspaceSettingFormat.read(lifecycleSettingNoPrefixes) + } + } + + "parses lifecycle settings with no age" in { + val lifecycleSettingNoAge = + """{ + | "type": "GcpBucketLifecycle", + | "config": { + | "rules": [ + | { + | "action": { + | "type": "Delete" + | }, + | "conditions": { + | "matchesPrefix": [ + | "prefix1", + | "prefix2" + | ] + | } + | } + | ] + | } + | }""".stripMargin.parseJson + assertResult { + WorkspaceSetting(WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig(List(GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("prefix1", "prefix2"), None))))) + } { + WorkspaceSettingFormat.read(lifecycleSettingNoAge) + } + } + + "parses lifecycle settings with no rules" in { + val lifecycleSettingNoRules = + """{ + | "type": "GcpBucketLifecycle", + | "config": { + | "rules": [] + | } + | }""".stripMargin.parseJson + assertResult { + WorkspaceSetting(WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig(List.empty)) + } { + WorkspaceSettingFormat.read(lifecycleSettingNoRules) + } + } + + "throws an exception for missing config" in { + val lifecycleSettingNoConfig = + """{ + | "type": "GcpBucketLifecycle" + | }""".stripMargin.parseJson + intercept[NoSuchElementException] { + WorkspaceSettingFormat.read(lifecycleSettingNoConfig) + } + } + + "throws an exception for incorrect format" in { + val lifecycleSettingBadConfig = + """{ + | "type": "GcpBucketLifecycle", + | "config": { + | "rules": "not a list" + | } + | }""".stripMargin.parseJson + intercept[DeserializationException] { + WorkspaceSettingFormat.read(lifecycleSettingBadConfig) + } + } + } + } } From 029d41db2683d49380b370f9a175eba8776ae96d Mon Sep 17 00:00:00 2001 From: Marcus Talbott Date: Thu, 8 Aug 2024 17:15:33 -0400 Subject: [PATCH 13/25] require config and update lastUpdated when updating setting record --- .../liquibase/changesets/20240723_workspace_settings.xml | 2 +- .../rawls/dataaccess/slick/WorkspaceSettingComponent.scala | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changesets/20240723_workspace_settings.xml b/core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changesets/20240723_workspace_settings.xml index 770a84013b..a797c53844 100644 --- a/core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changesets/20240723_workspace_settings.xml +++ b/core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changesets/20240723_workspace_settings.xml @@ -14,7 +14,7 @@ - + diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala index f36c897697..9e7e8ed202 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala @@ -87,8 +87,8 @@ trait WorkspaceSettingComponent { .filter(record => record.workspaceId === workspaceId && record.`type` === workspaceSettingType.toString && record.status === currentStatus.toString ) - .map(_.status) - .update(newStatus.toString) + .map(rec => (rec.status, rec.lastUpdated)) + .update((newStatus.toString, new Timestamp(new Date().getTime))) def deleteSettingTypeForWorkspaceByStatus(workspaceId: UUID, settingType: WorkspaceSettingType, From 73a61e340ae562c872e1ce2e173e862128fb157c Mon Sep 17 00:00:00 2001 From: Marcus Talbott Date: Fri, 9 Aug 2024 08:54:56 -0400 Subject: [PATCH 14/25] cleanup, move lifecycle action verification earlier --- core/src/main/resources/swagger/api-docs.yaml | 6 ++++++ .../slick/WorkspaceSettingComponent.scala | 8 ++++---- .../dsde/rawls/workspace/WorkspaceService.scala | 14 ++++++++++---- .../rawls/workspace/WorkspaceRepositorySpec.scala | 15 ++++++++------- .../workspace/WorkspaceServiceUnitTests.scala | 12 ++++++------ .../dsde/rawls/model/WorkspaceModel.scala | 4 ++-- 6 files changed, 36 insertions(+), 23 deletions(-) diff --git a/core/src/main/resources/swagger/api-docs.yaml b/core/src/main/resources/swagger/api-docs.yaml index 073a249ce1..f389f5f59f 100644 --- a/core/src/main/resources/swagger/api-docs.yaml +++ b/core/src/main/resources/swagger/api-docs.yaml @@ -4481,6 +4481,12 @@ paths: 'application/json': schema: $ref: '#/components/schemas/WorkspaceSettings' + 400: + description: Invalid settings + content: + 'application/json': + schema: + $ref: '#/components/schemas/ErrorReport' 403: description: User must be an owner of the workspace to overwrite settings content: diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala index 9e7e8ed202..0b6f74a279 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala @@ -40,7 +40,7 @@ object WorkspaceSettingRecord { ) } - def toWorkspaceSettings(workspaceSettingRecord: WorkspaceSettingRecord): WorkspaceSetting = { + def toWorkspaceSetting(workspaceSettingRecord: WorkspaceSettingRecord): WorkspaceSetting = { import spray.json._ val settingType = WorkspaceSettingTypes.withName(workspaceSettingRecord.`type`) @@ -79,13 +79,13 @@ trait WorkspaceSettingComponent { } def updateSettingStatus(workspaceId: UUID, - workspaceSettingType: WorkspaceSettingType, + settingType: WorkspaceSettingType, currentStatus: WorkspaceSettingRecord.SettingStatus.SettingStatus, newStatus: WorkspaceSettingRecord.SettingStatus.SettingStatus ): ReadWriteAction[Int] = workspaceSettingQuery .filter(record => - record.workspaceId === workspaceId && record.`type` === workspaceSettingType.toString && record.status === currentStatus.toString + record.workspaceId === workspaceId && record.`type` === settingType.toString && record.status === currentStatus.toString ) .map(rec => (rec.status, rec.lastUpdated)) .update((newStatus.toString, new Timestamp(new Date().getTime))) @@ -102,6 +102,6 @@ trait WorkspaceSettingComponent { status: WorkspaceSettingRecord.SettingStatus.SettingStatus ): ReadAction[List[WorkspaceSetting]] = filter(rec => rec.workspaceId === workspaceId && rec.status === status.toString).result - .map(_.map(WorkspaceSettingRecord.toWorkspaceSettings).toList) + .map(_.map(WorkspaceSettingRecord.toWorkspaceSetting).toList) } } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala index 5fd9a509fb..5e956f0568 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala @@ -2018,21 +2018,27 @@ class WorkspaceService( * Perform basic validation checks on requested settings. */ def validateSettings(requestedSettings: List[WorkspaceSetting]): Unit = { + def validationErrorReport(settingType: WorkspaceSettingType, reason: String): ErrorReport = ErrorReport(s"Invalid $settingType configuration: $reason.") val validationErrors = requestedSettings.flatMap { case WorkspaceSetting(settingType @ WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig(rules) ) => rules.flatMap { rule => - rule.conditions.age.collect { + val actionValidation = rule.action.`type` match { + case actionType if actionType.equals("Delete") => None + case actionType => Some(validationErrorReport(settingType, s"unsupported lifecycle action $actionType")) + } + val ageValidation = rule.conditions.age.collect { case age if age < 0 => - ErrorReport(s"invalid $settingType configuration: age must be a non-negative integer.") + validationErrorReport(settingType, "age must be a non-negative integer") } + actionValidation ++ ageValidation } } if (validationErrors.nonEmpty) { throw new RawlsExceptionWithErrorReport( - ErrorReport(StatusCodes.BadRequest, "invalid settings requested", validationErrors) + ErrorReport(StatusCodes.BadRequest, "Invalid settings requested.", validationErrors) ) } } @@ -2074,7 +2080,7 @@ class WorkspaceService( val action = rule.action.`type` match { case actionType if actionType.equals("Delete") => LifecycleAction.newDeleteAction() - case _ => throw new RawlsException("unsupported lifecycle action") + case _ => throw new RawlsException("unsupported lifecycle action") // validated earlier but needed for completeness } new LifecycleRule(action, conditionBuilder.build()) diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepositorySpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepositorySpec.scala index 2ae341e29b..63ff0474da 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepositorySpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepositorySpec.scala @@ -187,7 +187,7 @@ class WorkspaceRepositorySpec ) ) - val result = Await.result(repo.createWorkspaceSettingsRecords(ws.workspaceIdAsUUID, List(setting)), Duration.Inf) + Await.result(repo.createWorkspaceSettingsRecords(ws.workspaceIdAsUUID, List(setting)), Duration.Inf) val newSettings = Await.result( slickDataSource.inTransaction( @@ -229,7 +229,7 @@ class WorkspaceRepositorySpec val repo = new WorkspaceRepository(slickDataSource) val ws: Workspace = makeWorkspace() Await.result(repo.createWorkspace(ws), Duration.Inf) - val appliedSetting = WorkspaceSetting( + val existingSetting = WorkspaceSetting( WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( @@ -237,7 +237,7 @@ class WorkspaceRepositorySpec ) ) ) - val pendingSetting = WorkspaceSetting( + val newSetting = WorkspaceSetting( WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( @@ -249,14 +249,14 @@ class WorkspaceRepositorySpec Await.result( slickDataSource.inTransaction { dataAccess => for { - _ <- dataAccess.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, List(appliedSetting)) + _ <- dataAccess.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, List(existingSetting)) _ <- dataAccess.workspaceSettingQuery.updateSettingStatus( ws.workspaceIdAsUUID, WorkspaceSettingTypes.GcpBucketLifecycle, WorkspaceSettingRecord.SettingStatus.Pending, WorkspaceSettingRecord.SettingStatus.Applied ) - _ <- dataAccess.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, List(pendingSetting)) + _ <- dataAccess.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, List(newSetting)) } yield () }, Duration.Inf @@ -275,7 +275,7 @@ class WorkspaceRepositorySpec ), Duration.Inf ) - assertResult(deletedSettings)(List(appliedSetting)) + assertResult(deletedSettings)(List(existingSetting)) // new settings should now be applied val appliedSettings = Await.result( @@ -286,7 +286,7 @@ class WorkspaceRepositorySpec ), Duration.Inf ) - assertResult(appliedSettings)(List(pendingSetting)) + assertResult(appliedSettings)(List(newSetting)) } behavior of "removePendingSetting" @@ -312,6 +312,7 @@ class WorkspaceRepositorySpec Duration.Inf ) + // There should be no workspace settings in the database. Failed pending settings are not kept. Await.result( slickDataSource.inTransaction( _.workspaceSettingQuery.listSettingsForWorkspaceByStatus(ws.workspaceIdAsUUID, diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala index 6cf8cfe488..63c3c9fc34 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala @@ -2,12 +2,9 @@ package org.broadinstitute.dsde.rawls.workspace import akka.http.scaladsl.model.StatusCodes import akka.http.scaladsl.model.headers.OAuth2BearerToken -import akka.stream.Materializer import bio.terra.workspace.model.{IamRole, RoleBinding, RoleBindingList} -import com.google.api.client.googleapis.json.GoogleJsonResponseException import com.google.cloud.storage.BucketInfo.LifecycleRule import com.google.cloud.storage.BucketInfo.LifecycleRule.{LifecycleAction, LifecycleCondition} -import org.broadinstitute.dsde.rawls import org.broadinstitute.dsde.rawls.billing.{BillingProfileManagerDAO, BillingRepository} import org.broadinstitute.dsde.rawls.config._ import org.broadinstitute.dsde.rawls.dataaccess._ @@ -1137,13 +1134,12 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki } it should "validate requested settings" in { - val workspaceId = workspace.workspaceIdAsUUID val workspaceName = workspace.toWorkspaceName val newSetting = WorkspaceSetting( WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("prefixToMatch"), Some(-1))) + GcpBucketLifecycleRule(GcpBucketLifecycleAction("SetStorageClass"), GcpBucketLifecycleCondition(Set("prefixToMatch"), Some(-1))) ) ) ) @@ -1154,6 +1150,10 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki Await.result(service.setWorkspaceSettings(workspaceName, List(newSetting)), Duration.Inf) } exception.errorReport.statusCode shouldBe Some(StatusCodes.BadRequest) - exception.errorReport.message should include("invalid settings requested") + exception.errorReport.message should include("Invalid settings requested.") + exception.errorReport.causes should contain theSameElementsAs List( + ErrorReport("Invalid GcpBucketLifecycle configuration: age must be a non-negative integer."), + ErrorReport("Invalid GcpBucketLifecycle configuration: unsupported lifecycle action SetStorageClass.") + ) } } diff --git a/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala b/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala index 1d5a3871fc..aa32c9ca2a 100644 --- a/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala +++ b/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala @@ -1197,8 +1197,8 @@ class WorkspaceJsonSupport extends JsonSupport { } } - implicit val GcpLifecycleConditionFormat: RootJsonFormat[GcpBucketLifecycleCondition] = jsonFormat2(GcpBucketLifecycleCondition.apply) - implicit val GcpLifecycleActionFormat: RootJsonFormat[GcpBucketLifecycleAction] = jsonFormat1(GcpBucketLifecycleAction.apply) + implicit val GcpBucketLifecycleConditionFormat: RootJsonFormat[GcpBucketLifecycleCondition] = jsonFormat2(GcpBucketLifecycleCondition.apply) + implicit val GcpBucketLifecycleActionFormat: RootJsonFormat[GcpBucketLifecycleAction] = jsonFormat1(GcpBucketLifecycleAction.apply) implicit val GcpBucketLifecycleRuleFormat: RootJsonFormat[GcpBucketLifecycleRule] = jsonFormat2(GcpBucketLifecycleRule.apply) implicit val GcpBucketLifecycleConfigFormat: RootJsonFormat[GcpBucketLifecycleConfig] = jsonFormat1(GcpBucketLifecycleConfig.apply) From 5178b1e84d12e115f6bbb3c5cd80922ffc6f67da Mon Sep 17 00:00:00 2001 From: Marcus Talbott Date: Fri, 9 Aug 2024 08:59:21 -0400 Subject: [PATCH 15/25] scalafmt --- .../rawls/workspace/WorkspaceService.scala | 11 +++- .../workspace/WorkspaceRepositorySpec.scala | 34 +++++++---- .../workspace/WorkspaceServiceUnitTests.scala | 30 +++++++--- .../dsde/rawls/model/WorkspaceModel.scala | 56 ++++++++++++------- 4 files changed, 88 insertions(+), 43 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala index 5e956f0568..0f693c4f99 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala @@ -2018,7 +2018,9 @@ class WorkspaceService( * Perform basic validation checks on requested settings. */ def validateSettings(requestedSettings: List[WorkspaceSetting]): Unit = { - def validationErrorReport(settingType: WorkspaceSettingType, reason: String): ErrorReport = ErrorReport(s"Invalid $settingType configuration: $reason.") + def validationErrorReport(settingType: WorkspaceSettingType, reason: String): ErrorReport = ErrorReport( + s"Invalid $settingType configuration: $reason." + ) val validationErrors = requestedSettings.flatMap { case WorkspaceSetting(settingType @ WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig(rules) @@ -2026,7 +2028,7 @@ class WorkspaceService( rules.flatMap { rule => val actionValidation = rule.action.`type` match { case actionType if actionType.equals("Delete") => None - case actionType => Some(validationErrorReport(settingType, s"unsupported lifecycle action $actionType")) + case actionType => Some(validationErrorReport(settingType, s"unsupported lifecycle action $actionType")) } val ageValidation = rule.conditions.age.collect { case age if age < 0 => @@ -2080,7 +2082,10 @@ class WorkspaceService( val action = rule.action.`type` match { case actionType if actionType.equals("Delete") => LifecycleAction.newDeleteAction() - case _ => throw new RawlsException("unsupported lifecycle action") // validated earlier but needed for completeness + case _ => + throw new RawlsException( + "unsupported lifecycle action" + ) // validated earlier but needed for completeness } new LifecycleRule(action, conditionBuilder.build()) diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepositorySpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepositorySpec.scala index 63ff0474da..b9222ca96d 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepositorySpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepositorySpec.scala @@ -4,10 +4,10 @@ import akka.http.scaladsl.model.StatusCodes import org.broadinstitute.dsde.rawls.RawlsExceptionWithErrorReport import org.broadinstitute.dsde.rawls.dataaccess.slick.{TestDriverComponent, WorkspaceSettingRecord} import org.broadinstitute.dsde.rawls.model.WorkspaceSettingConfig.{ - GcpBucketLifecycleConfig, - GcpBucketLifecycleRule, GcpBucketLifecycleAction, - GcpBucketLifecycleCondition + GcpBucketLifecycleCondition, + GcpBucketLifecycleConfig, + GcpBucketLifecycleRule } import org.broadinstitute.dsde.rawls.model.{ Workspace, @@ -138,7 +138,9 @@ class WorkspaceRepositorySpec WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("applied"), Some(30))) + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), + GcpBucketLifecycleCondition(Set("applied"), Some(30)) + ) ) ) ) @@ -146,7 +148,9 @@ class WorkspaceRepositorySpec WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("pending"), Some(31))) + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), + GcpBucketLifecycleCondition(Set("pending"), Some(31)) + ) ) ) ) @@ -182,7 +186,9 @@ class WorkspaceRepositorySpec WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("newSetting"), Some(30))) + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), + GcpBucketLifecycleCondition(Set("newSetting"), Some(30)) + ) ) ) ) @@ -208,7 +214,9 @@ class WorkspaceRepositorySpec WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("newSetting"), Some(30))) + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), + GcpBucketLifecycleCondition(Set("newSetting"), Some(30)) + ) ) ) ) @@ -233,7 +241,9 @@ class WorkspaceRepositorySpec WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("applied"), Some(30))) + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), + GcpBucketLifecycleCondition(Set("applied"), Some(30)) + ) ) ) ) @@ -241,7 +251,9 @@ class WorkspaceRepositorySpec WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("pending"), Some(31))) + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), + GcpBucketLifecycleCondition(Set("pending"), Some(31)) + ) ) ) ) @@ -299,7 +311,9 @@ class WorkspaceRepositorySpec WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("newSetting"), Some(30))) + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), + GcpBucketLifecycleCondition(Set("newSetting"), Some(30)) + ) ) ) ) diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala index 63c3c9fc34..5057ca15de 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala @@ -12,10 +12,10 @@ import org.broadinstitute.dsde.rawls.dataaccess.leonardo.LeonardoService import org.broadinstitute.dsde.rawls.dataaccess.workspacemanager.WorkspaceManagerDAO import org.broadinstitute.dsde.rawls.fastpass.FastPassServiceImpl import org.broadinstitute.dsde.rawls.model.WorkspaceSettingConfig.{ - GcpBucketLifecycleConfig, - GcpBucketLifecycleRule, GcpBucketLifecycleAction, - GcpBucketLifecycleCondition + GcpBucketLifecycleCondition, + GcpBucketLifecycleConfig, + GcpBucketLifecycleRule } import org.broadinstitute.dsde.rawls.model.WorkspaceType.WorkspaceType import org.broadinstitute.dsde.rawls.model._ @@ -939,7 +939,9 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("prefixToMatch"), Some(30))) + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), + GcpBucketLifecycleCondition(Set("prefixToMatch"), Some(30)) + ) ) ) ) @@ -947,7 +949,9 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("muchBetterPrefix"), Some(31))) + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), + GcpBucketLifecycleCondition(Set("muchBetterPrefix"), Some(31)) + ) ) ) ) @@ -995,7 +999,9 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("prefixToMatch"), Some(30))) + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), + GcpBucketLifecycleCondition(Set("prefixToMatch"), Some(30)) + ) ) ) ) @@ -1042,7 +1048,9 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("prefixToMatch"), Some(30))) + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), + GcpBucketLifecycleCondition(Set("prefixToMatch"), Some(30)) + ) ) ) ) @@ -1050,7 +1058,9 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("muchBetterPrefix"), Some(31))) + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), + GcpBucketLifecycleCondition(Set("muchBetterPrefix"), Some(31)) + ) ) ) ) @@ -1139,7 +1149,9 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("SetStorageClass"), GcpBucketLifecycleCondition(Set("prefixToMatch"), Some(-1))) + GcpBucketLifecycleRule(GcpBucketLifecycleAction("SetStorageClass"), + GcpBucketLifecycleCondition(Set("prefixToMatch"), Some(-1)) + ) ) ) ) diff --git a/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala b/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala index aa32c9ca2a..4c9dfc35fa 100644 --- a/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala +++ b/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala @@ -11,7 +11,12 @@ import org.broadinstitute.dsde.rawls.model.SortDirections.SortDirection import org.broadinstitute.dsde.rawls.model.UserModelJsonSupport.ManagedGroupRefFormat import org.broadinstitute.dsde.rawls.model.WorkspaceAccessLevels.WorkspaceAccessLevel import org.broadinstitute.dsde.rawls.model.WorkspaceCloudPlatform.WorkspaceCloudPlatform -import org.broadinstitute.dsde.rawls.model.WorkspaceSettingConfig.{GcpBucketLifecycleConfig, GcpBucketLifecycleRule, GcpBucketLifecycleAction, GcpBucketLifecycleCondition} +import org.broadinstitute.dsde.rawls.model.WorkspaceSettingConfig.{ + GcpBucketLifecycleAction, + GcpBucketLifecycleCondition, + GcpBucketLifecycleConfig, + GcpBucketLifecycleRule +} import org.broadinstitute.dsde.rawls.model.WorkspaceSettingTypes.{GcpBucketLifecycle, WorkspaceSettingType} import org.broadinstitute.dsde.rawls.model.WorkspaceState.WorkspaceState import org.broadinstitute.dsde.rawls.model.WorkspaceType.WorkspaceType @@ -566,20 +571,20 @@ object WorkspaceState { case class WorkspaceSetting(`type`: WorkspaceSettingType, config: WorkspaceSettingConfig) object WorkspaceSettingTypes { - sealed trait WorkspaceSettingType extends RawlsEnumeration[WorkspaceSettingType] { - override def toString: String = getClass.getSimpleName.stripSuffix("$") - override def withName(name: String): WorkspaceSettingType = WorkspaceSettingTypes.withName(name) - def defaultConfig(): WorkspaceSettingConfig - } + sealed trait WorkspaceSettingType extends RawlsEnumeration[WorkspaceSettingType] { + override def toString: String = getClass.getSimpleName.stripSuffix("$") + override def withName(name: String): WorkspaceSettingType = WorkspaceSettingTypes.withName(name) + def defaultConfig(): WorkspaceSettingConfig + } - def withName(name: String): WorkspaceSettingType = name.toLowerCase match { - case "gcpbucketlifecycle" => GcpBucketLifecycle - case _ => throw new RawlsException(s"invalid WorkspaceSetting [$name]") - } + def withName(name: String): WorkspaceSettingType = name.toLowerCase match { + case "gcpbucketlifecycle" => GcpBucketLifecycle + case _ => throw new RawlsException(s"invalid WorkspaceSetting [$name]") + } - case object GcpBucketLifecycle extends WorkspaceSettingType { - override def defaultConfig(): WorkspaceSettingConfig = GcpBucketLifecycleConfig(List.empty) - } + case object GcpBucketLifecycle extends WorkspaceSettingType { + override def defaultConfig(): WorkspaceSettingConfig = GcpBucketLifecycleConfig(List.empty) + } } sealed trait WorkspaceSettingConfig @@ -1197,10 +1202,18 @@ class WorkspaceJsonSupport extends JsonSupport { } } - implicit val GcpBucketLifecycleConditionFormat: RootJsonFormat[GcpBucketLifecycleCondition] = jsonFormat2(GcpBucketLifecycleCondition.apply) - implicit val GcpBucketLifecycleActionFormat: RootJsonFormat[GcpBucketLifecycleAction] = jsonFormat1(GcpBucketLifecycleAction.apply) - implicit val GcpBucketLifecycleRuleFormat: RootJsonFormat[GcpBucketLifecycleRule] = jsonFormat2(GcpBucketLifecycleRule.apply) - implicit val GcpBucketLifecycleConfigFormat: RootJsonFormat[GcpBucketLifecycleConfig] = jsonFormat1(GcpBucketLifecycleConfig.apply) + implicit val GcpBucketLifecycleConditionFormat: RootJsonFormat[GcpBucketLifecycleCondition] = jsonFormat2( + GcpBucketLifecycleCondition.apply + ) + implicit val GcpBucketLifecycleActionFormat: RootJsonFormat[GcpBucketLifecycleAction] = jsonFormat1( + GcpBucketLifecycleAction.apply + ) + implicit val GcpBucketLifecycleRuleFormat: RootJsonFormat[GcpBucketLifecycleRule] = jsonFormat2( + GcpBucketLifecycleRule.apply + ) + implicit val GcpBucketLifecycleConfigFormat: RootJsonFormat[GcpBucketLifecycleConfig] = jsonFormat1( + GcpBucketLifecycleConfig.apply + ) implicit object WorkspaceSettingTypeFormat extends RootJsonFormat[WorkspaceSettingType] { override def write(obj: WorkspaceSettingType): JsValue = JsString(obj.toString) @@ -1218,9 +1231,8 @@ class WorkspaceJsonSupport extends JsonSupport { // We prevent reading WorkspaceSettingConfig directly because we need // the corresponding WorkspaceSettingType to know how to read it - def read(json: JsValue): WorkspaceSettingConfig = { + def read(json: JsValue): WorkspaceSettingConfig = throw DeserializationException("WorkspaceSettingConfig cannot be read directly") - } } implicit object WorkspaceSettingFormat extends RootJsonFormat[WorkspaceSetting] { @@ -1234,7 +1246,7 @@ class WorkspaceJsonSupport extends JsonSupport { val settingType = fields("type").convertTo[WorkspaceSettingType] val configuration = settingType match { case GcpBucketLifecycle => fields("config").convertTo[GcpBucketLifecycleConfig] - case _ => throw DeserializationException(s"unexpected setting type $settingType") + case _ => throw DeserializationException(s"unexpected setting type $settingType") } WorkspaceSetting(settingType, configuration) } @@ -1482,7 +1494,9 @@ class WorkspaceJsonSupport extends JsonSupport { ) ) - implicit val WorkspaceSettingResponseFormat: RootJsonFormat[WorkspaceSettingResponse] = jsonFormat2(WorkspaceSettingResponse) + implicit val WorkspaceSettingResponseFormat: RootJsonFormat[WorkspaceSettingResponse] = jsonFormat2( + WorkspaceSettingResponse + ) implicit val ApplicationVersionFormat: RootJsonFormat[ApplicationVersion] = jsonFormat3(ApplicationVersion) } From 2c30dbaf6e1399766fd41780c4f2b647b4690d5e Mon Sep 17 00:00:00 2001 From: Marcus Talbott Date: Fri, 9 Aug 2024 14:52:06 -0400 Subject: [PATCH 16/25] change transaction isolation level, add comments, track user who made settings change, enforce at least one condition, make prefixes an Option --- .../20240723_workspace_settings.xml | 3 + .../slick/WorkspaceSettingComponent.scala | 21 +++-- .../rawls/workspace/WorkspaceRepository.scala | 41 +++++--- .../rawls/workspace/WorkspaceService.scala | 24 ++++- .../workspace/WorkspaceRepositorySpec.scala | 47 +++++++--- .../workspace/WorkspaceServiceUnitTests.scala | 40 ++++++-- .../dsde/rawls/model/WorkspaceModel.scala | 3 +- .../dsde/rawls/model/WorkspaceModelSpec.scala | 93 ++++++++++++++++++- 8 files changed, 216 insertions(+), 56 deletions(-) diff --git a/core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changesets/20240723_workspace_settings.xml b/core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changesets/20240723_workspace_settings.xml index a797c53844..a9216906a4 100644 --- a/core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changesets/20240723_workspace_settings.xml +++ b/core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changesets/20240723_workspace_settings.xml @@ -25,6 +25,9 @@ + + + diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala index 0b6f74a279..6a0c4d72b1 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala @@ -13,7 +13,8 @@ case class WorkspaceSettingRecord(`type`: String, config: String, status: String, createdTime: Timestamp, - lastUpdated: Timestamp + lastUpdated: Timestamp, + user: String ) object WorkspaceSettingRecord { @@ -24,7 +25,10 @@ object WorkspaceSettingRecord { val Deleted: Value = Value("Deleted") } - def toWorkspaceSettingRecord(workspaceId: UUID, workspaceSettings: WorkspaceSetting): WorkspaceSettingRecord = { + def toWorkspaceSettingRecord(workspaceId: UUID, + workspaceSettings: WorkspaceSetting, + user: RawlsUserSubjectId + ): WorkspaceSettingRecord = { import spray.json._ import DefaultJsonProtocol._ import WorkspaceJsonSupport._ @@ -36,7 +40,8 @@ object WorkspaceSettingRecord { configString, WorkspaceSettingRecord.SettingStatus.Pending.toString, currentTime, - currentTime + currentTime, + user.value ) } @@ -64,17 +69,19 @@ trait WorkspaceSettingComponent { def status = column[String]("status", O.Length(254)) def createdTime = column[Timestamp]("created_time") def lastUpdated = column[Timestamp]("last_updated") + def user = column[String]("user", O.Length(254)) - def * = (`type`, workspaceId, config, status, createdTime, lastUpdated) <> (WorkspaceSettingRecord.tupled, - WorkspaceSettingRecord.unapply + def * = (`type`, workspaceId, config, status, createdTime, lastUpdated, user) <> (WorkspaceSettingRecord.tupled, + WorkspaceSettingRecord.unapply ) } object workspaceSettingQuery extends TableQuery(new WorkspaceSettingTable(_)) { def saveAll(workspaceId: UUID, - workspaceSettings: List[WorkspaceSetting] + workspaceSettings: List[WorkspaceSetting], + user: RawlsUserSubjectId ): ReadWriteAction[List[WorkspaceSetting]] = { - val records = workspaceSettings.map(WorkspaceSettingRecord.toWorkspaceSettingRecord(workspaceId, _)) + val records = workspaceSettings.map(WorkspaceSettingRecord.toWorkspaceSettingRecord(workspaceId, _, user)) (workspaceSettingQuery ++= records).map(_ => workspaceSettings) } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala index 85ca49b562..72e8400aae 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala @@ -8,6 +8,7 @@ import org.broadinstitute.dsde.rawls.model.Attributable.AttributeMap import org.broadinstitute.dsde.rawls.model.{ ErrorReport, RawlsRequestContext, + RawlsUserSubjectId, Workspace, WorkspaceAttributeSpecs, WorkspaceName, @@ -18,6 +19,7 @@ import org.broadinstitute.dsde.rawls.model.WorkspaceState.WorkspaceState import org.broadinstitute.dsde.rawls.model.WorkspaceSettingTypes.WorkspaceSettingType import org.broadinstitute.dsde.rawls.util.TracingUtils.traceDBIOWithParent import org.joda.time.DateTime +import slick.jdbc.TransactionIsolation import java.util.UUID import scala.concurrent.{ExecutionContext, Future} @@ -102,6 +104,7 @@ class WorkspaceRepository(dataSource: SlickDataSource) { } yield newWorkspace } + // Return all applied settings for a workspace. Deleted and pending settings are not returned. def getWorkspaceSettings(workspaceId: UUID): Future[List[WorkspaceSetting]] = dataSource.inTransaction { access => access.workspaceSettingQuery.listSettingsForWorkspaceByStatus(workspaceId, @@ -109,23 +112,32 @@ class WorkspaceRepository(dataSource: SlickDataSource) { ) } - def createWorkspaceSettingsRecords(workspaceId: UUID, workspaceSettings: List[WorkspaceSetting])(implicit + // Create new settings for a workspace as pending. If there are any existing pending settings, throw an exception. + def createWorkspaceSettingsRecords(workspaceId: UUID, + workspaceSettings: List[WorkspaceSetting], + user: RawlsUserSubjectId + )(implicit ec: ExecutionContext ): Future[List[WorkspaceSetting]] = - dataSource.inTransaction { access => - for { - pendingSettingsForWorkspace <- access.workspaceSettingQuery.listSettingsForWorkspaceByStatus( - workspaceId, - WorkspaceSettingRecord.SettingStatus.Pending - ) - _ = if (pendingSettingsForWorkspace.nonEmpty) { - throw new RawlsExceptionWithErrorReport( - ErrorReport(StatusCodes.Conflict, s"Workspace $workspaceId already has pending settings") + dataSource.inTransaction( + access => + for { + pendingSettingsForWorkspace <- access.workspaceSettingQuery.listSettingsForWorkspaceByStatus( + workspaceId, + WorkspaceSettingRecord.SettingStatus.Pending ) - } - _ <- access.workspaceSettingQuery.saveAll(workspaceId, workspaceSettings) - } yield workspaceSettings - } + _ = if (pendingSettingsForWorkspace.nonEmpty) { + throw new RawlsExceptionWithErrorReport( + ErrorReport(StatusCodes.Conflict, s"Workspace $workspaceId already has pending settings") + ) + } + _ <- access.workspaceSettingQuery.saveAll(workspaceId, workspaceSettings, user) + } yield workspaceSettings + // We use Serializable here to ensure that concurrent transactions to create settings + // for the same workspace are committed in order and do not interfere with each other + , + TransactionIsolation.Serializable + ) // Transition old Applied settings to Deleted and Pending settings to Applied def markWorkspaceSettingApplied(workspaceId: UUID, workspaceSettingType: WorkspaceSettingType)(implicit @@ -146,6 +158,7 @@ class WorkspaceRepository(dataSource: SlickDataSource) { } yield res } + // Fully remove all pending records for a workspace from database. Does not transition records to Deleted. def removePendingSetting(workspaceId: UUID, workspaceSettingType: WorkspaceSettingType): Future[Int] = dataSource.inTransaction { access => access.workspaceSettingQuery.deleteSettingTypeForWorkspaceByStatus(workspaceId, diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala index 0f693c4f99..c4fc179218 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala @@ -2034,7 +2034,12 @@ class WorkspaceService( case age if age < 0 => validationErrorReport(settingType, "age must be a non-negative integer") } - actionValidation ++ ageValidation + val atLeastOneConditionValidation = (rule.conditions.age, rule.conditions.matchesPrefix) match { + case (None, None) => + Some(validationErrorReport(settingType, "at least one condition must be specified")) + case _ => None + } + actionValidation ++ ageValidation ++ atLeastOneConditionValidation } } @@ -2076,16 +2081,18 @@ class WorkspaceService( (setting match { case WorkspaceSetting(WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig(rules)) => val googleRules = rules.map { rule => - val conditionBuilder = - LifecycleCondition.newBuilder().setMatchesPrefix(rule.conditions.matchesPrefix.toList.asJava) + val conditionBuilder = LifecycleCondition.newBuilder() + rule.conditions.matchesPrefix.map(prefixes => conditionBuilder.setMatchesPrefix(prefixes.toList.asJava)) rule.conditions.age.map(age => conditionBuilder.setAge(age)) val action = rule.action.`type` match { case actionType if actionType.equals("Delete") => LifecycleAction.newDeleteAction() + + // validated earlier but needed for completeness case _ => throw new RawlsException( "unsupported lifecycle action" - ) // validated earlier but needed for completeness + ) } new LifecycleRule(action, conditionBuilder.build()) @@ -2097,6 +2104,10 @@ class WorkspaceService( } yield None case _ => throw new RawlsException("unsupported workspace setting") }).recoverWith { case e => + logger.error( + s"Failed to apply settings. [workspaceId=${workspace.workspaceIdAsUUID},settingType=${setting.`type`}]", + e + ) workspaceRepository .removePendingSetting(workspace.workspaceIdAsUUID, setting.`type`) .map(_ => Some((setting.`type`, ErrorReport(StatusCodes.InternalServerError, e.getMessage)))) @@ -2107,7 +2118,10 @@ class WorkspaceService( workspace <- getV2WorkspaceContextAndPermissions(workspaceName, SamWorkspaceActions.own) currentSettings <- workspaceRepository.getWorkspaceSettings(workspace.workspaceIdAsUUID) newSettings = computeNewSettings(workspace, workspaceSettings, currentSettings) - _ <- workspaceRepository.createWorkspaceSettingsRecords(workspace.workspaceIdAsUUID, workspaceSettings) + _ <- workspaceRepository.createWorkspaceSettingsRecords(workspace.workspaceIdAsUUID, + workspaceSettings, + ctx.userInfo.userSubjectId + ) applyFailures <- newSettings.traverse(s => applySetting(workspace, s)) } yield { val successes = newSettings.filterNot { s => diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepositorySpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepositorySpec.scala index b9222ca96d..7c48a541bf 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepositorySpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepositorySpec.scala @@ -139,7 +139,7 @@ class WorkspaceRepositorySpec GcpBucketLifecycleConfig( List( GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), - GcpBucketLifecycleCondition(Set("applied"), Some(30)) + GcpBucketLifecycleCondition(Some(Set("applied")), Some(30)) ) ) ) @@ -149,7 +149,7 @@ class WorkspaceRepositorySpec GcpBucketLifecycleConfig( List( GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), - GcpBucketLifecycleCondition(Set("pending"), Some(31)) + GcpBucketLifecycleCondition(Some(Set("pending")), Some(31)) ) ) ) @@ -158,14 +158,20 @@ class WorkspaceRepositorySpec Await.result( slickDataSource.inTransaction { dataAccess => for { - _ <- dataAccess.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, List(appliedSetting)) + _ <- dataAccess.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, + List(appliedSetting), + userInfo.userSubjectId + ) _ <- dataAccess.workspaceSettingQuery.updateSettingStatus( ws.workspaceIdAsUUID, WorkspaceSettingTypes.GcpBucketLifecycle, WorkspaceSettingRecord.SettingStatus.Pending, WorkspaceSettingRecord.SettingStatus.Applied ) - _ <- dataAccess.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, List(pendingSetting)) + _ <- dataAccess.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, + List(pendingSetting), + userInfo.userSubjectId + ) } yield () }, Duration.Inf @@ -187,13 +193,15 @@ class WorkspaceRepositorySpec GcpBucketLifecycleConfig( List( GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), - GcpBucketLifecycleCondition(Set("newSetting"), Some(30)) + GcpBucketLifecycleCondition(Some(Set("newSetting")), Some(30)) ) ) ) ) - Await.result(repo.createWorkspaceSettingsRecords(ws.workspaceIdAsUUID, List(setting)), Duration.Inf) + Await.result(repo.createWorkspaceSettingsRecords(ws.workspaceIdAsUUID, List(setting), userInfo.userSubjectId), + Duration.Inf + ) val newSettings = Await.result( slickDataSource.inTransaction( @@ -215,18 +223,22 @@ class WorkspaceRepositorySpec GcpBucketLifecycleConfig( List( GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), - GcpBucketLifecycleCondition(Set("newSetting"), Some(30)) + GcpBucketLifecycleCondition(Some(Set("newSetting")), Some(30)) ) ) ) ) - Await.result(slickDataSource.inTransaction(_.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, List(setting))), + Await.result(slickDataSource.inTransaction( + _.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, List(setting), userInfo.userSubjectId) + ), Duration.Inf ) val thrown = intercept[RawlsExceptionWithErrorReport] { - Await.result(repo.createWorkspaceSettingsRecords(ws.workspaceIdAsUUID, List(setting)), Duration.Inf) + Await.result(repo.createWorkspaceSettingsRecords(ws.workspaceIdAsUUID, List(setting), userInfo.userSubjectId), + Duration.Inf + ) } thrown.errorReport.statusCode shouldBe Some(StatusCodes.Conflict) } @@ -242,7 +254,7 @@ class WorkspaceRepositorySpec GcpBucketLifecycleConfig( List( GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), - GcpBucketLifecycleCondition(Set("applied"), Some(30)) + GcpBucketLifecycleCondition(Some(Set("applied")), Some(30)) ) ) ) @@ -252,7 +264,7 @@ class WorkspaceRepositorySpec GcpBucketLifecycleConfig( List( GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), - GcpBucketLifecycleCondition(Set("pending"), Some(31)) + GcpBucketLifecycleCondition(Some(Set("pending")), Some(31)) ) ) ) @@ -261,14 +273,17 @@ class WorkspaceRepositorySpec Await.result( slickDataSource.inTransaction { dataAccess => for { - _ <- dataAccess.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, List(existingSetting)) + _ <- dataAccess.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, + List(existingSetting), + userInfo.userSubjectId + ) _ <- dataAccess.workspaceSettingQuery.updateSettingStatus( ws.workspaceIdAsUUID, WorkspaceSettingTypes.GcpBucketLifecycle, WorkspaceSettingRecord.SettingStatus.Pending, WorkspaceSettingRecord.SettingStatus.Applied ) - _ <- dataAccess.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, List(newSetting)) + _ <- dataAccess.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, List(newSetting), userInfo.userSubjectId) } yield () }, Duration.Inf @@ -312,13 +327,15 @@ class WorkspaceRepositorySpec GcpBucketLifecycleConfig( List( GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), - GcpBucketLifecycleCondition(Set("newSetting"), Some(30)) + GcpBucketLifecycleCondition(Some(Set("newSetting")), Some(30)) ) ) ) ) - Await.result(slickDataSource.inTransaction(_.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, List(setting))), + Await.result(slickDataSource.inTransaction( + _.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, List(setting), userInfo.userSubjectId) + ), Duration.Inf ) diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala index 5057ca15de..47159be63c 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala @@ -903,7 +903,12 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki val workspaceRepository = mock[WorkspaceRepository] when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(List.empty)) - when(workspaceRepository.createWorkspaceSettingsRecords(workspaceId, List(workspaceSetting))) + when( + workspaceRepository.createWorkspaceSettingsRecords(workspaceId, + List(workspaceSetting), + defaultRequestContext.userInfo.userSubjectId + ) + ) .thenReturn(Future.successful(List(workspaceSetting))) when(workspaceRepository.markWorkspaceSettingApplied(workspaceId, workspaceSetting.`type`)) .thenReturn(Future.successful(1)) @@ -940,7 +945,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki GcpBucketLifecycleConfig( List( GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), - GcpBucketLifecycleCondition(Set("prefixToMatch"), Some(30)) + GcpBucketLifecycleCondition(Some(Set("prefixToMatch")), Some(30)) ) ) ) @@ -950,7 +955,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki GcpBucketLifecycleConfig( List( GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), - GcpBucketLifecycleCondition(Set("muchBetterPrefix"), Some(31)) + GcpBucketLifecycleCondition(Some(Set("muchBetterPrefix")), Some(31)) ) ) ) @@ -959,7 +964,12 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki val workspaceRepository = mock[WorkspaceRepository] when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(List(existingSetting))) - when(workspaceRepository.createWorkspaceSettingsRecords(workspaceId, List(newSetting))) + when( + workspaceRepository.createWorkspaceSettingsRecords(workspaceId, + List(newSetting), + defaultRequestContext.userInfo.userSubjectId + ) + ) .thenReturn(Future.successful(List(newSetting))) when(workspaceRepository.markWorkspaceSettingApplied(workspaceId, newSetting.`type`)) .thenReturn(Future.successful(1)) @@ -1000,7 +1010,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki GcpBucketLifecycleConfig( List( GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), - GcpBucketLifecycleCondition(Set("prefixToMatch"), Some(30)) + GcpBucketLifecycleCondition(Some(Set("prefixToMatch")), Some(30)) ) ) ) @@ -1012,7 +1022,12 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki val workspaceRepository = mock[WorkspaceRepository] when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(List(existingSetting))) - when(workspaceRepository.createWorkspaceSettingsRecords(workspaceId, List.empty)) + when( + workspaceRepository.createWorkspaceSettingsRecords(workspaceId, + List.empty, + defaultRequestContext.userInfo.userSubjectId + ) + ) .thenReturn(Future.successful(List.empty)) when(workspaceRepository.markWorkspaceSettingApplied(workspaceId, defaultSetting.`type`)) .thenReturn(Future.successful(1)) @@ -1049,7 +1064,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki GcpBucketLifecycleConfig( List( GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), - GcpBucketLifecycleCondition(Set("prefixToMatch"), Some(30)) + GcpBucketLifecycleCondition(Some(Set("prefixToMatch")), Some(30)) ) ) ) @@ -1059,7 +1074,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki GcpBucketLifecycleConfig( List( GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), - GcpBucketLifecycleCondition(Set("muchBetterPrefix"), Some(31)) + GcpBucketLifecycleCondition(Some(Set("muchBetterPrefix")), Some(31)) ) ) ) @@ -1068,7 +1083,12 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki val workspaceRepository = mock[WorkspaceRepository] when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(List(existingSetting))) - when(workspaceRepository.createWorkspaceSettingsRecords(workspaceId, List(newSetting))) + when( + workspaceRepository.createWorkspaceSettingsRecords(workspaceId, + List(newSetting), + defaultRequestContext.userInfo.userSubjectId + ) + ) .thenReturn(Future.successful(List(newSetting))) when(workspaceRepository.removePendingSetting(workspaceId, newSetting.`type`)).thenReturn(Future.successful(1)) @@ -1150,7 +1170,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki GcpBucketLifecycleConfig( List( GcpBucketLifecycleRule(GcpBucketLifecycleAction("SetStorageClass"), - GcpBucketLifecycleCondition(Set("prefixToMatch"), Some(-1)) + GcpBucketLifecycleCondition(Some(Set("prefixToMatch")), Some(-1)) ) ) ) diff --git a/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala b/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala index 4c9dfc35fa..a26bb71640 100644 --- a/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala +++ b/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala @@ -589,11 +589,12 @@ object WorkspaceSettingTypes { sealed trait WorkspaceSettingConfig object WorkspaceSettingConfig { + type Days = Int case class GcpBucketLifecycleConfig(rules: List[GcpBucketLifecycleRule]) extends WorkspaceSettingConfig case class GcpBucketLifecycleRule(action: GcpBucketLifecycleAction, conditions: GcpBucketLifecycleCondition) case class GcpBucketLifecycleAction(`type`: String) - case class GcpBucketLifecycleCondition(matchesPrefix: Set[String], age: Option[Int]) + case class GcpBucketLifecycleCondition(matchesPrefix: Option[Set[String]], age: Option[Days]) } case class WorkspaceSettingResponse(successes: List[WorkspaceSetting], failures: Map[WorkspaceSettingType, ErrorReport]) diff --git a/model/src/test/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModelSpec.scala b/model/src/test/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModelSpec.scala index da03f7bd31..2359e46351 100644 --- a/model/src/test/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModelSpec.scala +++ b/model/src/test/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModelSpec.scala @@ -4,7 +4,12 @@ import akka.http.scaladsl.model.StatusCodes.BadRequest import org.broadinstitute.dsde.rawls.{RawlsException, RawlsExceptionWithErrorReport} import org.broadinstitute.dsde.rawls.model.Attributable.AttributeMap import org.broadinstitute.dsde.rawls.model.WorkspaceJsonSupport.{MethodRepoMethodFormat, WorkspaceSettingFormat} -import org.broadinstitute.dsde.rawls.model.WorkspaceSettingConfig.{GcpBucketLifecycleConfig, GcpBucketLifecycleRule, GcpBucketLifecycleAction, GcpBucketLifecycleCondition} +import org.broadinstitute.dsde.rawls.model.WorkspaceSettingConfig.{ + GcpBucketLifecycleAction, + GcpBucketLifecycleCondition, + GcpBucketLifecycleConfig, + GcpBucketLifecycleRule +} import org.joda.time.DateTime import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers @@ -708,7 +713,16 @@ class WorkspaceModelSpec extends AnyFreeSpec with Matchers { | } | }""".stripMargin.parseJson assertResult { - WorkspaceSetting(WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig(List(GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("prefix1", "prefix2"), Some(30)))))) + WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), + GcpBucketLifecycleCondition(Some(Set("prefix1", "prefix2")), Some(30)) + ) + ) + ) + ) } { WorkspaceSettingFormat.read(lifecycleSetting) } @@ -733,7 +747,16 @@ class WorkspaceModelSpec extends AnyFreeSpec with Matchers { | } | }""".stripMargin.parseJson assertResult { - WorkspaceSetting(WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig(List(GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set.empty, Some(30)))))) + WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), + GcpBucketLifecycleCondition(Some(Set.empty), Some(30)) + ) + ) + ) + ) } { WorkspaceSettingFormat.read(lifecycleSettingNoPrefixes) } @@ -760,7 +783,16 @@ class WorkspaceModelSpec extends AnyFreeSpec with Matchers { | } | }""".stripMargin.parseJson assertResult { - WorkspaceSetting(WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig(List(GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Set("prefix1", "prefix2"), None))))) + WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), + GcpBucketLifecycleCondition(Some(Set("prefix1", "prefix2")), None) + ) + ) + ) + ) } { WorkspaceSettingFormat.read(lifecycleSettingNoAge) } @@ -803,6 +835,59 @@ class WorkspaceModelSpec extends AnyFreeSpec with Matchers { WorkspaceSettingFormat.read(lifecycleSettingBadConfig) } } + + "throws an exception for missing rules" in { + val lifecycleSettingNoRules = + """{ + | "type": "GcpBucketLifecycle", + | "config": {} + | }""".stripMargin.parseJson + intercept[DeserializationException] { + WorkspaceSettingFormat.read(lifecycleSettingNoRules) + } + } + + "throws an exception for missing rule action" in { + val lifecycleSettingNoAction = + """{ + | "type": "GcpBucketLifecycle", + | "config": { + | "rules": [ + | { + | "conditions": { + | "age": 30, + | "matchesPrefix": [ + | "prefix1", + | "prefix2" + | ] + | } + | } + | ] + | } + | }""".stripMargin.parseJson + intercept[DeserializationException] { + WorkspaceSettingFormat.read(lifecycleSettingNoAction) + } + } + + "throws an exception for missing rule conditions" in { + val lifecycleSettingNoConditions = + """{ + | "type": "GcpBucketLifecycle", + | "config": { + | "rules": [ + | { + | "action": { + | "type": "Delete" + | } + | } + | ] + | } + | }""".stripMargin.parseJson + intercept[DeserializationException] { + WorkspaceSettingFormat.read(lifecycleSettingNoConditions) + } + } } } } From 5dde8bbca8f0e7ca74e633538bc9601811ca81c4 Mon Sep 17 00:00:00 2001 From: Marcus Talbott Date: Fri, 9 Aug 2024 15:00:41 -0400 Subject: [PATCH 17/25] no backticks --- .../20240723_workspace_settings.xml | 2 +- core/src/main/resources/swagger/api-docs.yaml | 8 ++++---- .../slick/WorkspaceSettingComponent.scala | 14 ++++++------- .../rawls/workspace/WorkspaceService.scala | 18 +++++++++-------- .../workspace/WorkspaceServiceUnitTests.scala | 10 +++++----- .../dsde/rawls/model/WorkspaceModel.scala | 10 +++++----- .../dsde/rawls/model/WorkspaceModelSpec.scala | 20 +++++++++---------- 7 files changed, 42 insertions(+), 40 deletions(-) diff --git a/core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changesets/20240723_workspace_settings.xml b/core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changesets/20240723_workspace_settings.xml index a9216906a4..d7fd3234ac 100644 --- a/core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changesets/20240723_workspace_settings.xml +++ b/core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changesets/20240723_workspace_settings.xml @@ -10,7 +10,7 @@ - + diff --git a/core/src/main/resources/swagger/api-docs.yaml b/core/src/main/resources/swagger/api-docs.yaml index f389f5f59f..e25bca6031 100644 --- a/core/src/main/resources/swagger/api-docs.yaml +++ b/core/src/main/resources/swagger/api-docs.yaml @@ -5898,11 +5898,11 @@ components: description: "" WorkspaceSettings: required: - - type + - settingType - config type: object properties: - 'type': + settingType: type: string description: The type of the workspace setting enum: @@ -5933,10 +5933,10 @@ components: $ref: '#/components/schemas/GcpBucketLifecycleCondition' GcpBucketLifecycleAction: required: - - type + - actionType type: object properties: - "type": + actionType: type: string description: The type of the lifecycle action enum: diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala index 6a0c4d72b1..6575d04dd8 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala @@ -8,7 +8,7 @@ import org.broadinstitute.dsde.rawls.model._ import java.sql.Timestamp import java.util.{Date, UUID} -case class WorkspaceSettingRecord(`type`: String, +case class WorkspaceSettingRecord(settingType: String, workspaceId: UUID, config: String, status: String, @@ -35,7 +35,7 @@ object WorkspaceSettingRecord { val currentTime = new Timestamp(new Date().getTime) val configString = workspaceSettings.config.toJson.compactPrint - WorkspaceSettingRecord(workspaceSettings.`type`.toString, + WorkspaceSettingRecord(workspaceSettings.settingType.toString, workspaceId, configString, WorkspaceSettingRecord.SettingStatus.Pending.toString, @@ -48,7 +48,7 @@ object WorkspaceSettingRecord { def toWorkspaceSetting(workspaceSettingRecord: WorkspaceSettingRecord): WorkspaceSetting = { import spray.json._ - val settingType = WorkspaceSettingTypes.withName(workspaceSettingRecord.`type`) + val settingType = WorkspaceSettingTypes.withName(workspaceSettingRecord.settingType) val settingConfig = settingType match { case WorkspaceSettingTypes.GcpBucketLifecycle => workspaceSettingRecord.config.parseJson.convertTo[GcpBucketLifecycleConfig] @@ -63,7 +63,7 @@ trait WorkspaceSettingComponent { import driver.api._ class WorkspaceSettingTable(tag: Tag) extends Table[WorkspaceSettingRecord](tag, "WORKSPACE_SETTINGS") { - def `type` = column[String]("type", O.Length(254)) + def settingType = column[String]("setting_type", O.Length(254)) def workspaceId = column[UUID]("workspace_id") def config = column[String]("config") def status = column[String]("status", O.Length(254)) @@ -71,7 +71,7 @@ trait WorkspaceSettingComponent { def lastUpdated = column[Timestamp]("last_updated") def user = column[String]("user", O.Length(254)) - def * = (`type`, workspaceId, config, status, createdTime, lastUpdated, user) <> (WorkspaceSettingRecord.tupled, + def * = (settingType, workspaceId, config, status, createdTime, lastUpdated, user) <> (WorkspaceSettingRecord.tupled, WorkspaceSettingRecord.unapply ) } @@ -92,7 +92,7 @@ trait WorkspaceSettingComponent { ): ReadWriteAction[Int] = workspaceSettingQuery .filter(record => - record.workspaceId === workspaceId && record.`type` === settingType.toString && record.status === currentStatus.toString + record.workspaceId === workspaceId && record.settingType === settingType.toString && record.status === currentStatus.toString ) .map(rec => (rec.status, rec.lastUpdated)) .update((newStatus.toString, new Timestamp(new Date().getTime))) @@ -102,7 +102,7 @@ trait WorkspaceSettingComponent { status: WorkspaceSettingRecord.SettingStatus.SettingStatus ): ReadWriteAction[Int] = filter(record => - record.workspaceId === workspaceId && record.`type` === settingType.toString && record.status === status.toString + record.workspaceId === workspaceId && record.settingType === settingType.toString && record.status === status.toString ).delete def listSettingsForWorkspaceByStatus(workspaceId: UUID, diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala index c4fc179218..332ca15267 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala @@ -2026,7 +2026,7 @@ class WorkspaceService( GcpBucketLifecycleConfig(rules) ) => rules.flatMap { rule => - val actionValidation = rule.action.`type` match { + val actionValidation = rule.action.actionType match { case actionType if actionType.equals("Delete") => None case actionType => Some(validationErrorReport(settingType, s"unsupported lifecycle action $actionType")) } @@ -2034,6 +2034,7 @@ class WorkspaceService( case age if age < 0 => validationErrorReport(settingType, "age must be a non-negative integer") } + // todo: what if matches prefix is defined, but it's an empty list? val atLeastOneConditionValidation = (rule.conditions.age, rule.conditions.matchesPrefix) match { case (None, None) => Some(validationErrorReport(settingType, "at least one condition must be specified")) @@ -2062,7 +2063,8 @@ class WorkspaceService( if (existingSettings.isEmpty) { requestedSettings } else { - val requestedSettingsMap = requestedSettings.map(s => s.`type` -> s.config).toMap + // todo: protect against identical settings + val requestedSettingsMap = requestedSettings.map(s => s.settingType -> s.config).toMap existingSettings.map { case WorkspaceSetting(settingType, _) => WorkspaceSetting(settingType, requestedSettingsMap.getOrElse(settingType, settingType.defaultConfig())) } @@ -2085,7 +2087,7 @@ class WorkspaceService( rule.conditions.matchesPrefix.map(prefixes => conditionBuilder.setMatchesPrefix(prefixes.toList.asJava)) rule.conditions.age.map(age => conditionBuilder.setAge(age)) - val action = rule.action.`type` match { + val action = rule.action.actionType match { case actionType if actionType.equals("Delete") => LifecycleAction.newDeleteAction() // validated earlier but needed for completeness @@ -2100,17 +2102,17 @@ class WorkspaceService( for { _ <- gcsDAO.setBucketLifecycle(workspace.bucketName, googleRules) - _ <- workspaceRepository.markWorkspaceSettingApplied(workspace.workspaceIdAsUUID, setting.`type`) + _ <- workspaceRepository.markWorkspaceSettingApplied(workspace.workspaceIdAsUUID, setting.settingType) } yield None case _ => throw new RawlsException("unsupported workspace setting") }).recoverWith { case e => logger.error( - s"Failed to apply settings. [workspaceId=${workspace.workspaceIdAsUUID},settingType=${setting.`type`}]", + s"Failed to apply settings. [workspaceId=${workspace.workspaceIdAsUUID},settingType=${setting.settingType}]", e ) workspaceRepository - .removePendingSetting(workspace.workspaceIdAsUUID, setting.`type`) - .map(_ => Some((setting.`type`, ErrorReport(StatusCodes.InternalServerError, e.getMessage)))) + .removePendingSetting(workspace.workspaceIdAsUUID, setting.settingType) + .map(_ => Some((setting.settingType, ErrorReport(StatusCodes.InternalServerError, e.getMessage)))) } validateSettings(workspaceSettings) @@ -2126,7 +2128,7 @@ class WorkspaceService( } yield { val successes = newSettings.filterNot { s => applyFailures.flatten.exists { case (failedSettingType, _) => - failedSettingType == s.`type` + failedSettingType == s.settingType } } WorkspaceSettingResponse(successes, applyFailures.flatten.toMap) diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala index 47159be63c..b9da9c2d3f 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala @@ -910,7 +910,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki ) ) .thenReturn(Future.successful(List(workspaceSetting))) - when(workspaceRepository.markWorkspaceSettingApplied(workspaceId, workspaceSetting.`type`)) + when(workspaceRepository.markWorkspaceSettingApplied(workspaceId, workspaceSetting.settingType)) .thenReturn(Future.successful(1)) val samDAO = mock[SamDAO] @@ -971,7 +971,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki ) ) .thenReturn(Future.successful(List(newSetting))) - when(workspaceRepository.markWorkspaceSettingApplied(workspaceId, newSetting.`type`)) + when(workspaceRepository.markWorkspaceSettingApplied(workspaceId, newSetting.settingType)) .thenReturn(Future.successful(1)) val samDAO = mock[SamDAO] @@ -1029,7 +1029,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki ) ) .thenReturn(Future.successful(List.empty)) - when(workspaceRepository.markWorkspaceSettingApplied(workspaceId, defaultSetting.`type`)) + when(workspaceRepository.markWorkspaceSettingApplied(workspaceId, defaultSetting.settingType)) .thenReturn(Future.successful(1)) val samDAO = mock[SamDAO] @@ -1090,7 +1090,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki ) ) .thenReturn(Future.successful(List(newSetting))) - when(workspaceRepository.removePendingSetting(workspaceId, newSetting.`type`)).thenReturn(Future.successful(1)) + when(workspaceRepository.removePendingSetting(workspaceId, newSetting.settingType)).thenReturn(Future.successful(1)) val samDAO = mock[SamDAO] when(samDAO.getUserStatus(any())) @@ -1121,7 +1121,7 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki res.failures(WorkspaceSettingTypes.GcpBucketLifecycle) shouldEqual ErrorReport(StatusCodes.InternalServerError, "failed to apply settings" ) - verify(workspaceRepository).removePendingSetting(workspaceId, newSetting.`type`) + verify(workspaceRepository).removePendingSetting(workspaceId, newSetting.settingType) } it should "be limited to owners" in { diff --git a/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala b/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala index a26bb71640..1b1f6bf72d 100644 --- a/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala +++ b/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala @@ -568,7 +568,7 @@ object WorkspaceState { case object DeleteFailed extends WorkspaceState } -case class WorkspaceSetting(`type`: WorkspaceSettingType, config: WorkspaceSettingConfig) +case class WorkspaceSetting(settingType: WorkspaceSettingType, config: WorkspaceSettingConfig) object WorkspaceSettingTypes { sealed trait WorkspaceSettingType extends RawlsEnumeration[WorkspaceSettingType] { @@ -593,7 +593,7 @@ object WorkspaceSettingConfig { case class GcpBucketLifecycleConfig(rules: List[GcpBucketLifecycleRule]) extends WorkspaceSettingConfig case class GcpBucketLifecycleRule(action: GcpBucketLifecycleAction, conditions: GcpBucketLifecycleCondition) - case class GcpBucketLifecycleAction(`type`: String) + case class GcpBucketLifecycleAction(actionType: String) case class GcpBucketLifecycleCondition(matchesPrefix: Option[Set[String]], age: Option[Days]) } @@ -1221,7 +1221,7 @@ class WorkspaceJsonSupport extends JsonSupport { override def read(json: JsValue): WorkspaceSettingType = json match { case JsString(name) => WorkspaceSettingTypes.withName(name) - case _ => throw DeserializationException("unexpected json type") + case _ => throw DeserializationException("unexpected setting type $settingType") } } @@ -1238,13 +1238,13 @@ class WorkspaceJsonSupport extends JsonSupport { implicit object WorkspaceSettingFormat extends RootJsonFormat[WorkspaceSetting] { def write(ws: WorkspaceSetting): JsValue = JsObject( - "type" -> ws.`type`.toJson, + "settingType" -> ws.settingType.toJson, "config" -> ws.config.toJson ) def read(json: JsValue): WorkspaceSetting = { val fields = json.asJsObject.fields - val settingType = fields("type").convertTo[WorkspaceSettingType] + val settingType = fields("settingType").convertTo[WorkspaceSettingType] val configuration = settingType match { case GcpBucketLifecycle => fields("config").convertTo[GcpBucketLifecycleConfig] case _ => throw DeserializationException(s"unexpected setting type $settingType") diff --git a/model/src/test/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModelSpec.scala b/model/src/test/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModelSpec.scala index 2359e46351..bfaf845e24 100644 --- a/model/src/test/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModelSpec.scala +++ b/model/src/test/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModelSpec.scala @@ -679,7 +679,7 @@ class WorkspaceModelSpec extends AnyFreeSpec with Matchers { "throws an exception for invalid workspace setting type" in { val fakeSetting = """{ - | "type": "FakeWorkspaceSetting", + | "settingType": "FakeWorkspaceSetting", | "config": { | "rules": [] | } @@ -694,7 +694,7 @@ class WorkspaceModelSpec extends AnyFreeSpec with Matchers { "parses lifecycle settings with matchesPrefix and age" in { val lifecycleSetting = """{ - | "type": "GcpBucketLifecycle", + | "settingType": "GcpBucketLifecycle", | "config": { | "rules": [ | { @@ -731,7 +731,7 @@ class WorkspaceModelSpec extends AnyFreeSpec with Matchers { "parses lifecycle settings with no prefixes" in { val lifecycleSettingNoPrefixes = """{ - | "type": "GcpBucketLifecycle", + | "settingType": "GcpBucketLifecycle", | "config": { | "rules": [ | { @@ -765,7 +765,7 @@ class WorkspaceModelSpec extends AnyFreeSpec with Matchers { "parses lifecycle settings with no age" in { val lifecycleSettingNoAge = """{ - | "type": "GcpBucketLifecycle", + | "settingType": "GcpBucketLifecycle", | "config": { | "rules": [ | { @@ -801,7 +801,7 @@ class WorkspaceModelSpec extends AnyFreeSpec with Matchers { "parses lifecycle settings with no rules" in { val lifecycleSettingNoRules = """{ - | "type": "GcpBucketLifecycle", + | "settingType": "GcpBucketLifecycle", | "config": { | "rules": [] | } @@ -816,7 +816,7 @@ class WorkspaceModelSpec extends AnyFreeSpec with Matchers { "throws an exception for missing config" in { val lifecycleSettingNoConfig = """{ - | "type": "GcpBucketLifecycle" + | "settingType": "GcpBucketLifecycle" | }""".stripMargin.parseJson intercept[NoSuchElementException] { WorkspaceSettingFormat.read(lifecycleSettingNoConfig) @@ -826,7 +826,7 @@ class WorkspaceModelSpec extends AnyFreeSpec with Matchers { "throws an exception for incorrect format" in { val lifecycleSettingBadConfig = """{ - | "type": "GcpBucketLifecycle", + | "settingType": "GcpBucketLifecycle", | "config": { | "rules": "not a list" | } @@ -839,7 +839,7 @@ class WorkspaceModelSpec extends AnyFreeSpec with Matchers { "throws an exception for missing rules" in { val lifecycleSettingNoRules = """{ - | "type": "GcpBucketLifecycle", + | "settingType": "GcpBucketLifecycle", | "config": {} | }""".stripMargin.parseJson intercept[DeserializationException] { @@ -850,7 +850,7 @@ class WorkspaceModelSpec extends AnyFreeSpec with Matchers { "throws an exception for missing rule action" in { val lifecycleSettingNoAction = """{ - | "type": "GcpBucketLifecycle", + | "settingType": "GcpBucketLifecycle", | "config": { | "rules": [ | { @@ -873,7 +873,7 @@ class WorkspaceModelSpec extends AnyFreeSpec with Matchers { "throws an exception for missing rule conditions" in { val lifecycleSettingNoConditions = """{ - | "type": "GcpBucketLifecycle", + | "settingType": "GcpBucketLifecycle", | "config": { | "rules": [ | { From b6e416a572de341bfd333d5ad1b96036dcb54581 Mon Sep 17 00:00:00 2001 From: Marcus Talbott Date: Mon, 12 Aug 2024 14:01:08 -0400 Subject: [PATCH 18/25] don't fully overwrite, update; correct field name; handle identical settings --- .../slick/WorkspaceSettingComponent.scala | 30 ++++++++++--------- .../rawls/workspace/WorkspaceRepository.scala | 4 +-- .../rawls/workspace/WorkspaceService.scala | 30 +++++-------------- 3 files changed, 26 insertions(+), 38 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala index 6575d04dd8..81f4ff1ef7 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceSettingComponent.scala @@ -14,7 +14,7 @@ case class WorkspaceSettingRecord(settingType: String, status: String, createdTime: Timestamp, lastUpdated: Timestamp, - user: String + userId: String ) object WorkspaceSettingRecord { @@ -27,7 +27,7 @@ object WorkspaceSettingRecord { def toWorkspaceSettingRecord(workspaceId: UUID, workspaceSettings: WorkspaceSetting, - user: RawlsUserSubjectId + userId: RawlsUserSubjectId ): WorkspaceSettingRecord = { import spray.json._ import DefaultJsonProtocol._ @@ -35,13 +35,14 @@ object WorkspaceSettingRecord { val currentTime = new Timestamp(new Date().getTime) val configString = workspaceSettings.config.toJson.compactPrint - WorkspaceSettingRecord(workspaceSettings.settingType.toString, - workspaceId, - configString, - WorkspaceSettingRecord.SettingStatus.Pending.toString, - currentTime, - currentTime, - user.value + WorkspaceSettingRecord( + workspaceSettings.settingType.toString, + workspaceId, + configString, + WorkspaceSettingRecord.SettingStatus.Pending.toString, + currentTime, + currentTime, + userId.value ) } @@ -69,19 +70,20 @@ trait WorkspaceSettingComponent { def status = column[String]("status", O.Length(254)) def createdTime = column[Timestamp]("created_time") def lastUpdated = column[Timestamp]("last_updated") - def user = column[String]("user", O.Length(254)) + def userId = column[String]("user_id", O.Length(254)) - def * = (settingType, workspaceId, config, status, createdTime, lastUpdated, user) <> (WorkspaceSettingRecord.tupled, - WorkspaceSettingRecord.unapply + def * = (settingType, workspaceId, config, status, createdTime, lastUpdated, userId) <> ( + WorkspaceSettingRecord.tupled, + WorkspaceSettingRecord.unapply ) } object workspaceSettingQuery extends TableQuery(new WorkspaceSettingTable(_)) { def saveAll(workspaceId: UUID, workspaceSettings: List[WorkspaceSetting], - user: RawlsUserSubjectId + userId: RawlsUserSubjectId ): ReadWriteAction[List[WorkspaceSetting]] = { - val records = workspaceSettings.map(WorkspaceSettingRecord.toWorkspaceSettingRecord(workspaceId, _, user)) + val records = workspaceSettings.map(WorkspaceSettingRecord.toWorkspaceSettingRecord(workspaceId, _, userId)) (workspaceSettingQuery ++= records).map(_ => workspaceSettings) } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala index 72e8400aae..8eea32dd58 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala @@ -115,7 +115,7 @@ class WorkspaceRepository(dataSource: SlickDataSource) { // Create new settings for a workspace as pending. If there are any existing pending settings, throw an exception. def createWorkspaceSettingsRecords(workspaceId: UUID, workspaceSettings: List[WorkspaceSetting], - user: RawlsUserSubjectId + userId: RawlsUserSubjectId )(implicit ec: ExecutionContext ): Future[List[WorkspaceSetting]] = @@ -131,7 +131,7 @@ class WorkspaceRepository(dataSource: SlickDataSource) { ErrorReport(StatusCodes.Conflict, s"Workspace $workspaceId already has pending settings") ) } - _ <- access.workspaceSettingQuery.saveAll(workspaceId, workspaceSettings, user) + _ <- access.workspaceSettingQuery.saveAll(workspaceId, workspaceSettings, userId) } yield workspaceSettings // We use Serializable here to ensure that concurrent transactions to create settings // for the same workspace are committed in order and do not interfere with each other diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala index 332ca15267..99f962489f 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala @@ -2034,10 +2034,15 @@ class WorkspaceService( case age if age < 0 => validationErrorReport(settingType, "age must be a non-negative integer") } - // todo: what if matches prefix is defined, but it's an empty list? val atLeastOneConditionValidation = (rule.conditions.age, rule.conditions.matchesPrefix) match { case (None, None) => Some(validationErrorReport(settingType, "at least one condition must be specified")) + case (None, Some(prefixes)) if prefixes.isEmpty => + Some( + validationErrorReport(settingType, + "at least one prefix must be specified if matchesPrefix is the only condition" + ) + ) case _ => None } actionValidation ++ ageValidation ++ atLeastOneConditionValidation @@ -2051,25 +2056,6 @@ class WorkspaceService( } } - /** - * Iterate over existing settings. If an existing setting type is included in the - * requestedSettings, use the requested setting. If it isn't, use the setting type's default - * config to restore the workspace to the default state. - */ - def computeNewSettings(workspace: Workspace, - requestedSettings: List[WorkspaceSetting], - existingSettings: List[WorkspaceSetting] - ): List[WorkspaceSetting] = - if (existingSettings.isEmpty) { - requestedSettings - } else { - // todo: protect against identical settings - val requestedSettingsMap = requestedSettings.map(s => s.settingType -> s.config).toMap - existingSettings.map { case WorkspaceSetting(settingType, _) => - WorkspaceSetting(settingType, requestedSettingsMap.getOrElse(settingType, settingType.defaultConfig())) - } - } - /** * Apply a setting to a workspace. If the setting is successfully applied, update the database * and return None. If the setting fails to apply, remove the failed setting from the database @@ -2119,9 +2105,9 @@ class WorkspaceService( for { workspace <- getV2WorkspaceContextAndPermissions(workspaceName, SamWorkspaceActions.own) currentSettings <- workspaceRepository.getWorkspaceSettings(workspace.workspaceIdAsUUID) - newSettings = computeNewSettings(workspace, workspaceSettings, currentSettings) + newSettings = workspaceSettings.filterNot(currentSettings.contains(_)) _ <- workspaceRepository.createWorkspaceSettingsRecords(workspace.workspaceIdAsUUID, - workspaceSettings, + newSettings, ctx.userInfo.userSubjectId ) applyFailures <- newSettings.traverse(s => applySetting(workspace, s)) From 7b538be4afe27403ee1428a529e30ee786039370 Mon Sep 17 00:00:00 2001 From: Marcus Talbott Date: Mon, 12 Aug 2024 17:27:36 -0400 Subject: [PATCH 19/25] separate settings service --- .../org/broadinstitute/dsde/rawls/Boot.scala | 7 +- .../rawls/webservice/RawlsApiService.scala | 3 +- .../webservice/WorkspaceApiServiceV2.scala | 7 +- .../rawls/workspace/WorkspaceService.scala | 116 ----- .../workspace/WorkspaceSettingService.scala | 149 ++++++ .../rawls/webservice/ApiServiceSpec.scala | 6 +- .../workspace/WorkspaceServiceUnitTests.scala | 397 ---------------- .../WorkspaceSettingServiceUnitTests.scala | 434 ++++++++++++++++++ 8 files changed, 600 insertions(+), 519 deletions(-) create mode 100644 core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingService.scala create mode 100644 core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingServiceUnitTests.scala diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/Boot.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/Boot.scala index c31af936de..88f143b80e 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/Boot.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/Boot.scala @@ -61,7 +61,8 @@ import org.broadinstitute.dsde.rawls.workspace.{ MultiCloudWorkspaceService, RawlsWorkspaceAclManager, WorkspaceRepository, - WorkspaceService + WorkspaceService, + WorkspaceSettingService } import org.broadinstitute.dsde.workbench.google.GoogleCredentialModes.Json import org.broadinstitute.dsde.workbench.google.{GoogleIamDAO, GoogleStorageDAO} @@ -510,9 +511,13 @@ object Boot extends IOApp with LazyLogging { val bucketMigrationServiceConstructor: RawlsRequestContext => BucketMigrationService = BucketMigrationServiceFactory.createBucketMigrationService(appConfigManager, slickDataSource, samDAO, gcsDAO) + val workspaceSettingServiceConstructor: RawlsRequestContext => WorkspaceSettingService = + new WorkspaceSettingService(_, workspaceRepository, gcsDAO, samDAO) + val service = new RawlsApiServiceImpl( multiCloudWorkspaceServiceConstructor, workspaceServiceConstructor, + workspaceSettingServiceConstructor, entityServiceConstructor, userServiceConstructor, genomicsServiceConstructor, diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/RawlsApiService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/RawlsApiService.scala index a2f60c0315..9b3d05d7c7 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/RawlsApiService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/RawlsApiService.scala @@ -32,7 +32,7 @@ import org.broadinstitute.dsde.rawls.spendreporting.SpendReportingService import org.broadinstitute.dsde.rawls.status.StatusService import org.broadinstitute.dsde.rawls.submissions.SubmissionsService import org.broadinstitute.dsde.rawls.user.UserService -import org.broadinstitute.dsde.rawls.workspace.{MultiCloudWorkspaceService, WorkspaceService} +import org.broadinstitute.dsde.rawls.workspace.{MultiCloudWorkspaceService, WorkspaceService, WorkspaceSettingService} import org.broadinstitute.dsde.workbench.oauth2.OpenIDConnectConfiguration import java.sql.{SQLException, SQLTransactionRollbackException} @@ -212,6 +212,7 @@ trait VersionApiService { class RawlsApiServiceImpl(val multiCloudWorkspaceServiceConstructor: RawlsRequestContext => MultiCloudWorkspaceService, val workspaceServiceConstructor: RawlsRequestContext => WorkspaceService, + val workspaceSettingServiceConstructor: RawlsRequestContext => WorkspaceSettingService, val entityServiceConstructor: RawlsRequestContext => EntityService, val userServiceConstructor: RawlsRequestContext => UserService, val genomicsServiceConstructor: RawlsRequestContext => GenomicsService, diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/WorkspaceApiServiceV2.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/WorkspaceApiServiceV2.scala index 7f49d10c87..3b1d2937e7 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/WorkspaceApiServiceV2.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/WorkspaceApiServiceV2.scala @@ -11,7 +11,7 @@ import org.broadinstitute.dsde.rawls.model._ import org.broadinstitute.dsde.rawls.monitor.migration.MultiregionalBucketMigrationJsonSupport._ import org.broadinstitute.dsde.rawls.openam.UserInfoDirectives import org.broadinstitute.dsde.rawls.webservice.CustomDirectives.addLocationHeader -import org.broadinstitute.dsde.rawls.workspace.{MultiCloudWorkspaceService, WorkspaceService} +import org.broadinstitute.dsde.rawls.workspace.{MultiCloudWorkspaceService, WorkspaceService, WorkspaceSettingService} import spray.json.DefaultJsonProtocol._ import spray.json._ @@ -23,6 +23,7 @@ trait WorkspaceApiServiceV2 extends UserInfoDirectives { val workspaceServiceConstructor: RawlsRequestContext => WorkspaceService val multiCloudWorkspaceServiceConstructor: RawlsRequestContext => MultiCloudWorkspaceService val bucketMigrationServiceConstructor: RawlsRequestContext => BucketMigrationService + val workspaceSettingServiceConstructor: RawlsRequestContext => WorkspaceSettingService def workspaceRoutesV2(otelContext: Context = Context.root()): server.Route = requireUserInfo(Option(otelContext)) { userInfo => @@ -91,7 +92,7 @@ trait WorkspaceApiServiceV2 extends UserInfoDirectives { pathEndOrSingleSlash { get { complete { - workspaceServiceConstructor(ctx) + workspaceSettingServiceConstructor(ctx) .getWorkspaceSettings(workspaceName) .map(StatusCodes.OK -> _) } @@ -99,7 +100,7 @@ trait WorkspaceApiServiceV2 extends UserInfoDirectives { put { entity(as[List[WorkspaceSetting]]) { settings => complete { - workspaceServiceConstructor(ctx) + workspaceSettingServiceConstructor(ctx) .setWorkspaceSettings(workspaceName, settings) .map(StatusCodes.OK -> _) } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala index 99f962489f..06910400fc 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala @@ -2005,122 +2005,6 @@ class WorkspaceService( } } yield {} - def getWorkspaceSettings(workspaceName: WorkspaceName): Future[List[WorkspaceSetting]] = - getV2WorkspaceContextAndPermissions(workspaceName, SamWorkspaceActions.read).flatMap { workspace => - workspaceRepository.getWorkspaceSettings(workspace.workspaceIdAsUUID) - } - - def setWorkspaceSettings(workspaceName: WorkspaceName, - workspaceSettings: List[WorkspaceSetting] - ): Future[WorkspaceSettingResponse] = { - - /** - * Perform basic validation checks on requested settings. - */ - def validateSettings(requestedSettings: List[WorkspaceSetting]): Unit = { - def validationErrorReport(settingType: WorkspaceSettingType, reason: String): ErrorReport = ErrorReport( - s"Invalid $settingType configuration: $reason." - ) - val validationErrors = requestedSettings.flatMap { - case WorkspaceSetting(settingType @ WorkspaceSettingTypes.GcpBucketLifecycle, - GcpBucketLifecycleConfig(rules) - ) => - rules.flatMap { rule => - val actionValidation = rule.action.actionType match { - case actionType if actionType.equals("Delete") => None - case actionType => Some(validationErrorReport(settingType, s"unsupported lifecycle action $actionType")) - } - val ageValidation = rule.conditions.age.collect { - case age if age < 0 => - validationErrorReport(settingType, "age must be a non-negative integer") - } - val atLeastOneConditionValidation = (rule.conditions.age, rule.conditions.matchesPrefix) match { - case (None, None) => - Some(validationErrorReport(settingType, "at least one condition must be specified")) - case (None, Some(prefixes)) if prefixes.isEmpty => - Some( - validationErrorReport(settingType, - "at least one prefix must be specified if matchesPrefix is the only condition" - ) - ) - case _ => None - } - actionValidation ++ ageValidation ++ atLeastOneConditionValidation - } - } - - if (validationErrors.nonEmpty) { - throw new RawlsExceptionWithErrorReport( - ErrorReport(StatusCodes.BadRequest, "Invalid settings requested.", validationErrors) - ) - } - } - - /** - * Apply a setting to a workspace. If the setting is successfully applied, update the database - * and return None. If the setting fails to apply, remove the failed setting from the database - * and return the setting type with an error report. If the setting is not supported, throw an - * exception. We make more trips to the database here than necessary, but we support a small - * number of setting types and it's easier to reason about this way. - */ - def applySetting(workspace: Workspace, - setting: WorkspaceSetting - ): Future[Option[(WorkspaceSettingType, ErrorReport)]] = - (setting match { - case WorkspaceSetting(WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig(rules)) => - val googleRules = rules.map { rule => - val conditionBuilder = LifecycleCondition.newBuilder() - rule.conditions.matchesPrefix.map(prefixes => conditionBuilder.setMatchesPrefix(prefixes.toList.asJava)) - rule.conditions.age.map(age => conditionBuilder.setAge(age)) - - val action = rule.action.actionType match { - case actionType if actionType.equals("Delete") => LifecycleAction.newDeleteAction() - - // validated earlier but needed for completeness - case _ => - throw new RawlsException( - "unsupported lifecycle action" - ) - } - - new LifecycleRule(action, conditionBuilder.build()) - } - - for { - _ <- gcsDAO.setBucketLifecycle(workspace.bucketName, googleRules) - _ <- workspaceRepository.markWorkspaceSettingApplied(workspace.workspaceIdAsUUID, setting.settingType) - } yield None - case _ => throw new RawlsException("unsupported workspace setting") - }).recoverWith { case e => - logger.error( - s"Failed to apply settings. [workspaceId=${workspace.workspaceIdAsUUID},settingType=${setting.settingType}]", - e - ) - workspaceRepository - .removePendingSetting(workspace.workspaceIdAsUUID, setting.settingType) - .map(_ => Some((setting.settingType, ErrorReport(StatusCodes.InternalServerError, e.getMessage)))) - } - - validateSettings(workspaceSettings) - for { - workspace <- getV2WorkspaceContextAndPermissions(workspaceName, SamWorkspaceActions.own) - currentSettings <- workspaceRepository.getWorkspaceSettings(workspace.workspaceIdAsUUID) - newSettings = workspaceSettings.filterNot(currentSettings.contains(_)) - _ <- workspaceRepository.createWorkspaceSettingsRecords(workspace.workspaceIdAsUUID, - newSettings, - ctx.userInfo.userSubjectId - ) - applyFailures <- newSettings.traverse(s => applySetting(workspace, s)) - } yield { - val successes = newSettings.filterNot { s => - applyFailures.flatten.exists { case (failedSettingType, _) => - failedSettingType == s.settingType - } - } - WorkspaceSettingResponse(successes, applyFailures.flatten.toMap) - } - } - // helper methods private def createWorkflowCollectionForWorkspace(workspaceId: String, diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingService.scala new file mode 100644 index 0000000000..02e1521796 --- /dev/null +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingService.scala @@ -0,0 +1,149 @@ +package org.broadinstitute.dsde.rawls.workspace + +import akka.http.scaladsl.model.StatusCodes +import com.google.cloud.storage.BucketInfo.LifecycleRule +import com.google.cloud.storage.BucketInfo.LifecycleRule.{LifecycleAction, LifecycleCondition} +import com.typesafe.scalalogging.LazyLogging +import cats.implicits._ +import org.broadinstitute.dsde.rawls.dataaccess.{GoogleServicesDAO, SamDAO} +import org.broadinstitute.dsde.rawls.{RawlsException, RawlsExceptionWithErrorReport} +import org.broadinstitute.dsde.rawls.model.WorkspaceSettingConfig.GcpBucketLifecycleConfig +import org.broadinstitute.dsde.rawls.model.{ + ErrorReport, + RawlsRequestContext, + SamWorkspaceActions, + Workspace, + WorkspaceName, + WorkspaceSetting, + WorkspaceSettingResponse, + WorkspaceSettingTypes +} +import org.broadinstitute.dsde.rawls.model.WorkspaceSettingTypes.WorkspaceSettingType +import org.broadinstitute.dsde.rawls.util.WorkspaceSupport + +import scala.concurrent.{ExecutionContext, Future} +import scala.jdk.CollectionConverters._ + +class WorkspaceSettingService(protected val ctx: RawlsRequestContext, + val workspaceRepository: WorkspaceRepository, + gcsDAO: GoogleServicesDAO, + val samDAO: SamDAO +)(implicit protected val executionContext: ExecutionContext) + extends WorkspaceSupport + with LazyLogging { + def getWorkspaceSettings(workspaceName: WorkspaceName): Future[List[WorkspaceSetting]] = + getV2WorkspaceContextAndPermissions(workspaceName, SamWorkspaceActions.read).flatMap { workspace => + workspaceRepository.getWorkspaceSettings(workspace.workspaceIdAsUUID) + } + + def setWorkspaceSettings(workspaceName: WorkspaceName, + workspaceSettings: List[WorkspaceSetting] + ): Future[WorkspaceSettingResponse] = { + + /** + * Perform basic validation checks on requested settings. + */ + def validateSettings(requestedSettings: List[WorkspaceSetting]): Unit = { + def validationErrorReport(settingType: WorkspaceSettingType, reason: String): ErrorReport = ErrorReport( + s"Invalid $settingType configuration: $reason." + ) + val validationErrors = requestedSettings.flatMap { + case WorkspaceSetting(settingType @ WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig(rules) + ) => + rules.flatMap { rule => + val actionValidation = rule.action.actionType match { + case actionType if actionType.equals("Delete") => None + case actionType => Some(validationErrorReport(settingType, s"unsupported lifecycle action $actionType")) + } + val ageValidation = rule.conditions.age.collect { + case age if age < 0 => + validationErrorReport(settingType, "age must be a non-negative integer") + } + val atLeastOneConditionValidation = (rule.conditions.age, rule.conditions.matchesPrefix) match { + case (None, None) => + Some(validationErrorReport(settingType, "at least one condition must be specified")) + case (None, Some(prefixes)) if prefixes.isEmpty => + Some( + validationErrorReport(settingType, + "at least one prefix must be specified if matchesPrefix is the only condition" + ) + ) + case _ => None + } + actionValidation ++ ageValidation ++ atLeastOneConditionValidation + } + } + + if (validationErrors.nonEmpty) { + throw new RawlsExceptionWithErrorReport( + ErrorReport(StatusCodes.BadRequest, "Invalid settings requested.", validationErrors) + ) + } + } + + /** + * Apply a setting to a workspace. If the setting is successfully applied, update the database + * and return None. If the setting fails to apply, remove the failed setting from the database + * and return the setting type with an error report. If the setting is not supported, throw an + * exception. We make more trips to the database here than necessary, but we support a small + * number of setting types and it's easier to reason about this way. + */ + def applySetting(workspace: Workspace, + setting: WorkspaceSetting + ): Future[Option[(WorkspaceSettingType, ErrorReport)]] = + (setting match { + case WorkspaceSetting(WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig(rules)) => + val googleRules = rules.map { rule => + val conditionBuilder = LifecycleCondition.newBuilder() + rule.conditions.matchesPrefix.map(prefixes => conditionBuilder.setMatchesPrefix(prefixes.toList.asJava)) + rule.conditions.age.map(age => conditionBuilder.setAge(age)) + + val action = rule.action.actionType match { + case actionType if actionType.equals("Delete") => LifecycleAction.newDeleteAction() + + // validated earlier but needed for completeness + case _ => + throw new RawlsException( + "unsupported lifecycle action" + ) + } + + new LifecycleRule(action, conditionBuilder.build()) + } + + for { + _ <- gcsDAO.setBucketLifecycle(workspace.bucketName, googleRules) + _ <- workspaceRepository.markWorkspaceSettingApplied(workspace.workspaceIdAsUUID, setting.settingType) + } yield None + case _ => throw new RawlsException("unsupported workspace setting") + }).recoverWith { case e => + logger.error( + s"Failed to apply settings. [workspaceId=${workspace.workspaceIdAsUUID},settingType=${setting.settingType}]", + e + ) + workspaceRepository + .removePendingSetting(workspace.workspaceIdAsUUID, setting.settingType) + .map(_ => Some((setting.settingType, ErrorReport(StatusCodes.InternalServerError, e.getMessage)))) + } + + validateSettings(workspaceSettings) + for { + workspace <- getV2WorkspaceContextAndPermissions(workspaceName, SamWorkspaceActions.own) + currentSettings <- workspaceRepository.getWorkspaceSettings(workspace.workspaceIdAsUUID) + newSettings = workspaceSettings.filterNot(currentSettings.contains(_)) + _ <- workspaceRepository.createWorkspaceSettingsRecords(workspace.workspaceIdAsUUID, + newSettings, + ctx.userInfo.userSubjectId + ) + applyFailures <- newSettings.traverse(s => applySetting(workspace, s)) + } yield { + val successes = newSettings.filterNot { s => + applyFailures.flatten.exists { case (failedSettingType, _) => + failedSettingType == s.settingType + } + } + WorkspaceSettingResponse(successes, applyFailures.flatten.toMap) + } + } +} diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/ApiServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/ApiServiceSpec.scala index d18d89334c..38592cd2dd 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/ApiServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/ApiServiceSpec.scala @@ -54,7 +54,8 @@ import org.broadinstitute.dsde.rawls.workspace.{ MultiCloudWorkspaceService, RawlsWorkspaceAclManager, WorkspaceRepository, - WorkspaceService + WorkspaceService, + WorkspaceSettingService } import org.broadinstitute.dsde.workbench.dataaccess.{NotificationDAO, PubSubNotificationDAO} import org.broadinstitute.dsde.workbench.google.mock.{MockGoogleBigQueryDAO, MockGoogleIamDAO, MockGoogleStorageDAO} @@ -393,6 +394,9 @@ trait ApiServiceSpec workbenchMetricBaseName ) + override val workspaceSettingServiceConstructor: RawlsRequestContext => WorkspaceSettingService = + new WorkspaceSettingService(_, new WorkspaceRepository(slickDataSource), gcsDAO, samDAO) + override val methodConfigurationServiceConstructor: RawlsRequestContext => MethodConfigurationService = MethodConfigurationService.constructor( slickDataSource, diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala index b9da9c2d3f..e13e81320a 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceUnitTests.scala @@ -3,20 +3,12 @@ package org.broadinstitute.dsde.rawls.workspace import akka.http.scaladsl.model.StatusCodes import akka.http.scaladsl.model.headers.OAuth2BearerToken import bio.terra.workspace.model.{IamRole, RoleBinding, RoleBindingList} -import com.google.cloud.storage.BucketInfo.LifecycleRule -import com.google.cloud.storage.BucketInfo.LifecycleRule.{LifecycleAction, LifecycleCondition} import org.broadinstitute.dsde.rawls.billing.{BillingProfileManagerDAO, BillingRepository} import org.broadinstitute.dsde.rawls.config._ import org.broadinstitute.dsde.rawls.dataaccess._ import org.broadinstitute.dsde.rawls.dataaccess.leonardo.LeonardoService import org.broadinstitute.dsde.rawls.dataaccess.workspacemanager.WorkspaceManagerDAO import org.broadinstitute.dsde.rawls.fastpass.FastPassServiceImpl -import org.broadinstitute.dsde.rawls.model.WorkspaceSettingConfig.{ - GcpBucketLifecycleAction, - GcpBucketLifecycleCondition, - GcpBucketLifecycleConfig, - GcpBucketLifecycleRule -} import org.broadinstitute.dsde.rawls.model.WorkspaceType.WorkspaceType import org.broadinstitute.dsde.rawls.model._ import org.broadinstitute.dsde.rawls.resourcebuffer.ResourceBufferServiceImpl @@ -38,7 +30,6 @@ import org.mockito.ArgumentMatchers._ import org.mockito.Mockito._ import org.scalatest.OptionValues import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.must.Matchers.{contain, include} import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper import spray.json.{JsObject, JsString} @@ -800,392 +791,4 @@ class WorkspaceServiceUnitTests extends AnyFlatSpec with OptionValues with Mocki Await.result(service.updateACL(WorkspaceName("fake_namespace", "fake_name"), aclUpdate, true), Duration.Inf) } - - "getWorkspaceSettings" should "return the workspace settings" in { - val workspaceId = workspace.workspaceIdAsUUID - val workspaceName = workspace.toWorkspaceName - val workspaceSettings = List( - WorkspaceSetting(WorkspaceSettingTypes.GcpBucketLifecycle, - WorkspaceSettingTypes.GcpBucketLifecycle.defaultConfig() - ) - ) - - val workspaceRepository = mock[WorkspaceRepository] - when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) - when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(workspaceSettings)) - - val samDAO = mock[SamDAO] - when(samDAO.getUserStatus(any())) - .thenReturn(Future.successful(Option(SamUserStatusResponse("fake_user_id", "user@example.com", true)))) - when( - samDAO.userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), - ArgumentMatchers.eq(workspaceId.toString), - ArgumentMatchers.eq(SamWorkspaceActions.read), - any() - ) - ).thenReturn(Future.successful(true)) - - val service = - workspaceServiceConstructor(samDAO = samDAO, workspaceRepository = workspaceRepository)(defaultRequestContext) - - val returnedSettings = Await.result(service.getWorkspaceSettings(workspaceName), Duration.Inf) - returnedSettings shouldEqual workspaceSettings - verify(samDAO).userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), - ArgumentMatchers.eq(workspaceId.toString), - ArgumentMatchers.eq(SamWorkspaceActions.read), - any() - ) - } - - it should "handle a workspace with no settings" in { - val workspaceId = workspace.workspaceIdAsUUID - val workspaceName = workspace.toWorkspaceName - - val workspaceRepository = mock[WorkspaceRepository] - when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) - when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(List.empty)) - - val samDAO = mock[SamDAO] - when(samDAO.getUserStatus(any())) - .thenReturn(Future.successful(Option(SamUserStatusResponse("fake_user_id", "user@example.com", true)))) - when( - samDAO.userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), - ArgumentMatchers.eq(workspaceId.toString), - ArgumentMatchers.eq(SamWorkspaceActions.read), - any() - ) - ).thenReturn(Future.successful(true)) - - val service = - workspaceServiceConstructor(samDAO = samDAO, workspaceRepository = workspaceRepository)(defaultRequestContext) - - val returnedSettings = Await.result(service.getWorkspaceSettings(workspaceName), Duration.Inf) - returnedSettings shouldEqual List.empty - verify(samDAO).userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), - ArgumentMatchers.eq(workspaceId.toString), - ArgumentMatchers.eq(SamWorkspaceActions.read), - any() - ) - } - - it should "return an error if the user does not have read access to the workspace" in { - val workspaceId = workspace.workspaceIdAsUUID - val workspaceName = workspace.toWorkspaceName - - val workspaceRepository = mock[WorkspaceRepository] - when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) - - val samDAO = mock[SamDAO] - when(samDAO.getUserStatus(any())) - .thenReturn(Future.successful(Option(SamUserStatusResponse("fake_user_id", "user@example.com", true)))) - when( - samDAO.userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), - ArgumentMatchers.eq(workspaceId.toString), - ArgumentMatchers.eq(SamWorkspaceActions.read), - any() - ) - ).thenReturn(Future.successful(false)) - val service = - workspaceServiceConstructor(samDAO = samDAO, workspaceRepository = workspaceRepository)(defaultRequestContext) - - assertThrows[NoSuchWorkspaceException] { - Await.result(service.getWorkspaceSettings(workspaceName), Duration.Inf) - } - } - - "setWorkspaceSettings" should "set the workspace settings if there aren't any set" in { - val workspaceId = workspace.workspaceIdAsUUID - val workspaceName = workspace.toWorkspaceName - val workspaceSetting = WorkspaceSetting(WorkspaceSettingTypes.GcpBucketLifecycle, - WorkspaceSettingTypes.GcpBucketLifecycle.defaultConfig() - ) - - val workspaceRepository = mock[WorkspaceRepository] - when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) - when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(List.empty)) - when( - workspaceRepository.createWorkspaceSettingsRecords(workspaceId, - List(workspaceSetting), - defaultRequestContext.userInfo.userSubjectId - ) - ) - .thenReturn(Future.successful(List(workspaceSetting))) - when(workspaceRepository.markWorkspaceSettingApplied(workspaceId, workspaceSetting.settingType)) - .thenReturn(Future.successful(1)) - - val samDAO = mock[SamDAO] - when(samDAO.getUserStatus(any())) - .thenReturn(Future.successful(Option(SamUserStatusResponse("fake_user_id", "user@example.com", true)))) - when( - samDAO.userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), - ArgumentMatchers.eq(workspaceId.toString), - ArgumentMatchers.eq(SamWorkspaceActions.own), - any() - ) - ).thenReturn(Future.successful(true)) - - val gcsDAO = mock[GoogleServicesDAO] - when(gcsDAO.setBucketLifecycle(workspace.bucketName, List())).thenReturn(Future.successful()) - - val service = workspaceServiceConstructor(samDAO = samDAO, - workspaceRepository = workspaceRepository, - gcsDAO = gcsDAO - )(defaultRequestContext) - - val res = Await.result(service.setWorkspaceSettings(workspaceName, List(workspaceSetting)), Duration.Inf) - res.successes should contain theSameElementsAs List(workspaceSetting) - res.failures shouldEqual Map.empty - } - - it should "overwrite existing settings" in { - val workspaceId = workspace.workspaceIdAsUUID - val workspaceName = workspace.toWorkspaceName - val existingSetting = WorkspaceSetting( - WorkspaceSettingTypes.GcpBucketLifecycle, - GcpBucketLifecycleConfig( - List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), - GcpBucketLifecycleCondition(Some(Set("prefixToMatch")), Some(30)) - ) - ) - ) - ) - val newSetting = WorkspaceSetting( - WorkspaceSettingTypes.GcpBucketLifecycle, - GcpBucketLifecycleConfig( - List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), - GcpBucketLifecycleCondition(Some(Set("muchBetterPrefix")), Some(31)) - ) - ) - ) - ) - - val workspaceRepository = mock[WorkspaceRepository] - when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) - when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(List(existingSetting))) - when( - workspaceRepository.createWorkspaceSettingsRecords(workspaceId, - List(newSetting), - defaultRequestContext.userInfo.userSubjectId - ) - ) - .thenReturn(Future.successful(List(newSetting))) - when(workspaceRepository.markWorkspaceSettingApplied(workspaceId, newSetting.settingType)) - .thenReturn(Future.successful(1)) - - val samDAO = mock[SamDAO] - when(samDAO.getUserStatus(any())) - .thenReturn(Future.successful(Option(SamUserStatusResponse("fake_user_id", "user@example.com", true)))) - when( - samDAO.userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), - ArgumentMatchers.eq(workspaceId.toString), - ArgumentMatchers.eq(SamWorkspaceActions.own), - any() - ) - ).thenReturn(Future.successful(true)) - - val gcsDAO = mock[GoogleServicesDAO] - val newSettingGoogleRule = new LifecycleRule( - LifecycleAction.newDeleteAction(), - LifecycleCondition.newBuilder().setMatchesPrefix(List("muchBetterPrefix").asJava).setAge(31).build() - ) - when(gcsDAO.setBucketLifecycle(workspace.bucketName, List(newSettingGoogleRule))).thenReturn(Future.successful()) - - val service = workspaceServiceConstructor(samDAO = samDAO, - workspaceRepository = workspaceRepository, - gcsDAO = gcsDAO - )(defaultRequestContext) - - val res = Await.result(service.setWorkspaceSettings(workspaceName, List(newSetting)), Duration.Inf) - res.successes should contain theSameElementsAs List(newSetting) - res.failures shouldEqual Map.empty - } - - it should "remove existing settings if no settings are specified" in { - val workspaceId = workspace.workspaceIdAsUUID - val workspaceName = workspace.toWorkspaceName - val existingSetting = WorkspaceSetting( - WorkspaceSettingTypes.GcpBucketLifecycle, - GcpBucketLifecycleConfig( - List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), - GcpBucketLifecycleCondition(Some(Set("prefixToMatch")), Some(30)) - ) - ) - ) - ) - val defaultSetting = WorkspaceSetting(WorkspaceSettingTypes.GcpBucketLifecycle, - WorkspaceSettingTypes.GcpBucketLifecycle.defaultConfig() - ) - - val workspaceRepository = mock[WorkspaceRepository] - when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) - when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(List(existingSetting))) - when( - workspaceRepository.createWorkspaceSettingsRecords(workspaceId, - List.empty, - defaultRequestContext.userInfo.userSubjectId - ) - ) - .thenReturn(Future.successful(List.empty)) - when(workspaceRepository.markWorkspaceSettingApplied(workspaceId, defaultSetting.settingType)) - .thenReturn(Future.successful(1)) - - val samDAO = mock[SamDAO] - when(samDAO.getUserStatus(any())) - .thenReturn(Future.successful(Option(SamUserStatusResponse("fake_user_id", "user@example.com", true)))) - when( - samDAO.userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), - ArgumentMatchers.eq(workspaceId.toString), - ArgumentMatchers.eq(SamWorkspaceActions.own), - any() - ) - ).thenReturn(Future.successful(true)) - - val gcsDAO = mock[GoogleServicesDAO] - when(gcsDAO.setBucketLifecycle(workspace.bucketName, List())).thenReturn(Future.successful()) - - val service = workspaceServiceConstructor(samDAO = samDAO, - workspaceRepository = workspaceRepository, - gcsDAO = gcsDAO - )(defaultRequestContext) - - val res = Await.result(service.setWorkspaceSettings(workspaceName, List.empty), Duration.Inf) - res.successes should contain theSameElementsAs List(defaultSetting) - res.failures shouldEqual Map.empty - } - - it should "report errors while applying settings and remove pending settings" in { - val workspaceId = workspace.workspaceIdAsUUID - val workspaceName = workspace.toWorkspaceName - val existingSetting = WorkspaceSetting( - WorkspaceSettingTypes.GcpBucketLifecycle, - GcpBucketLifecycleConfig( - List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), - GcpBucketLifecycleCondition(Some(Set("prefixToMatch")), Some(30)) - ) - ) - ) - ) - val newSetting = WorkspaceSetting( - WorkspaceSettingTypes.GcpBucketLifecycle, - GcpBucketLifecycleConfig( - List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), - GcpBucketLifecycleCondition(Some(Set("muchBetterPrefix")), Some(31)) - ) - ) - ) - ) - - val workspaceRepository = mock[WorkspaceRepository] - when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) - when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(List(existingSetting))) - when( - workspaceRepository.createWorkspaceSettingsRecords(workspaceId, - List(newSetting), - defaultRequestContext.userInfo.userSubjectId - ) - ) - .thenReturn(Future.successful(List(newSetting))) - when(workspaceRepository.removePendingSetting(workspaceId, newSetting.settingType)).thenReturn(Future.successful(1)) - - val samDAO = mock[SamDAO] - when(samDAO.getUserStatus(any())) - .thenReturn(Future.successful(Option(SamUserStatusResponse("fake_user_id", "user@example.com", true)))) - when( - samDAO.userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), - ArgumentMatchers.eq(workspaceId.toString), - ArgumentMatchers.eq(SamWorkspaceActions.own), - any() - ) - ).thenReturn(Future.successful(true)) - - val gcsDAO = mock[GoogleServicesDAO] - val newSettingGoogleRule = new LifecycleRule( - LifecycleAction.newDeleteAction(), - LifecycleCondition.newBuilder().setMatchesPrefix(List("muchBetterPrefix").asJava).setAge(31).build() - ) - when(gcsDAO.setBucketLifecycle(workspace.bucketName, List(newSettingGoogleRule))) - .thenReturn(Future.failed(new Exception("failed to apply settings"))) - - val service = workspaceServiceConstructor(samDAO = samDAO, - workspaceRepository = workspaceRepository, - gcsDAO = gcsDAO - )(defaultRequestContext) - - val res = Await.result(service.setWorkspaceSettings(workspaceName, List(newSetting)), Duration.Inf) - res.successes shouldEqual List.empty - res.failures(WorkspaceSettingTypes.GcpBucketLifecycle) shouldEqual ErrorReport(StatusCodes.InternalServerError, - "failed to apply settings" - ) - verify(workspaceRepository).removePendingSetting(workspaceId, newSetting.settingType) - } - - it should "be limited to owners" in { - val workspaceId = workspace.workspaceIdAsUUID - val workspaceName = workspace.toWorkspaceName - val newSetting = WorkspaceSetting(WorkspaceSettingTypes.GcpBucketLifecycle, - WorkspaceSettingTypes.GcpBucketLifecycle.defaultConfig() - ) - - val workspaceRepository = mock[WorkspaceRepository] - when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) - - val samDAO = mock[SamDAO] - when(samDAO.getUserStatus(any())) - .thenReturn(Future.successful(Option(SamUserStatusResponse("fake_user_id", "user@example.com", true)))) - when( - samDAO.userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), - ArgumentMatchers.eq(workspaceId.toString), - ArgumentMatchers.eq(SamWorkspaceActions.own), - any() - ) - ).thenReturn(Future.successful(false)) - // Rawls confirms a user has at least read access to the workspace after a failed authz check - // to determine if it should throw a 403 or 404 - when( - samDAO.userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), - ArgumentMatchers.eq(workspaceId.toString), - ArgumentMatchers.eq(SamWorkspaceActions.read), - any() - ) - ).thenReturn(Future.successful(true)) - - val service = - workspaceServiceConstructor(samDAO = samDAO, workspaceRepository = workspaceRepository)(defaultRequestContext) - - val exception = intercept[RawlsExceptionWithErrorReport] { - Await.result(service.setWorkspaceSettings(workspaceName, List(newSetting)), Duration.Inf) - } - exception.errorReport.statusCode shouldBe Some(StatusCodes.Forbidden) - } - - it should "validate requested settings" in { - val workspaceName = workspace.toWorkspaceName - val newSetting = WorkspaceSetting( - WorkspaceSettingTypes.GcpBucketLifecycle, - GcpBucketLifecycleConfig( - List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("SetStorageClass"), - GcpBucketLifecycleCondition(Some(Set("prefixToMatch")), Some(-1)) - ) - ) - ) - ) - - val service = workspaceServiceConstructor()(defaultRequestContext) - - val exception = intercept[RawlsExceptionWithErrorReport] { - Await.result(service.setWorkspaceSettings(workspaceName, List(newSetting)), Duration.Inf) - } - exception.errorReport.statusCode shouldBe Some(StatusCodes.BadRequest) - exception.errorReport.message should include("Invalid settings requested.") - exception.errorReport.causes should contain theSameElementsAs List( - ErrorReport("Invalid GcpBucketLifecycle configuration: age must be a non-negative integer."), - ErrorReport("Invalid GcpBucketLifecycle configuration: unsupported lifecycle action SetStorageClass.") - ) - } } diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingServiceUnitTests.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingServiceUnitTests.scala new file mode 100644 index 0000000000..076b916b68 --- /dev/null +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingServiceUnitTests.scala @@ -0,0 +1,434 @@ +package org.broadinstitute.dsde.rawls.workspace + +import akka.http.scaladsl.model.StatusCodes +import akka.http.scaladsl.model.headers.OAuth2BearerToken +import com.google.cloud.storage.BucketInfo.LifecycleRule +import com.google.cloud.storage.BucketInfo.LifecycleRule.{LifecycleAction, LifecycleCondition} +import org.broadinstitute.dsde.rawls.{NoSuchWorkspaceException, RawlsExceptionWithErrorReport} +import org.broadinstitute.dsde.rawls.dataaccess.{GoogleServicesDAO, SamDAO} +import org.broadinstitute.dsde.rawls.model.WorkspaceSettingConfig.{ + GcpBucketLifecycleAction, + GcpBucketLifecycleCondition, + GcpBucketLifecycleConfig, + GcpBucketLifecycleRule +} +import org.broadinstitute.dsde.rawls.model.{ + ErrorReport, + RawlsRequestContext, + RawlsUserEmail, + RawlsUserSubjectId, + SamResourceTypeNames, + SamUserStatusResponse, + SamWorkspaceActions, + UserInfo, + Workspace, + WorkspaceSetting, + WorkspaceSettingTypes +} +import org.broadinstitute.dsde.rawls.util.MockitoTestUtils +import org.joda.time.DateTime +import org.mockito.ArgumentMatchers +import org.mockito.ArgumentMatchers.any +import org.mockito.Mockito.{verify, when, RETURNS_SMART_NULLS} +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.must.Matchers.{contain, include} +import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper + +import java.util.UUID +import scala.jdk.CollectionConverters._ +import scala.concurrent.{Await, Future} +import scala.concurrent.duration.Duration + +class WorkspaceSettingServiceUnitTests extends AnyFlatSpec with MockitoTestUtils { + + implicit val ec: scala.concurrent.ExecutionContext = scala.concurrent.ExecutionContext.global + + val defaultRequestContext: RawlsRequestContext = + RawlsRequestContext( + UserInfo(RawlsUserEmail("test"), OAuth2BearerToken("Bearer 123"), 123, RawlsUserSubjectId("abc")) + ) + + def workspaceSettingServiceConstructor(ctx: RawlsRequestContext = defaultRequestContext, + workspaceRepository: WorkspaceRepository = + mock[WorkspaceRepository](RETURNS_SMART_NULLS), + gcsDAO: GoogleServicesDAO = mock[GoogleServicesDAO](RETURNS_SMART_NULLS), + samDAO: SamDAO = mock[SamDAO](RETURNS_SMART_NULLS) + ): WorkspaceSettingService = + new WorkspaceSettingService(ctx, workspaceRepository, gcsDAO, samDAO) + + val workspace: Workspace = Workspace( + "settingsTestWorkspace", + "settingsTestNamespace", + UUID.randomUUID.toString, + "bucketName", + Some("workflowCollection"), + new DateTime(), + new DateTime(), + "creator", + Map.empty + ) + + "getWorkspaceSettings" should "return the workspace settings" in { + val workspaceId = workspace.workspaceIdAsUUID + val workspaceName = workspace.toWorkspaceName + val workspaceSettings = List( + WorkspaceSetting(WorkspaceSettingTypes.GcpBucketLifecycle, + WorkspaceSettingTypes.GcpBucketLifecycle.defaultConfig() + ) + ) + + val workspaceRepository = mock[WorkspaceRepository] + when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) + when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(workspaceSettings)) + + val samDAO = mock[SamDAO] + when(samDAO.getUserStatus(any())) + .thenReturn(Future.successful(Option(SamUserStatusResponse("fake_user_id", "user@example.com", true)))) + when( + samDAO.userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), + ArgumentMatchers.eq(workspaceId.toString), + ArgumentMatchers.eq(SamWorkspaceActions.read), + any() + ) + ).thenReturn(Future.successful(true)) + + val service = + workspaceSettingServiceConstructor(samDAO = samDAO, workspaceRepository = workspaceRepository) + + val returnedSettings = Await.result(service.getWorkspaceSettings(workspaceName), Duration.Inf) + returnedSettings shouldEqual workspaceSettings + verify(samDAO).userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), + ArgumentMatchers.eq(workspaceId.toString), + ArgumentMatchers.eq(SamWorkspaceActions.read), + any() + ) + } + + it should "handle a workspace with no settings" in { + val workspaceId = workspace.workspaceIdAsUUID + val workspaceName = workspace.toWorkspaceName + + val workspaceRepository = mock[WorkspaceRepository] + when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) + when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(List.empty)) + + val samDAO = mock[SamDAO] + when(samDAO.getUserStatus(any())) + .thenReturn(Future.successful(Option(SamUserStatusResponse("fake_user_id", "user@example.com", true)))) + when( + samDAO.userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), + ArgumentMatchers.eq(workspaceId.toString), + ArgumentMatchers.eq(SamWorkspaceActions.read), + any() + ) + ).thenReturn(Future.successful(true)) + + val service = + workspaceSettingServiceConstructor(samDAO = samDAO, workspaceRepository = workspaceRepository) + + val returnedSettings = Await.result(service.getWorkspaceSettings(workspaceName), Duration.Inf) + returnedSettings shouldEqual List.empty + verify(samDAO).userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), + ArgumentMatchers.eq(workspaceId.toString), + ArgumentMatchers.eq(SamWorkspaceActions.read), + any() + ) + } + + it should "return an error if the user does not have read access to the workspace" in { + val workspaceId = workspace.workspaceIdAsUUID + val workspaceName = workspace.toWorkspaceName + + val workspaceRepository = mock[WorkspaceRepository] + when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) + + val samDAO = mock[SamDAO] + when(samDAO.getUserStatus(any())) + .thenReturn(Future.successful(Option(SamUserStatusResponse("fake_user_id", "user@example.com", true)))) + when( + samDAO.userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), + ArgumentMatchers.eq(workspaceId.toString), + ArgumentMatchers.eq(SamWorkspaceActions.read), + any() + ) + ).thenReturn(Future.successful(false)) + val service = + workspaceSettingServiceConstructor(samDAO = samDAO, workspaceRepository = workspaceRepository) + + assertThrows[NoSuchWorkspaceException] { + Await.result(service.getWorkspaceSettings(workspaceName), Duration.Inf) + } + } + + "setWorkspaceSettings" should "set the workspace settings if there aren't any set" in { + val workspaceId = workspace.workspaceIdAsUUID + val workspaceName = workspace.toWorkspaceName + val workspaceSetting = WorkspaceSetting(WorkspaceSettingTypes.GcpBucketLifecycle, + WorkspaceSettingTypes.GcpBucketLifecycle.defaultConfig() + ) + + val workspaceRepository = mock[WorkspaceRepository] + when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) + when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(List.empty)) + when( + workspaceRepository.createWorkspaceSettingsRecords(workspaceId, + List(workspaceSetting), + defaultRequestContext.userInfo.userSubjectId + ) + ) + .thenReturn(Future.successful(List(workspaceSetting))) + when(workspaceRepository.markWorkspaceSettingApplied(workspaceId, workspaceSetting.settingType)) + .thenReturn(Future.successful(1)) + + val samDAO = mock[SamDAO] + when(samDAO.getUserStatus(any())) + .thenReturn(Future.successful(Option(SamUserStatusResponse("fake_user_id", "user@example.com", true)))) + when( + samDAO.userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), + ArgumentMatchers.eq(workspaceId.toString), + ArgumentMatchers.eq(SamWorkspaceActions.own), + any() + ) + ).thenReturn(Future.successful(true)) + + val gcsDAO = mock[GoogleServicesDAO] + when(gcsDAO.setBucketLifecycle(workspace.bucketName, List())).thenReturn(Future.successful()) + + val service = + workspaceSettingServiceConstructor(samDAO = samDAO, workspaceRepository = workspaceRepository, gcsDAO = gcsDAO) + + val res = Await.result(service.setWorkspaceSettings(workspaceName, List(workspaceSetting)), Duration.Inf) + res.successes should contain theSameElementsAs List(workspaceSetting) + res.failures shouldEqual Map.empty + } + + it should "overwrite existing settings" in { + val workspaceId = workspace.workspaceIdAsUUID + val workspaceName = workspace.toWorkspaceName + val existingSetting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), + GcpBucketLifecycleCondition(Some(Set("prefixToMatch")), Some(30)) + ) + ) + ) + ) + val newSetting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), + GcpBucketLifecycleCondition(Some(Set("muchBetterPrefix")), Some(31)) + ) + ) + ) + ) + + val workspaceRepository = mock[WorkspaceRepository] + when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) + when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(List(existingSetting))) + when( + workspaceRepository.createWorkspaceSettingsRecords(workspaceId, + List(newSetting), + defaultRequestContext.userInfo.userSubjectId + ) + ) + .thenReturn(Future.successful(List(newSetting))) + when(workspaceRepository.markWorkspaceSettingApplied(workspaceId, newSetting.settingType)) + .thenReturn(Future.successful(1)) + + val samDAO = mock[SamDAO] + when(samDAO.getUserStatus(any())) + .thenReturn(Future.successful(Option(SamUserStatusResponse("fake_user_id", "user@example.com", true)))) + when( + samDAO.userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), + ArgumentMatchers.eq(workspaceId.toString), + ArgumentMatchers.eq(SamWorkspaceActions.own), + any() + ) + ).thenReturn(Future.successful(true)) + + val gcsDAO = mock[GoogleServicesDAO] + val newSettingGoogleRule = new LifecycleRule( + LifecycleAction.newDeleteAction(), + LifecycleCondition.newBuilder().setMatchesPrefix(List("muchBetterPrefix").asJava).setAge(31).build() + ) + when(gcsDAO.setBucketLifecycle(workspace.bucketName, List(newSettingGoogleRule))).thenReturn(Future.successful()) + + val service = + workspaceSettingServiceConstructor(samDAO = samDAO, workspaceRepository = workspaceRepository, gcsDAO = gcsDAO) + + val res = Await.result(service.setWorkspaceSettings(workspaceName, List(newSetting)), Duration.Inf) + res.successes should contain theSameElementsAs List(newSetting) + res.failures shouldEqual Map.empty + } + + it should "not remove existing settings if no settings are specified" in { + val workspaceId = workspace.workspaceIdAsUUID + val workspaceName = workspace.toWorkspaceName + val existingSetting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), + GcpBucketLifecycleCondition(Some(Set("prefixToMatch")), Some(30)) + ) + ) + ) + ) + + val workspaceRepository = mock[WorkspaceRepository] + when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) + when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(List(existingSetting))) + + val samDAO = mock[SamDAO] + when(samDAO.getUserStatus(any())) + .thenReturn(Future.successful(Option(SamUserStatusResponse("fake_user_id", "user@example.com", true)))) + when( + samDAO.userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), + ArgumentMatchers.eq(workspaceId.toString), + ArgumentMatchers.eq(SamWorkspaceActions.own), + any() + ) + ).thenReturn(Future.successful(true)) + + val service = workspaceSettingServiceConstructor(samDAO = samDAO, workspaceRepository = workspaceRepository) + + val res = Await.result(service.setWorkspaceSettings(workspaceName, List.empty), Duration.Inf) + res.successes shouldEqual List.empty + res.failures shouldEqual Map.empty + } + + it should "report errors while applying settings and remove pending settings" in { + val workspaceId = workspace.workspaceIdAsUUID + val workspaceName = workspace.toWorkspaceName + val existingSetting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), + GcpBucketLifecycleCondition(Some(Set("prefixToMatch")), Some(30)) + ) + ) + ) + ) + val newSetting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), + GcpBucketLifecycleCondition(Some(Set("muchBetterPrefix")), Some(31)) + ) + ) + ) + ) + + val workspaceRepository = mock[WorkspaceRepository] + when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) + when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(List(existingSetting))) + when( + workspaceRepository.createWorkspaceSettingsRecords(workspaceId, + List(newSetting), + defaultRequestContext.userInfo.userSubjectId + ) + ) + .thenReturn(Future.successful(List(newSetting))) + when(workspaceRepository.removePendingSetting(workspaceId, newSetting.settingType)).thenReturn(Future.successful(1)) + + val samDAO = mock[SamDAO] + when(samDAO.getUserStatus(any())) + .thenReturn(Future.successful(Option(SamUserStatusResponse("fake_user_id", "user@example.com", true)))) + when( + samDAO.userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), + ArgumentMatchers.eq(workspaceId.toString), + ArgumentMatchers.eq(SamWorkspaceActions.own), + any() + ) + ).thenReturn(Future.successful(true)) + + val gcsDAO = mock[GoogleServicesDAO] + val newSettingGoogleRule = new LifecycleRule( + LifecycleAction.newDeleteAction(), + LifecycleCondition.newBuilder().setMatchesPrefix(List("muchBetterPrefix").asJava).setAge(31).build() + ) + when(gcsDAO.setBucketLifecycle(workspace.bucketName, List(newSettingGoogleRule))) + .thenReturn(Future.failed(new Exception("failed to apply settings"))) + + val service = + workspaceSettingServiceConstructor(samDAO = samDAO, workspaceRepository = workspaceRepository, gcsDAO = gcsDAO) + + val res = Await.result(service.setWorkspaceSettings(workspaceName, List(newSetting)), Duration.Inf) + res.successes shouldEqual List.empty + res.failures(WorkspaceSettingTypes.GcpBucketLifecycle) shouldEqual ErrorReport(StatusCodes.InternalServerError, + "failed to apply settings" + ) + verify(workspaceRepository).removePendingSetting(workspaceId, newSetting.settingType) + } + + it should "be limited to owners" in { + val workspaceId = workspace.workspaceIdAsUUID + val workspaceName = workspace.toWorkspaceName + val newSetting = WorkspaceSetting(WorkspaceSettingTypes.GcpBucketLifecycle, + WorkspaceSettingTypes.GcpBucketLifecycle.defaultConfig() + ) + + val workspaceRepository = mock[WorkspaceRepository] + when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) + + val samDAO = mock[SamDAO] + when(samDAO.getUserStatus(any())) + .thenReturn(Future.successful(Option(SamUserStatusResponse("fake_user_id", "user@example.com", true)))) + when( + samDAO.userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), + ArgumentMatchers.eq(workspaceId.toString), + ArgumentMatchers.eq(SamWorkspaceActions.own), + any() + ) + ).thenReturn(Future.successful(false)) + // Rawls confirms a user has at least read access to the workspace after a failed authz check + // to determine if it should throw a 403 or 404 + when( + samDAO.userHasAction(ArgumentMatchers.eq(SamResourceTypeNames.workspace), + ArgumentMatchers.eq(workspaceId.toString), + ArgumentMatchers.eq(SamWorkspaceActions.read), + any() + ) + ).thenReturn(Future.successful(true)) + + val service = + workspaceSettingServiceConstructor(samDAO = samDAO, workspaceRepository = workspaceRepository) + + val exception = intercept[RawlsExceptionWithErrorReport] { + Await.result(service.setWorkspaceSettings(workspaceName, List(newSetting)), Duration.Inf) + } + exception.errorReport.statusCode shouldBe Some(StatusCodes.Forbidden) + } + + it should "validate requested settings" in { + val workspaceName = workspace.toWorkspaceName + val newSetting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpBucketLifecycleAction("SetStorageClass"), + GcpBucketLifecycleCondition(Some(Set("prefixToMatch")), Some(-1)) + ) + ) + ) + ) + + val service = workspaceSettingServiceConstructor() + + val exception = intercept[RawlsExceptionWithErrorReport] { + Await.result(service.setWorkspaceSettings(workspaceName, List(newSetting)), Duration.Inf) + } + exception.errorReport.statusCode shouldBe Some(StatusCodes.BadRequest) + exception.errorReport.message should include("Invalid settings requested.") + exception.errorReport.causes should contain theSameElementsAs List( + ErrorReport("Invalid GcpBucketLifecycle configuration: age must be a non-negative integer."), + ErrorReport("Invalid GcpBucketLifecycle configuration: unsupported lifecycle action SetStorageClass.") + ) + } +} From 7686821195d8f6d7de0752cc7e83ea858fa49d9f Mon Sep 17 00:00:00 2001 From: Marcus Talbott Date: Tue, 13 Aug 2024 10:21:14 -0400 Subject: [PATCH 20/25] separate validation test cases --- .../workspace/WorkspaceSettingService.scala | 11 ++- .../WorkspaceSettingServiceUnitTests.scala | 89 +++++++++++++++++-- 2 files changed, 90 insertions(+), 10 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingService.scala index 02e1521796..6489bd7db1 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingService.scala @@ -7,7 +7,10 @@ import com.typesafe.scalalogging.LazyLogging import cats.implicits._ import org.broadinstitute.dsde.rawls.dataaccess.{GoogleServicesDAO, SamDAO} import org.broadinstitute.dsde.rawls.{RawlsException, RawlsExceptionWithErrorReport} -import org.broadinstitute.dsde.rawls.model.WorkspaceSettingConfig.GcpBucketLifecycleConfig +import org.broadinstitute.dsde.rawls.model.WorkspaceSettingConfig.{ + GcpBucketLifecycleCondition, + GcpBucketLifecycleConfig +} import org.broadinstitute.dsde.rawls.model.{ ErrorReport, RawlsRequestContext, @@ -60,10 +63,10 @@ class WorkspaceSettingService(protected val ctx: RawlsRequestContext, case age if age < 0 => validationErrorReport(settingType, "age must be a non-negative integer") } - val atLeastOneConditionValidation = (rule.conditions.age, rule.conditions.matchesPrefix) match { - case (None, None) => + val atLeastOneConditionValidation = rule.conditions match { + case GcpBucketLifecycleCondition(None, None) => Some(validationErrorReport(settingType, "at least one condition must be specified")) - case (None, Some(prefixes)) if prefixes.isEmpty => + case GcpBucketLifecycleCondition(Some(prefixes), None) if prefixes.isEmpty => Some( validationErrorReport(settingType, "at least one prefix must be specified if matchesPrefix is the only condition" diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingServiceUnitTests.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingServiceUnitTests.scala index 076b916b68..297cda9cc0 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingServiceUnitTests.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingServiceUnitTests.scala @@ -282,6 +282,13 @@ class WorkspaceSettingServiceUnitTests extends AnyFlatSpec with MockitoTestUtils val workspaceRepository = mock[WorkspaceRepository] when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(List(existingSetting))) + when( + workspaceRepository.createWorkspaceSettingsRecords(workspaceId, + List.empty, + defaultRequestContext.userInfo.userSubjectId + ) + ) + .thenReturn(Future.successful(List.empty)) val samDAO = mock[SamDAO] when(samDAO.getUserStatus(any())) @@ -406,13 +413,12 @@ class WorkspaceSettingServiceUnitTests extends AnyFlatSpec with MockitoTestUtils exception.errorReport.statusCode shouldBe Some(StatusCodes.Forbidden) } - it should "validate requested settings" in { - val workspaceName = workspace.toWorkspaceName - val newSetting = WorkspaceSetting( + "validateSettings" should "require a non-negative age for GcpBucketLifecycle settings" in { + val negativeAgeSetting = WorkspaceSetting( WorkspaceSettingTypes.GcpBucketLifecycle, GcpBucketLifecycleConfig( List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("SetStorageClass"), + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Some(Set("prefixToMatch")), Some(-1)) ) ) @@ -422,13 +428,84 @@ class WorkspaceSettingServiceUnitTests extends AnyFlatSpec with MockitoTestUtils val service = workspaceSettingServiceConstructor() val exception = intercept[RawlsExceptionWithErrorReport] { - Await.result(service.setWorkspaceSettings(workspaceName, List(newSetting)), Duration.Inf) + Await.result(service.setWorkspaceSettings(workspace.toWorkspaceName, List(negativeAgeSetting)), Duration.Inf) + } + exception.errorReport.statusCode shouldBe Some(StatusCodes.BadRequest) + exception.errorReport.message should include("Invalid settings requested.") + exception.errorReport.causes should contain theSameElementsAs List( + ErrorReport("Invalid GcpBucketLifecycle configuration: age must be a non-negative integer.") + ) + } + + it should "not allow unsupported lifecycle actions for GcpBucketLifecycle settings" in { + val unsupportedActionSetting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpBucketLifecycleAction("SetStorageClass"), + GcpBucketLifecycleCondition(Some(Set("prefixToMatch")), Some(10)) + ) + ) + ) + ) + + val service = workspaceSettingServiceConstructor() + + val exception = intercept[RawlsExceptionWithErrorReport] { + Await.result(service.setWorkspaceSettings(workspace.toWorkspaceName, List(unsupportedActionSetting)), + Duration.Inf + ) } exception.errorReport.statusCode shouldBe Some(StatusCodes.BadRequest) exception.errorReport.message should include("Invalid settings requested.") exception.errorReport.causes should contain theSameElementsAs List( - ErrorReport("Invalid GcpBucketLifecycle configuration: age must be a non-negative integer."), ErrorReport("Invalid GcpBucketLifecycle configuration: unsupported lifecycle action SetStorageClass.") ) } + + it should "require at least one condition for GcpBucketLifecycle settings" in { + val noConditionsSetting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(None, None)) + ) + ) + ) + + val service = workspaceSettingServiceConstructor() + + val exception = intercept[RawlsExceptionWithErrorReport] { + Await.result(service.setWorkspaceSettings(workspace.toWorkspaceName, List(noConditionsSetting)), Duration.Inf) + } + exception.errorReport.statusCode shouldBe Some(StatusCodes.BadRequest) + exception.errorReport.message should include("Invalid settings requested.") + exception.errorReport.causes should contain theSameElementsAs List( + ErrorReport("Invalid GcpBucketLifecycle configuration: at least one condition must be specified.") + ) + } + + it should "require at least one prefix if matchesPrefix is the only condition for GcpBucketLifecycle settings" in { + val noPrefixSetting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), GcpBucketLifecycleCondition(Some(Set.empty), None)) + ) + ) + ) + + val service = workspaceSettingServiceConstructor() + + val exception = intercept[RawlsExceptionWithErrorReport] { + Await.result(service.setWorkspaceSettings(workspace.toWorkspaceName, List(noPrefixSetting)), Duration.Inf) + } + exception.errorReport.statusCode shouldBe Some(StatusCodes.BadRequest) + exception.errorReport.message should include("Invalid settings requested.") + exception.errorReport.causes should contain theSameElementsAs List( + ErrorReport( + "Invalid GcpBucketLifecycle configuration: at least one prefix must be specified if matchesPrefix is the only condition." + ) + ) + } } From 13c934da315e65a00f9b6cf530507c1bfa2cf78b Mon Sep 17 00:00:00 2001 From: Marcus Talbott Date: Tue, 13 Aug 2024 11:00:20 -0400 Subject: [PATCH 21/25] swagger and fix provider routes --- core/src/main/resources/swagger/api-docs.yaml | 33 +++++++++++++++---- .../rawls/provider/RawlsProviderSpec.scala | 7 +++- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/core/src/main/resources/swagger/api-docs.yaml b/core/src/main/resources/swagger/api-docs.yaml index e25bca6031..e14839db63 100644 --- a/core/src/main/resources/swagger/api-docs.yaml +++ b/core/src/main/resources/swagger/api-docs.yaml @@ -4441,7 +4441,7 @@ paths: schema: type: array items: - $ref: '#/components/schemas/WorkspaceSettings' + $ref: '#/components/schemas/WorkspaceSetting' 403: description: User does not have access to requested workspace content: @@ -4459,9 +4459,9 @@ paths: put: tags: - workspaces_v2 - summary: Overwrite workspace settings - description: Overwrite the settings for a workspace - operationId: overwriteWorkspaceSettings + summary: Update workspace settings + description: Update the settings for a workspace. Unspecified setting types will be left unchanged. + operationId: updateWorkspaceSettings parameters: - $ref: '#/components/parameters/workspaceNamespacePathParam' - $ref: '#/components/parameters/workspaceNamePathParam' @@ -4472,7 +4472,7 @@ paths: schema: type: array items: - $ref: '#/components/schemas/WorkspaceSettings' + $ref: '#/components/schemas/WorkspaceSetting' required: true responses: 200: @@ -4480,7 +4480,7 @@ paths: content: 'application/json': schema: - $ref: '#/components/schemas/WorkspaceSettings' + $ref: '#/components/schemas/WorkspaceSettingsResponse' 400: description: Invalid settings content: @@ -5896,7 +5896,25 @@ components: - Deleting - DeleteFailed description: "" - WorkspaceSettings: + WorkspaceSettingsResponse: + required: + - successes + - failures + type: object + properties: + successes: + type: array + description: The settings that were successfully applied + items: + $ref: '#/components/schemas/WorkspaceSetting' + failures: + $ref: '#/components/schemas/FailedWorkspaceSettingsResponse' + FailedWorkspaceSettingsResponse: + type: object + description: The settings that failed to apply. The key is the setting type that failed. + additionalProperties: + $ref: '#/components/schemas/ErrorReport' + WorkspaceSetting: required: - settingType - config @@ -5943,6 +5961,7 @@ components: - Delete GcpBucketLifecycleCondition: type: object + minProperties: 1 properties: age: type: integer diff --git a/pact4s/src/test/scala/org/broadinstitute/dsde/rawls/provider/RawlsProviderSpec.scala b/pact4s/src/test/scala/org/broadinstitute/dsde/rawls/provider/RawlsProviderSpec.scala index aa28912730..3d96dd2f52 100644 --- a/pact4s/src/test/scala/org/broadinstitute/dsde/rawls/provider/RawlsProviderSpec.scala +++ b/pact4s/src/test/scala/org/broadinstitute/dsde/rawls/provider/RawlsProviderSpec.scala @@ -19,7 +19,7 @@ import org.broadinstitute.dsde.rawls.spendreporting.SpendReportingService import org.broadinstitute.dsde.rawls.status.StatusService import org.broadinstitute.dsde.rawls.user.UserService import org.broadinstitute.dsde.rawls.webservice.RawlsApiServiceImpl -import org.broadinstitute.dsde.rawls.workspace.{MultiCloudWorkspaceService, WorkspaceService} +import org.broadinstitute.dsde.rawls.workspace.{MultiCloudWorkspaceService, WorkspaceService, WorkspaceSettingService} import org.broadinstitute.dsde.workbench.oauth2.OpenIDConnectConfiguration import org.mockito.ArgumentMatchers.{any, anyInt, anyString} import org.mockito.Mockito.{reset, when} @@ -95,6 +95,10 @@ class RawlsProviderSpec extends AnyFlatSpec with BeforeAndAfterAll with PactVeri lazy val mockWorkspaceService: WorkspaceService = mock[WorkspaceService] _ => mockWorkspaceService } + val mockWorkspaceSettingServiceConstructor: RawlsRequestContext => WorkspaceSettingService = { + lazy val mockWorkspaceSettingService: WorkspaceSettingService = mock[WorkspaceSettingService] + _ => mockWorkspaceSettingService + } val mockMethodConfigServiceConstructor: RawlsRequestContext => MethodConfigurationService = { lazy val mockMethodConfigService: MethodConfigurationService = mock[MethodConfigurationService] _ => mockMethodConfigService @@ -142,6 +146,7 @@ class RawlsProviderSpec extends AnyFlatSpec with BeforeAndAfterAll with PactVeri val rawlsApiService = new RawlsApiServiceImpl( mockMultiCloudWorkspaceServiceConstructor, mockWorkspaceServiceConstructor, + mockWorkspaceSettingServiceConstructor, mockEntityServiceConstructor, mockUserServiceConstructor, mockGenomicsServiceConstructor, From c5232c2c9c63be1ac4ead61073eb6550800ef35b Mon Sep 17 00:00:00 2001 From: Marcus Talbott Date: Tue, 13 Aug 2024 15:38:39 -0400 Subject: [PATCH 22/25] fix unit tests --- .../dsde/rawls/dataaccess/HttpGoogleServicesDAOSpec.scala | 4 +--- .../dsde/rawls/model/WorkspaceModelSpec.scala | 8 ++++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpGoogleServicesDAOSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpGoogleServicesDAOSpec.scala index a776785643..4ece28149d 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpGoogleServicesDAOSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpGoogleServicesDAOSpec.scala @@ -211,7 +211,7 @@ class HttpGoogleServicesDAOSpec extends AnyFlatSpec with Matchers with MockitoTe ) verify( - googleStorageService.insertBucket( + googleStorageService, times(1)).insertBucket( ArgumentMatchers.eq(GoogleProject(googleProjectId)), ArgumentMatchers.eq(bucketName), any(), @@ -225,8 +225,6 @@ class HttpGoogleServicesDAOSpec extends AnyFlatSpec with Matchers with MockitoTe autoclassEnabled = ArgumentMatchers.eq(true), autoclassTerminalStorageClass = ArgumentMatchers.eq(Option(StorageClass.ARCHIVE)), cors = ArgumentMatchers.eq(expectedCorsPolicy) - ), - times(1) ) } } diff --git a/model/src/test/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModelSpec.scala b/model/src/test/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModelSpec.scala index bfaf845e24..0837aeb169 100644 --- a/model/src/test/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModelSpec.scala +++ b/model/src/test/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModelSpec.scala @@ -699,7 +699,7 @@ class WorkspaceModelSpec extends AnyFreeSpec with Matchers { | "rules": [ | { | "action": { - | "type": "Delete" + | "actionType": "Delete" | }, | "conditions": { | "age": 30, @@ -736,7 +736,7 @@ class WorkspaceModelSpec extends AnyFreeSpec with Matchers { | "rules": [ | { | "action": { - | "type": "Delete" + | "actionType": "Delete" | }, | "conditions": { | "age": 30, @@ -770,7 +770,7 @@ class WorkspaceModelSpec extends AnyFreeSpec with Matchers { | "rules": [ | { | "action": { - | "type": "Delete" + | "actionType": "Delete" | }, | "conditions": { | "matchesPrefix": [ @@ -878,7 +878,7 @@ class WorkspaceModelSpec extends AnyFreeSpec with Matchers { | "rules": [ | { | "action": { - | "type": "Delete" + | "actionType": "Delete" | } | } | ] From 118f779f035cb04c049fb2524011416faec76aa8 Mon Sep 17 00:00:00 2001 From: Marcus Talbott Date: Tue, 13 Aug 2024 15:39:07 -0400 Subject: [PATCH 23/25] scalafmt --- .../HttpGoogleServicesDAOSpec.scala | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpGoogleServicesDAOSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpGoogleServicesDAOSpec.scala index 4ece28149d..f99cb52bb6 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpGoogleServicesDAOSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpGoogleServicesDAOSpec.scala @@ -210,21 +210,20 @@ class HttpGoogleServicesDAOSpec extends AnyFlatSpec with Matchers with MockitoTe None ) - verify( - googleStorageService, times(1)).insertBucket( - ArgumentMatchers.eq(GoogleProject(googleProjectId)), - ArgumentMatchers.eq(bucketName), - any(), - any(), - any(), - any(), - any(), - any(), - any(), - any(), - autoclassEnabled = ArgumentMatchers.eq(true), - autoclassTerminalStorageClass = ArgumentMatchers.eq(Option(StorageClass.ARCHIVE)), - cors = ArgumentMatchers.eq(expectedCorsPolicy) + verify(googleStorageService, times(1)).insertBucket( + ArgumentMatchers.eq(GoogleProject(googleProjectId)), + ArgumentMatchers.eq(bucketName), + any(), + any(), + any(), + any(), + any(), + any(), + any(), + any(), + autoclassEnabled = ArgumentMatchers.eq(true), + autoclassTerminalStorageClass = ArgumentMatchers.eq(Option(StorageClass.ARCHIVE)), + cors = ArgumentMatchers.eq(expectedCorsPolicy) ) } } From 45ab8261780e4cc65c1e3df5bc92e4dfb6e3995e Mon Sep 17 00:00:00 2001 From: Marcus Talbott Date: Wed, 14 Aug 2024 15:08:49 -0400 Subject: [PATCH 24/25] break out WorkspaceSettingRepository --- .../org/broadinstitute/dsde/rawls/Boot.scala | 4 +- .../rawls/workspace/WorkspaceRepository.scala | 63 ---- .../WorkspaceSettingRepository.scala | 84 ++++++ .../workspace/WorkspaceSettingService.scala | 19 +- .../rawls/webservice/ApiServiceSpec.scala | 8 +- .../workspace/WorkspaceRepositorySpec.scala | 242 --------------- .../WorkspaceSettingRepositorySpec.scala | 285 ++++++++++++++++++ .../WorkspaceSettingServiceUnitTests.scala | 108 ++++--- 8 files changed, 465 insertions(+), 348 deletions(-) create mode 100644 core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingRepository.scala create mode 100644 core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingRepositorySpec.scala diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/Boot.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/Boot.scala index 88f143b80e..4ca1058d20 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/Boot.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/Boot.scala @@ -62,6 +62,7 @@ import org.broadinstitute.dsde.rawls.workspace.{ RawlsWorkspaceAclManager, WorkspaceRepository, WorkspaceService, + WorkspaceSettingRepository, WorkspaceSettingService } import org.broadinstitute.dsde.workbench.google.GoogleCredentialModes.Json @@ -511,8 +512,9 @@ object Boot extends IOApp with LazyLogging { val bucketMigrationServiceConstructor: RawlsRequestContext => BucketMigrationService = BucketMigrationServiceFactory.createBucketMigrationService(appConfigManager, slickDataSource, samDAO, gcsDAO) + val workspaceSettingRepository = new WorkspaceSettingRepository(slickDataSource) val workspaceSettingServiceConstructor: RawlsRequestContext => WorkspaceSettingService = - new WorkspaceSettingService(_, workspaceRepository, gcsDAO, samDAO) + new WorkspaceSettingService(_, workspaceSettingRepository, workspaceRepository, gcsDAO, samDAO) val service = new RawlsApiServiceImpl( multiCloudWorkspaceServiceConstructor, diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala index 8eea32dd58..42ae0fb46d 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala @@ -103,67 +103,4 @@ class WorkspaceRepository(dataSource: SlickDataSource) { ) } yield newWorkspace } - - // Return all applied settings for a workspace. Deleted and pending settings are not returned. - def getWorkspaceSettings(workspaceId: UUID): Future[List[WorkspaceSetting]] = - dataSource.inTransaction { access => - access.workspaceSettingQuery.listSettingsForWorkspaceByStatus(workspaceId, - WorkspaceSettingRecord.SettingStatus.Applied - ) - } - - // Create new settings for a workspace as pending. If there are any existing pending settings, throw an exception. - def createWorkspaceSettingsRecords(workspaceId: UUID, - workspaceSettings: List[WorkspaceSetting], - userId: RawlsUserSubjectId - )(implicit - ec: ExecutionContext - ): Future[List[WorkspaceSetting]] = - dataSource.inTransaction( - access => - for { - pendingSettingsForWorkspace <- access.workspaceSettingQuery.listSettingsForWorkspaceByStatus( - workspaceId, - WorkspaceSettingRecord.SettingStatus.Pending - ) - _ = if (pendingSettingsForWorkspace.nonEmpty) { - throw new RawlsExceptionWithErrorReport( - ErrorReport(StatusCodes.Conflict, s"Workspace $workspaceId already has pending settings") - ) - } - _ <- access.workspaceSettingQuery.saveAll(workspaceId, workspaceSettings, userId) - } yield workspaceSettings - // We use Serializable here to ensure that concurrent transactions to create settings - // for the same workspace are committed in order and do not interfere with each other - , - TransactionIsolation.Serializable - ) - - // Transition old Applied settings to Deleted and Pending settings to Applied - def markWorkspaceSettingApplied(workspaceId: UUID, workspaceSettingType: WorkspaceSettingType)(implicit - ec: ExecutionContext - ): Future[Int] = - dataSource.inTransaction { access => - for { - _ <- access.workspaceSettingQuery.updateSettingStatus(workspaceId, - workspaceSettingType, - WorkspaceSettingRecord.SettingStatus.Applied, - WorkspaceSettingRecord.SettingStatus.Deleted - ) - res <- access.workspaceSettingQuery.updateSettingStatus(workspaceId, - workspaceSettingType, - WorkspaceSettingRecord.SettingStatus.Pending, - WorkspaceSettingRecord.SettingStatus.Applied - ) - } yield res - } - - // Fully remove all pending records for a workspace from database. Does not transition records to Deleted. - def removePendingSetting(workspaceId: UUID, workspaceSettingType: WorkspaceSettingType): Future[Int] = - dataSource.inTransaction { access => - access.workspaceSettingQuery.deleteSettingTypeForWorkspaceByStatus(workspaceId, - workspaceSettingType, - WorkspaceSettingRecord.SettingStatus.Pending - ) - } } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingRepository.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingRepository.scala new file mode 100644 index 0000000000..3fcbbbc9de --- /dev/null +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingRepository.scala @@ -0,0 +1,84 @@ +package org.broadinstitute.dsde.rawls.workspace + +import akka.http.scaladsl.model.StatusCodes +import org.broadinstitute.dsde.rawls.RawlsExceptionWithErrorReport +import org.broadinstitute.dsde.rawls.dataaccess.SlickDataSource +import org.broadinstitute.dsde.rawls.dataaccess.slick.WorkspaceSettingRecord +import org.broadinstitute.dsde.rawls.model.{ErrorReport, RawlsUserSubjectId, WorkspaceSetting} +import org.broadinstitute.dsde.rawls.model.WorkspaceSettingTypes.WorkspaceSettingType +import slick.jdbc.TransactionIsolation + +import java.util.UUID +import scala.concurrent.{ExecutionContext, Future} + +/** + * Data access for rawls workspace settings + * + * The intention of this class is to hide direct dependencies on Slick behind a relatively clean interface + * to ease testability of higher level business logic. + */ +class WorkspaceSettingRepository(dataSource: SlickDataSource) { + + // Return all applied settings for a workspace. Deleted and pending settings are not returned. + def getWorkspaceSettings(workspaceId: UUID): Future[List[WorkspaceSetting]] = + dataSource.inTransaction { access => + access.workspaceSettingQuery.listSettingsForWorkspaceByStatus(workspaceId, + WorkspaceSettingRecord.SettingStatus.Applied + ) + } + + // Create new settings for a workspace as pending. If there are any existing pending settings, throw an exception. + def createWorkspaceSettingsRecords(workspaceId: UUID, + workspaceSettings: List[WorkspaceSetting], + userId: RawlsUserSubjectId + )(implicit + ec: ExecutionContext + ): Future[List[WorkspaceSetting]] = + dataSource.inTransaction( + access => + for { + pendingSettingsForWorkspace <- access.workspaceSettingQuery.listSettingsForWorkspaceByStatus( + workspaceId, + WorkspaceSettingRecord.SettingStatus.Pending + ) + _ = if (pendingSettingsForWorkspace.nonEmpty) { + throw new RawlsExceptionWithErrorReport( + ErrorReport(StatusCodes.Conflict, s"Workspace $workspaceId already has pending settings") + ) + } + _ <- access.workspaceSettingQuery.saveAll(workspaceId, workspaceSettings, userId) + } yield workspaceSettings + // We use Serializable here to ensure that concurrent transactions to create settings + // for the same workspace are committed in order and do not interfere with each other + , + TransactionIsolation.Serializable + ) + + // Transition old Applied settings to Deleted and Pending settings to Applied + def markWorkspaceSettingApplied(workspaceId: UUID, workspaceSettingType: WorkspaceSettingType)(implicit + ec: ExecutionContext + ): Future[Int] = + dataSource.inTransaction { access => + for { + _ <- access.workspaceSettingQuery.updateSettingStatus(workspaceId, + workspaceSettingType, + WorkspaceSettingRecord.SettingStatus.Applied, + WorkspaceSettingRecord.SettingStatus.Deleted + ) + res <- access.workspaceSettingQuery.updateSettingStatus(workspaceId, + workspaceSettingType, + WorkspaceSettingRecord.SettingStatus.Pending, + WorkspaceSettingRecord.SettingStatus.Applied + ) + } yield res + } + + // Fully remove all pending records for a workspace from database. Does not transition records to Deleted. + def removePendingSetting(workspaceId: UUID, workspaceSettingType: WorkspaceSettingType): Future[Int] = + dataSource.inTransaction { access => + access.workspaceSettingQuery.deleteSettingTypeForWorkspaceByStatus(workspaceId, + workspaceSettingType, + WorkspaceSettingRecord.SettingStatus.Pending + ) + } +} diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingService.scala index 6489bd7db1..5035a6e9b3 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingService.scala @@ -28,15 +28,18 @@ import scala.concurrent.{ExecutionContext, Future} import scala.jdk.CollectionConverters._ class WorkspaceSettingService(protected val ctx: RawlsRequestContext, + workspaceSettingRepository: WorkspaceSettingRepository, val workspaceRepository: WorkspaceRepository, gcsDAO: GoogleServicesDAO, val samDAO: SamDAO )(implicit protected val executionContext: ExecutionContext) extends WorkspaceSupport with LazyLogging { + + // Returns applied settings on a workspace. def getWorkspaceSettings(workspaceName: WorkspaceName): Future[List[WorkspaceSetting]] = getV2WorkspaceContextAndPermissions(workspaceName, SamWorkspaceActions.read).flatMap { workspace => - workspaceRepository.getWorkspaceSettings(workspace.workspaceIdAsUUID) + workspaceSettingRepository.getWorkspaceSettings(workspace.workspaceIdAsUUID) } def setWorkspaceSettings(workspaceName: WorkspaceName, @@ -117,7 +120,9 @@ class WorkspaceSettingService(protected val ctx: RawlsRequestContext, for { _ <- gcsDAO.setBucketLifecycle(workspace.bucketName, googleRules) - _ <- workspaceRepository.markWorkspaceSettingApplied(workspace.workspaceIdAsUUID, setting.settingType) + _ <- workspaceSettingRepository.markWorkspaceSettingApplied(workspace.workspaceIdAsUUID, + setting.settingType + ) } yield None case _ => throw new RawlsException("unsupported workspace setting") }).recoverWith { case e => @@ -125,7 +130,7 @@ class WorkspaceSettingService(protected val ctx: RawlsRequestContext, s"Failed to apply settings. [workspaceId=${workspace.workspaceIdAsUUID},settingType=${setting.settingType}]", e ) - workspaceRepository + workspaceSettingRepository .removePendingSetting(workspace.workspaceIdAsUUID, setting.settingType) .map(_ => Some((setting.settingType, ErrorReport(StatusCodes.InternalServerError, e.getMessage)))) } @@ -133,11 +138,11 @@ class WorkspaceSettingService(protected val ctx: RawlsRequestContext, validateSettings(workspaceSettings) for { workspace <- getV2WorkspaceContextAndPermissions(workspaceName, SamWorkspaceActions.own) - currentSettings <- workspaceRepository.getWorkspaceSettings(workspace.workspaceIdAsUUID) + currentSettings <- workspaceSettingRepository.getWorkspaceSettings(workspace.workspaceIdAsUUID) newSettings = workspaceSettings.filterNot(currentSettings.contains(_)) - _ <- workspaceRepository.createWorkspaceSettingsRecords(workspace.workspaceIdAsUUID, - newSettings, - ctx.userInfo.userSubjectId + _ <- workspaceSettingRepository.createWorkspaceSettingsRecords(workspace.workspaceIdAsUUID, + newSettings, + ctx.userInfo.userSubjectId ) applyFailures <- newSettings.traverse(s => applySetting(workspace, s)) } yield { diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/ApiServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/ApiServiceSpec.scala index 38592cd2dd..535628642e 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/ApiServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/ApiServiceSpec.scala @@ -55,6 +55,7 @@ import org.broadinstitute.dsde.rawls.workspace.{ RawlsWorkspaceAclManager, WorkspaceRepository, WorkspaceService, + WorkspaceSettingRepository, WorkspaceSettingService } import org.broadinstitute.dsde.workbench.dataaccess.{NotificationDAO, PubSubNotificationDAO} @@ -395,7 +396,12 @@ trait ApiServiceSpec ) override val workspaceSettingServiceConstructor: RawlsRequestContext => WorkspaceSettingService = - new WorkspaceSettingService(_, new WorkspaceRepository(slickDataSource), gcsDAO, samDAO) + new WorkspaceSettingService(_, + new WorkspaceSettingRepository(slickDataSource), + new WorkspaceRepository(slickDataSource), + gcsDAO, + samDAO + ) override val methodConfigurationServiceConstructor: RawlsRequestContext => MethodConfigurationService = MethodConfigurationService.constructor( diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepositorySpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepositorySpec.scala index 7c48a541bf..2a55e79f1d 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepositorySpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepositorySpec.scala @@ -127,246 +127,4 @@ class WorkspaceRepositorySpec assertResult(readback)(None) assertResult(result)(true) } - - behavior of "getWorkspaceSettings" - - it should "return the applied workspace settings" in { - val repo = new WorkspaceRepository(slickDataSource) - val ws: Workspace = makeWorkspace() - Await.result(repo.createWorkspace(ws), Duration.Inf) - val appliedSetting = WorkspaceSetting( - WorkspaceSettingTypes.GcpBucketLifecycle, - GcpBucketLifecycleConfig( - List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), - GcpBucketLifecycleCondition(Some(Set("applied")), Some(30)) - ) - ) - ) - ) - val pendingSetting = WorkspaceSetting( - WorkspaceSettingTypes.GcpBucketLifecycle, - GcpBucketLifecycleConfig( - List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), - GcpBucketLifecycleCondition(Some(Set("pending")), Some(31)) - ) - ) - ) - ) - - Await.result( - slickDataSource.inTransaction { dataAccess => - for { - _ <- dataAccess.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, - List(appliedSetting), - userInfo.userSubjectId - ) - _ <- dataAccess.workspaceSettingQuery.updateSettingStatus( - ws.workspaceIdAsUUID, - WorkspaceSettingTypes.GcpBucketLifecycle, - WorkspaceSettingRecord.SettingStatus.Pending, - WorkspaceSettingRecord.SettingStatus.Applied - ) - _ <- dataAccess.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, - List(pendingSetting), - userInfo.userSubjectId - ) - } yield () - }, - Duration.Inf - ) - - val result = Await.result(repo.getWorkspaceSettings(ws.workspaceIdAsUUID), Duration.Inf) - - assertResult(result)(List(appliedSetting)) - } - - behavior of "createWorkspaceSettingsRecords" - - it should "create pending workspace settings" in { - val repo = new WorkspaceRepository(slickDataSource) - val ws: Workspace = makeWorkspace() - Await.result(repo.createWorkspace(ws), Duration.Inf) - val setting = WorkspaceSetting( - WorkspaceSettingTypes.GcpBucketLifecycle, - GcpBucketLifecycleConfig( - List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), - GcpBucketLifecycleCondition(Some(Set("newSetting")), Some(30)) - ) - ) - ) - ) - - Await.result(repo.createWorkspaceSettingsRecords(ws.workspaceIdAsUUID, List(setting), userInfo.userSubjectId), - Duration.Inf - ) - - val newSettings = Await.result( - slickDataSource.inTransaction( - _.workspaceSettingQuery.listSettingsForWorkspaceByStatus(ws.workspaceIdAsUUID, - WorkspaceSettingRecord.SettingStatus.Pending - ) - ), - Duration.Inf - ) - assertResult(newSettings)(List(setting)) - } - - it should "throw an exception if there are already pending settings" in { - val repo = new WorkspaceRepository(slickDataSource) - val ws: Workspace = makeWorkspace() - Await.result(repo.createWorkspace(ws), Duration.Inf) - val setting = WorkspaceSetting( - WorkspaceSettingTypes.GcpBucketLifecycle, - GcpBucketLifecycleConfig( - List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), - GcpBucketLifecycleCondition(Some(Set("newSetting")), Some(30)) - ) - ) - ) - ) - - Await.result(slickDataSource.inTransaction( - _.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, List(setting), userInfo.userSubjectId) - ), - Duration.Inf - ) - - val thrown = intercept[RawlsExceptionWithErrorReport] { - Await.result(repo.createWorkspaceSettingsRecords(ws.workspaceIdAsUUID, List(setting), userInfo.userSubjectId), - Duration.Inf - ) - } - thrown.errorReport.statusCode shouldBe Some(StatusCodes.Conflict) - } - - behavior of "markWorkspaceSettingApplied" - - it should "mark Pending settings as Applied and Applied settings as Deleted" in { - val repo = new WorkspaceRepository(slickDataSource) - val ws: Workspace = makeWorkspace() - Await.result(repo.createWorkspace(ws), Duration.Inf) - val existingSetting = WorkspaceSetting( - WorkspaceSettingTypes.GcpBucketLifecycle, - GcpBucketLifecycleConfig( - List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), - GcpBucketLifecycleCondition(Some(Set("applied")), Some(30)) - ) - ) - ) - ) - val newSetting = WorkspaceSetting( - WorkspaceSettingTypes.GcpBucketLifecycle, - GcpBucketLifecycleConfig( - List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), - GcpBucketLifecycleCondition(Some(Set("pending")), Some(31)) - ) - ) - ) - ) - - Await.result( - slickDataSource.inTransaction { dataAccess => - for { - _ <- dataAccess.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, - List(existingSetting), - userInfo.userSubjectId - ) - _ <- dataAccess.workspaceSettingQuery.updateSettingStatus( - ws.workspaceIdAsUUID, - WorkspaceSettingTypes.GcpBucketLifecycle, - WorkspaceSettingRecord.SettingStatus.Pending, - WorkspaceSettingRecord.SettingStatus.Applied - ) - _ <- dataAccess.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, List(newSetting), userInfo.userSubjectId) - } yield () - }, - Duration.Inf - ) - - Await.result(repo.markWorkspaceSettingApplied(ws.workspaceIdAsUUID, WorkspaceSettingTypes.GcpBucketLifecycle), - Duration.Inf - ) - - // existing settings should now be deleted - val deletedSettings = Await.result( - slickDataSource.inTransaction( - _.workspaceSettingQuery.listSettingsForWorkspaceByStatus(ws.workspaceIdAsUUID, - WorkspaceSettingRecord.SettingStatus.Deleted - ) - ), - Duration.Inf - ) - assertResult(deletedSettings)(List(existingSetting)) - - // new settings should now be applied - val appliedSettings = Await.result( - slickDataSource.inTransaction( - _.workspaceSettingQuery.listSettingsForWorkspaceByStatus(ws.workspaceIdAsUUID, - WorkspaceSettingRecord.SettingStatus.Applied - ) - ), - Duration.Inf - ) - assertResult(appliedSettings)(List(newSetting)) - } - - behavior of "removePendingSetting" - - it should "delete the pending workspace setting" in { - val repo = new WorkspaceRepository(slickDataSource) - val ws: Workspace = makeWorkspace() - Await.result(repo.createWorkspace(ws), Duration.Inf) - val setting = WorkspaceSetting( - WorkspaceSettingTypes.GcpBucketLifecycle, - GcpBucketLifecycleConfig( - List( - GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), - GcpBucketLifecycleCondition(Some(Set("newSetting")), Some(30)) - ) - ) - ) - ) - - Await.result(slickDataSource.inTransaction( - _.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, List(setting), userInfo.userSubjectId) - ), - Duration.Inf - ) - - Await.result(repo.removePendingSetting(ws.workspaceIdAsUUID, WorkspaceSettingTypes.GcpBucketLifecycle), - Duration.Inf - ) - - // There should be no workspace settings in the database. Failed pending settings are not kept. - Await.result( - slickDataSource.inTransaction( - _.workspaceSettingQuery.listSettingsForWorkspaceByStatus(ws.workspaceIdAsUUID, - WorkspaceSettingRecord.SettingStatus.Deleted - ) - ), - Duration.Inf - ) shouldBe empty - Await.result( - slickDataSource.inTransaction( - _.workspaceSettingQuery.listSettingsForWorkspaceByStatus(ws.workspaceIdAsUUID, - WorkspaceSettingRecord.SettingStatus.Applied - ) - ), - Duration.Inf - ) shouldBe empty - Await.result( - slickDataSource.inTransaction( - _.workspaceSettingQuery.listSettingsForWorkspaceByStatus(ws.workspaceIdAsUUID, - WorkspaceSettingRecord.SettingStatus.Pending - ) - ), - Duration.Inf - ) shouldBe empty - } } diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingRepositorySpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingRepositorySpec.scala new file mode 100644 index 0000000000..e9dfe31753 --- /dev/null +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingRepositorySpec.scala @@ -0,0 +1,285 @@ +package org.broadinstitute.dsde.rawls.workspace + +import akka.http.scaladsl.model.StatusCodes +import org.broadinstitute.dsde.rawls.RawlsExceptionWithErrorReport +import org.broadinstitute.dsde.rawls.dataaccess.slick.{TestDriverComponent, WorkspaceSettingRecord} +import org.broadinstitute.dsde.rawls.model.WorkspaceSettingConfig.{ + GcpBucketLifecycleAction, + GcpBucketLifecycleCondition, + GcpBucketLifecycleConfig, + GcpBucketLifecycleRule +} +import org.broadinstitute.dsde.rawls.model.{Workspace, WorkspaceSetting, WorkspaceSettingTypes} +import org.joda.time.DateTime +import org.scalatest.concurrent.ScalaFutures +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.mockito.MockitoSugar + +import java.util.UUID +import scala.concurrent.Await +import scala.concurrent.duration.Duration + +class WorkspaceSettingRepositorySpec + extends AnyFlatSpec + with MockitoSugar + with ScalaFutures + with Matchers + with TestDriverComponent { + + def makeWorkspace(): Workspace = Workspace.buildReadyMcWorkspace("fake-ns", + s"test-${UUID.randomUUID().toString}", + UUID.randomUUID().toString, + DateTime.now(), + DateTime.now(), + "fake@example.com", + Map.empty + ) + + behavior of "getWorkspaceSettings" + + it should "return the applied workspace settings" in { + val repo = new WorkspaceSettingRepository(slickDataSource) + val workspaceRepo = new WorkspaceRepository(slickDataSource) + val ws: Workspace = makeWorkspace() + Await.result(workspaceRepo.createWorkspace(ws), Duration.Inf) + val appliedSetting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), + GcpBucketLifecycleCondition(Some(Set("applied")), Some(30)) + ) + ) + ) + ) + val pendingSetting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), + GcpBucketLifecycleCondition(Some(Set("pending")), Some(31)) + ) + ) + ) + ) + + Await.result( + slickDataSource.inTransaction { dataAccess => + for { + _ <- dataAccess.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, + List(appliedSetting), + userInfo.userSubjectId + ) + _ <- dataAccess.workspaceSettingQuery.updateSettingStatus( + ws.workspaceIdAsUUID, + WorkspaceSettingTypes.GcpBucketLifecycle, + WorkspaceSettingRecord.SettingStatus.Pending, + WorkspaceSettingRecord.SettingStatus.Applied + ) + _ <- dataAccess.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, + List(pendingSetting), + userInfo.userSubjectId + ) + } yield () + }, + Duration.Inf + ) + + val result = Await.result(repo.getWorkspaceSettings(ws.workspaceIdAsUUID), Duration.Inf) + + assertResult(result)(List(appliedSetting)) + } + + behavior of "createWorkspaceSettingsRecords" + + it should "create pending workspace settings" in { + val repo = new WorkspaceSettingRepository(slickDataSource) + val workspaceRepo = new WorkspaceRepository(slickDataSource) + val ws: Workspace = makeWorkspace() + Await.result(workspaceRepo.createWorkspace(ws), Duration.Inf) + val setting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), + GcpBucketLifecycleCondition(Some(Set("newSetting")), Some(30)) + ) + ) + ) + ) + + Await.result(repo.createWorkspaceSettingsRecords(ws.workspaceIdAsUUID, List(setting), userInfo.userSubjectId), + Duration.Inf + ) + + val newSettings = Await.result( + slickDataSource.inTransaction( + _.workspaceSettingQuery.listSettingsForWorkspaceByStatus(ws.workspaceIdAsUUID, + WorkspaceSettingRecord.SettingStatus.Pending + ) + ), + Duration.Inf + ) + assertResult(newSettings)(List(setting)) + } + + it should "throw an exception if there are already pending settings" in { + val repo = new WorkspaceSettingRepository(slickDataSource) + val workspaceRepo = new WorkspaceRepository(slickDataSource) + val ws: Workspace = makeWorkspace() + Await.result(workspaceRepo.createWorkspace(ws), Duration.Inf) + val setting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), + GcpBucketLifecycleCondition(Some(Set("newSetting")), Some(30)) + ) + ) + ) + ) + + Await.result(slickDataSource.inTransaction( + _.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, List(setting), userInfo.userSubjectId) + ), + Duration.Inf + ) + + val thrown = intercept[RawlsExceptionWithErrorReport] { + Await.result(repo.createWorkspaceSettingsRecords(ws.workspaceIdAsUUID, List(setting), userInfo.userSubjectId), + Duration.Inf + ) + } + thrown.errorReport.statusCode shouldBe Some(StatusCodes.Conflict) + } + + behavior of "markWorkspaceSettingApplied" + + it should "mark Pending settings as Applied and Applied settings as Deleted" in { + val repo = new WorkspaceSettingRepository(slickDataSource) + val workspaceRepo = new WorkspaceRepository(slickDataSource) + val ws: Workspace = makeWorkspace() + Await.result(workspaceRepo.createWorkspace(ws), Duration.Inf) + val existingSetting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), + GcpBucketLifecycleCondition(Some(Set("applied")), Some(30)) + ) + ) + ) + ) + val newSetting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), + GcpBucketLifecycleCondition(Some(Set("pending")), Some(31)) + ) + ) + ) + ) + + Await.result( + slickDataSource.inTransaction { dataAccess => + for { + _ <- dataAccess.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, + List(existingSetting), + userInfo.userSubjectId + ) + _ <- dataAccess.workspaceSettingQuery.updateSettingStatus( + ws.workspaceIdAsUUID, + WorkspaceSettingTypes.GcpBucketLifecycle, + WorkspaceSettingRecord.SettingStatus.Pending, + WorkspaceSettingRecord.SettingStatus.Applied + ) + _ <- dataAccess.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, List(newSetting), userInfo.userSubjectId) + } yield () + }, + Duration.Inf + ) + + Await.result(repo.markWorkspaceSettingApplied(ws.workspaceIdAsUUID, WorkspaceSettingTypes.GcpBucketLifecycle), + Duration.Inf + ) + + // existing settings should now be deleted + val deletedSettings = Await.result( + slickDataSource.inTransaction( + _.workspaceSettingQuery.listSettingsForWorkspaceByStatus(ws.workspaceIdAsUUID, + WorkspaceSettingRecord.SettingStatus.Deleted + ) + ), + Duration.Inf + ) + assertResult(deletedSettings)(List(existingSetting)) + + // new settings should now be applied + val appliedSettings = Await.result( + slickDataSource.inTransaction( + _.workspaceSettingQuery.listSettingsForWorkspaceByStatus(ws.workspaceIdAsUUID, + WorkspaceSettingRecord.SettingStatus.Applied + ) + ), + Duration.Inf + ) + assertResult(appliedSettings)(List(newSetting)) + } + + behavior of "removePendingSetting" + + it should "delete the pending workspace setting" in { + val repo = new WorkspaceSettingRepository(slickDataSource) + val workspaceRepo = new WorkspaceRepository(slickDataSource) + val ws: Workspace = makeWorkspace() + Await.result(workspaceRepo.createWorkspace(ws), Duration.Inf) + val setting = WorkspaceSetting( + WorkspaceSettingTypes.GcpBucketLifecycle, + GcpBucketLifecycleConfig( + List( + GcpBucketLifecycleRule(GcpBucketLifecycleAction("Delete"), + GcpBucketLifecycleCondition(Some(Set("newSetting")), Some(30)) + ) + ) + ) + ) + + Await.result(slickDataSource.inTransaction( + _.workspaceSettingQuery.saveAll(ws.workspaceIdAsUUID, List(setting), userInfo.userSubjectId) + ), + Duration.Inf + ) + + Await.result(repo.removePendingSetting(ws.workspaceIdAsUUID, WorkspaceSettingTypes.GcpBucketLifecycle), + Duration.Inf + ) + + // There should be no workspace settings in the database. Failed pending settings are not kept. + Await.result( + slickDataSource.inTransaction( + _.workspaceSettingQuery.listSettingsForWorkspaceByStatus(ws.workspaceIdAsUUID, + WorkspaceSettingRecord.SettingStatus.Deleted + ) + ), + Duration.Inf + ) shouldBe empty + Await.result( + slickDataSource.inTransaction( + _.workspaceSettingQuery.listSettingsForWorkspaceByStatus(ws.workspaceIdAsUUID, + WorkspaceSettingRecord.SettingStatus.Applied + ) + ), + Duration.Inf + ) shouldBe empty + Await.result( + slickDataSource.inTransaction( + _.workspaceSettingQuery.listSettingsForWorkspaceByStatus(ws.workspaceIdAsUUID, + WorkspaceSettingRecord.SettingStatus.Pending + ) + ), + Duration.Inf + ) shouldBe empty + } +} diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingServiceUnitTests.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingServiceUnitTests.scala index 297cda9cc0..2e9cfaa3ae 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingServiceUnitTests.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceSettingServiceUnitTests.scala @@ -48,13 +48,16 @@ class WorkspaceSettingServiceUnitTests extends AnyFlatSpec with MockitoTestUtils UserInfo(RawlsUserEmail("test"), OAuth2BearerToken("Bearer 123"), 123, RawlsUserSubjectId("abc")) ) - def workspaceSettingServiceConstructor(ctx: RawlsRequestContext = defaultRequestContext, - workspaceRepository: WorkspaceRepository = - mock[WorkspaceRepository](RETURNS_SMART_NULLS), - gcsDAO: GoogleServicesDAO = mock[GoogleServicesDAO](RETURNS_SMART_NULLS), - samDAO: SamDAO = mock[SamDAO](RETURNS_SMART_NULLS) + def workspaceSettingServiceConstructor( + ctx: RawlsRequestContext = defaultRequestContext, + workspaceSettingRepository: WorkspaceSettingRepository = mock[WorkspaceSettingRepository]( + RETURNS_SMART_NULLS + ), + workspaceRepository: WorkspaceRepository = mock[WorkspaceRepository](RETURNS_SMART_NULLS), + gcsDAO: GoogleServicesDAO = mock[GoogleServicesDAO](RETURNS_SMART_NULLS), + samDAO: SamDAO = mock[SamDAO](RETURNS_SMART_NULLS) ): WorkspaceSettingService = - new WorkspaceSettingService(ctx, workspaceRepository, gcsDAO, samDAO) + new WorkspaceSettingService(ctx, workspaceSettingRepository, workspaceRepository, gcsDAO, samDAO) val workspace: Workspace = Workspace( "settingsTestWorkspace", @@ -79,7 +82,9 @@ class WorkspaceSettingServiceUnitTests extends AnyFlatSpec with MockitoTestUtils val workspaceRepository = mock[WorkspaceRepository] when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) - when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(workspaceSettings)) + + val workspaceSettingRepository = mock[WorkspaceSettingRepository] + when(workspaceSettingRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(workspaceSettings)) val samDAO = mock[SamDAO] when(samDAO.getUserStatus(any())) @@ -93,7 +98,10 @@ class WorkspaceSettingServiceUnitTests extends AnyFlatSpec with MockitoTestUtils ).thenReturn(Future.successful(true)) val service = - workspaceSettingServiceConstructor(samDAO = samDAO, workspaceRepository = workspaceRepository) + workspaceSettingServiceConstructor(samDAO = samDAO, + workspaceRepository = workspaceRepository, + workspaceSettingRepository = workspaceSettingRepository + ) val returnedSettings = Await.result(service.getWorkspaceSettings(workspaceName), Duration.Inf) returnedSettings shouldEqual workspaceSettings @@ -110,7 +118,9 @@ class WorkspaceSettingServiceUnitTests extends AnyFlatSpec with MockitoTestUtils val workspaceRepository = mock[WorkspaceRepository] when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) - when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(List.empty)) + + val workspaceSettingRepository = mock[WorkspaceSettingRepository] + when(workspaceSettingRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(List.empty)) val samDAO = mock[SamDAO] when(samDAO.getUserStatus(any())) @@ -124,7 +134,10 @@ class WorkspaceSettingServiceUnitTests extends AnyFlatSpec with MockitoTestUtils ).thenReturn(Future.successful(true)) val service = - workspaceSettingServiceConstructor(samDAO = samDAO, workspaceRepository = workspaceRepository) + workspaceSettingServiceConstructor(samDAO = samDAO, + workspaceRepository = workspaceRepository, + workspaceSettingRepository = workspaceSettingRepository + ) val returnedSettings = Await.result(service.getWorkspaceSettings(workspaceName), Duration.Inf) returnedSettings shouldEqual List.empty @@ -169,15 +182,17 @@ class WorkspaceSettingServiceUnitTests extends AnyFlatSpec with MockitoTestUtils val workspaceRepository = mock[WorkspaceRepository] when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) - when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(List.empty)) + + val workspaceSettingRepository = mock[WorkspaceSettingRepository] + when(workspaceSettingRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(List.empty)) when( - workspaceRepository.createWorkspaceSettingsRecords(workspaceId, - List(workspaceSetting), - defaultRequestContext.userInfo.userSubjectId + workspaceSettingRepository.createWorkspaceSettingsRecords(workspaceId, + List(workspaceSetting), + defaultRequestContext.userInfo.userSubjectId ) ) .thenReturn(Future.successful(List(workspaceSetting))) - when(workspaceRepository.markWorkspaceSettingApplied(workspaceId, workspaceSetting.settingType)) + when(workspaceSettingRepository.markWorkspaceSettingApplied(workspaceId, workspaceSetting.settingType)) .thenReturn(Future.successful(1)) val samDAO = mock[SamDAO] @@ -195,7 +210,11 @@ class WorkspaceSettingServiceUnitTests extends AnyFlatSpec with MockitoTestUtils when(gcsDAO.setBucketLifecycle(workspace.bucketName, List())).thenReturn(Future.successful()) val service = - workspaceSettingServiceConstructor(samDAO = samDAO, workspaceRepository = workspaceRepository, gcsDAO = gcsDAO) + workspaceSettingServiceConstructor(samDAO = samDAO, + workspaceRepository = workspaceRepository, + gcsDAO = gcsDAO, + workspaceSettingRepository = workspaceSettingRepository + ) val res = Await.result(service.setWorkspaceSettings(workspaceName, List(workspaceSetting)), Duration.Inf) res.successes should contain theSameElementsAs List(workspaceSetting) @@ -228,15 +247,18 @@ class WorkspaceSettingServiceUnitTests extends AnyFlatSpec with MockitoTestUtils val workspaceRepository = mock[WorkspaceRepository] when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) - when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(List(existingSetting))) + + val workspaceSettingRepository = mock[WorkspaceSettingRepository] + when(workspaceSettingRepository.getWorkspaceSettings(workspaceId)) + .thenReturn(Future.successful(List(existingSetting))) when( - workspaceRepository.createWorkspaceSettingsRecords(workspaceId, - List(newSetting), - defaultRequestContext.userInfo.userSubjectId + workspaceSettingRepository.createWorkspaceSettingsRecords(workspaceId, + List(newSetting), + defaultRequestContext.userInfo.userSubjectId ) ) .thenReturn(Future.successful(List(newSetting))) - when(workspaceRepository.markWorkspaceSettingApplied(workspaceId, newSetting.settingType)) + when(workspaceSettingRepository.markWorkspaceSettingApplied(workspaceId, newSetting.settingType)) .thenReturn(Future.successful(1)) val samDAO = mock[SamDAO] @@ -258,7 +280,11 @@ class WorkspaceSettingServiceUnitTests extends AnyFlatSpec with MockitoTestUtils when(gcsDAO.setBucketLifecycle(workspace.bucketName, List(newSettingGoogleRule))).thenReturn(Future.successful()) val service = - workspaceSettingServiceConstructor(samDAO = samDAO, workspaceRepository = workspaceRepository, gcsDAO = gcsDAO) + workspaceSettingServiceConstructor(samDAO = samDAO, + workspaceRepository = workspaceRepository, + gcsDAO = gcsDAO, + workspaceSettingRepository = workspaceSettingRepository + ) val res = Await.result(service.setWorkspaceSettings(workspaceName, List(newSetting)), Duration.Inf) res.successes should contain theSameElementsAs List(newSetting) @@ -281,11 +307,14 @@ class WorkspaceSettingServiceUnitTests extends AnyFlatSpec with MockitoTestUtils val workspaceRepository = mock[WorkspaceRepository] when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) - when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(List(existingSetting))) + + val workspaceSettingRepository = mock[WorkspaceSettingRepository] + when(workspaceSettingRepository.getWorkspaceSettings(workspaceId)) + .thenReturn(Future.successful(List(existingSetting))) when( - workspaceRepository.createWorkspaceSettingsRecords(workspaceId, - List.empty, - defaultRequestContext.userInfo.userSubjectId + workspaceSettingRepository.createWorkspaceSettingsRecords(workspaceId, + List.empty, + defaultRequestContext.userInfo.userSubjectId ) ) .thenReturn(Future.successful(List.empty)) @@ -301,7 +330,10 @@ class WorkspaceSettingServiceUnitTests extends AnyFlatSpec with MockitoTestUtils ) ).thenReturn(Future.successful(true)) - val service = workspaceSettingServiceConstructor(samDAO = samDAO, workspaceRepository = workspaceRepository) + val service = workspaceSettingServiceConstructor(samDAO = samDAO, + workspaceRepository = workspaceRepository, + workspaceSettingRepository = workspaceSettingRepository + ) val res = Await.result(service.setWorkspaceSettings(workspaceName, List.empty), Duration.Inf) res.successes shouldEqual List.empty @@ -334,15 +366,19 @@ class WorkspaceSettingServiceUnitTests extends AnyFlatSpec with MockitoTestUtils val workspaceRepository = mock[WorkspaceRepository] when(workspaceRepository.getWorkspace(workspaceName, None)).thenReturn(Future.successful(Option(workspace))) - when(workspaceRepository.getWorkspaceSettings(workspaceId)).thenReturn(Future.successful(List(existingSetting))) + + val workspaceSettingRepository = mock[WorkspaceSettingRepository] + when(workspaceSettingRepository.getWorkspaceSettings(workspaceId)) + .thenReturn(Future.successful(List(existingSetting))) when( - workspaceRepository.createWorkspaceSettingsRecords(workspaceId, - List(newSetting), - defaultRequestContext.userInfo.userSubjectId + workspaceSettingRepository.createWorkspaceSettingsRecords(workspaceId, + List(newSetting), + defaultRequestContext.userInfo.userSubjectId ) ) .thenReturn(Future.successful(List(newSetting))) - when(workspaceRepository.removePendingSetting(workspaceId, newSetting.settingType)).thenReturn(Future.successful(1)) + when(workspaceSettingRepository.removePendingSetting(workspaceId, newSetting.settingType)) + .thenReturn(Future.successful(1)) val samDAO = mock[SamDAO] when(samDAO.getUserStatus(any())) @@ -364,14 +400,18 @@ class WorkspaceSettingServiceUnitTests extends AnyFlatSpec with MockitoTestUtils .thenReturn(Future.failed(new Exception("failed to apply settings"))) val service = - workspaceSettingServiceConstructor(samDAO = samDAO, workspaceRepository = workspaceRepository, gcsDAO = gcsDAO) + workspaceSettingServiceConstructor(samDAO = samDAO, + workspaceRepository = workspaceRepository, + gcsDAO = gcsDAO, + workspaceSettingRepository = workspaceSettingRepository + ) val res = Await.result(service.setWorkspaceSettings(workspaceName, List(newSetting)), Duration.Inf) res.successes shouldEqual List.empty res.failures(WorkspaceSettingTypes.GcpBucketLifecycle) shouldEqual ErrorReport(StatusCodes.InternalServerError, "failed to apply settings" ) - verify(workspaceRepository).removePendingSetting(workspaceId, newSetting.settingType) + verify(workspaceSettingRepository).removePendingSetting(workspaceId, newSetting.settingType) } it should "be limited to owners" in { From 90b7f6baf9ceca4d28e9bd331223823ba1dbea81 Mon Sep 17 00:00:00 2001 From: Marcus Talbott Date: Wed, 14 Aug 2024 18:07:53 -0400 Subject: [PATCH 25/25] remove leftover imports --- .../dsde/rawls/workspace/WorkspaceRepository.scala | 5 ----- .../dsde/rawls/workspace/WorkspaceService.scala | 4 ---- 2 files changed, 9 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala index 42ae0fb46d..0be845c021 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala @@ -3,23 +3,18 @@ package org.broadinstitute.dsde.rawls.workspace import akka.http.scaladsl.model.StatusCodes import org.broadinstitute.dsde.rawls.RawlsExceptionWithErrorReport import org.broadinstitute.dsde.rawls.dataaccess.SlickDataSource -import org.broadinstitute.dsde.rawls.dataaccess.slick.WorkspaceSettingRecord import org.broadinstitute.dsde.rawls.model.Attributable.AttributeMap import org.broadinstitute.dsde.rawls.model.{ ErrorReport, RawlsRequestContext, - RawlsUserSubjectId, Workspace, WorkspaceAttributeSpecs, WorkspaceName, - WorkspaceSetting, WorkspaceState } import org.broadinstitute.dsde.rawls.model.WorkspaceState.WorkspaceState -import org.broadinstitute.dsde.rawls.model.WorkspaceSettingTypes.WorkspaceSettingType import org.broadinstitute.dsde.rawls.util.TracingUtils.traceDBIOWithParent import org.joda.time.DateTime -import slick.jdbc.TransactionIsolation import java.util.UUID import scala.concurrent.{ExecutionContext, Future} diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala index 06910400fc..23e477bb93 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala @@ -28,7 +28,6 @@ import org.broadinstitute.dsde.rawls.model.WorkspaceJsonSupport._ import org.broadinstitute.dsde.rawls.model.WorkspaceState.WorkspaceState import org.broadinstitute.dsde.rawls.model.WorkspaceType.WorkspaceType import org.broadinstitute.dsde.rawls.model.WorkspaceVersions.WorkspaceVersion -import org.broadinstitute.dsde.rawls.model.WorkspaceSettingTypes.WorkspaceSettingType import org.broadinstitute.dsde.rawls.model._ import org.broadinstitute.dsde.rawls.monitor.migration.MigrationUtils.Implicits.monadThrowDBIOAction import org.broadinstitute.dsde.rawls.resourcebuffer.ResourceBufferService @@ -48,10 +47,7 @@ import spray.json.DefaultJsonProtocol._ import spray.json._ import org.broadinstitute.dsde.rawls.metrics.MetricsHelper import cats.effect.unsafe.implicits.global -import com.google.cloud.storage.BucketInfo.LifecycleRule -import com.google.cloud.storage.BucketInfo.LifecycleRule.{LifecycleAction, LifecycleCondition} import org.broadinstitute.dsde.rawls.billing.BillingRepository -import org.broadinstitute.dsde.rawls.model.WorkspaceSettingConfig._ import java.io.IOException import java.util.UUID