Skip to content

Commit

Permalink
[WOR-1690] Update slick, slick-hikaricp from 3.4.1 to 3.5.1 (#2830)
Browse files Browse the repository at this point in the history 
* Update slick, slick-hikaricp from 3.4.1 to 3.5.1

* explicitly get .value from ColumnNames

* cast to correct class and scalafmt

* fix concatSqlActions

* remove stripMargin

* fix typo

---------

Co-authored-by: aherbst-broad <[email protected]>
Co-authored-by: Christina Ahrens Roberts <[email protected]>
Co-authored-by: Marcus Talbott <[email protected]>
  • Loading branch information
@broadbot @aherbst-broad
@cahrens @marctalbott
4 people authored Aug 16, 2024
1 parent bd2e5eb commit b182a1c
Show file tree
Hide file tree
Showing 5 changed files with 80 additions and 73 deletions.
17 changes: 11 additions & 6 deletions core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/DriverComponent.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,21 +109,26 @@ trait RawSqlQuery {

import driver.api._

implicit val GetUUIDResult: GetResult[UUID] = GetResult(r => uuidColumnType.fromBytes(r.nextBytes()))
implicit val GetUUIDResult: GetResult[UUID] =
GetResult(r => uuidColumnType.asInstanceOf[driver.columnTypes.UUIDJdbcType].fromBytes(r.nextBytes()))
implicit val GetUUIDOptionResult: GetResult[Option[UUID]] =
GetResult(r => Option(uuidColumnType.fromBytes(r.nextBytes())))
GetResult(r => Option(uuidColumnType.asInstanceOf[driver.columnTypes.UUIDJdbcType].fromBytes(r.nextBytes())))
implicit object SetUUIDParameter extends SetParameter[UUID] {
def apply(v: UUID, pp: PositionedParameters) { pp.setBytes(uuidColumnType.toBytes(v)) }
def apply(v: UUID, pp: PositionedParameters) {
pp.setBytes(uuidColumnType.asInstanceOf[driver.columnTypes.UUIDJdbcType].toBytes(v))
}
}
implicit object SetUUIDOptionParameter extends SetParameter[Option[UUID]] {
def apply(v: Option[UUID], pp: PositionedParameters) { pp.setBytesOption(v.map(uuidColumnType.toBytes)) }
def apply(v: Option[UUID], pp: PositionedParameters) {
pp.setBytesOption(v.map(uuidColumnType.asInstanceOf[driver.columnTypes.UUIDJdbcType].toBytes))
}
}

def concatSqlActions(builders: SQLActionBuilder*): SQLActionBuilder =
SQLActionBuilder(builders.flatMap(_.queryParts),
SQLActionBuilder(builders.flatMap(_.sql).mkString,
new SetParameter[Unit] {
def apply(p: Unit, pp: PositionedParameters): Unit =
builders.foreach(_.unitPConv.apply(p, pp))
builders.foreach(_.setParameter.apply(p, pp))
}
)

Expand Down
12 changes: 6 additions & 6 deletions .../src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/EntityCacheComponent.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,12 @@ trait EntityCacheComponent {
// C. lastModified is before @param maxModifiedTime, meaning the workspace isn't likely actively being updated
// D. Ordered by lastModified from oldest to newest. Meaning, return the workspace that was modified the longest ago
sql"""SELECT w.id, w.last_modified, c.entity_cache_last_updated
|FROM WORKSPACE w LEFT OUTER JOIN WORKSPACE_ENTITY_CACHE c
| on w.id = c.workspace_id
|where (c.entity_cache_last_updated > $minCacheTime or c.entity_cache_last_updated is null)
| and w.last_modified < $maxModifiedTime
| and (c.entity_cache_last_updated < w.last_modified or c.entity_cache_last_updated is null)
|order by w.last_modified asc limit $numResults""".stripMargin.as[(UUID, Timestamp, Option[Timestamp])]
FROM WORKSPACE w LEFT OUTER JOIN WORKSPACE_ENTITY_CACHE c
on w.id = c.workspace_id
where (c.entity_cache_last_updated > $minCacheTime or c.entity_cache_last_updated is null)
and w.last_modified < $maxModifiedTime
and (c.entity_cache_last_updated < w.last_modified or c.entity_cache_last_updated is null)
order by w.last_modified asc limit $numResults""".as[(UUID, Timestamp, Option[Timestamp])]

// insert if not exist
def updateCacheLastUpdated(workspaceId: UUID,
Expand Down
12 changes: 6 additions & 6 deletions core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/EntityComponent.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -694,12 +694,12 @@ trait EntityComponent {

val countQuery =
sql"""select count(ea.id) from ENTITY doing_reference, ENTITY being_referenced, ENTITY_ATTRIBUTE_#$shardId ea where ea.value_entity_ref = being_referenced.id
|and ea.owner_id = doing_reference.id
|and being_referenced.entity_type=$entityType
|and doing_reference.entity_type!=$entityType
|and ea.deleted = 0
|and doing_reference.deleted = 0
|and being_referenced.workspace_id=${workspaceContext.workspaceIdAsUUID}""".stripMargin
and ea.owner_id = doing_reference.id
and being_referenced.entity_type=$entityType
and doing_reference.entity_type!=$entityType
and ea.deleted = 0
and doing_reference.deleted = 0
and being_referenced.workspace_id=${workspaceContext.workspaceIdAsUUID}"""

countQuery.as[Int]
}
Expand Down
110 changes: 56 additions & 54 deletions .../scala/org/broadinstitute/dsde/rawls/monitor/migration/MultiregionalBucketMigration.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -288,7 +288,7 @@ trait MultiregionalBucketMigrationHistory extends DriverComponent with RawSqlQue
}

final def isMigrating(workspace: Workspace): ReadAction[Boolean] =
sql"select count(*) from #$tableName where #$workspaceIdCol = ${workspace.workspaceIdAsUUID} and #$startedCol is not null and #$finishedCol is null"
sql"select count(*) from #$tableName where #${workspaceIdCol.value} = ${workspace.workspaceIdAsUUID} and #${startedCol.value} is not null and #${finishedCol.value} is null"
.as[Int]
.map(_.head > 0)

Expand Down Expand Up @@ -332,7 +332,7 @@ trait MultiregionalBucketMigrationHistory extends DriverComponent with RawSqlQue
)
}

_ <- sqlu"insert into #$tableName (#$workspaceIdCol) values (${workspace.workspaceIdAsUUID})"
_ <- sqlu"insert into #$tableName (#${workspaceIdCol.value}) values (${workspace.workspaceIdAsUUID})"
id <- sql"select LAST_INSERT_ID()".as[Long].head
} yield id

Expand Down Expand Up @@ -411,11 +411,11 @@ trait MultiregionalBucketMigrationHistory extends DriverComponent with RawSqlQue

final def getMetadata(migrationId: Long): ReadAction[MultiregionalBucketMigrationMetadata] =
sql"""
select b.normalized_id, a.#$createdCol, a.#$startedCol, a.#$updatedCol, a.#$finishedCol, a.#$outcomeCol, a.#$messageCol
select b.normalized_id, a.#${createdCol.value}, a.#${startedCol.value}, a.#${updatedCol.value}, a.#${finishedCol.value}, a.#${outcomeCol.value}, a.#${messageCol.value}
from #$tableName a
join (select count(*) - 1 as normalized_id, max(#$idCol) as last_id from #$tableName group by #$workspaceIdCol) b
on a.#$idCol = b.last_id
where a.#$idCol = $migrationId
join (select count(*) - 1 as normalized_id, max(#${idCol.value}) as last_id from #$tableName group by #${workspaceIdCol.value}) b
on a.#${idCol.value} = b.last_id
where a.#${idCol.value} = $migrationId
"""
.as[MultiregionalBucketMigrationMetadata]
.headOption
Expand All @@ -426,32 +426,34 @@ trait MultiregionalBucketMigrationHistory extends DriverComponent with RawSqlQue
)

final def getMigrationAttempts(workspace: Workspace): ReadAction[List[MultiregionalBucketMigration]] =
sql"select #$allColumns from #$tableName where #$workspaceIdCol = ${workspace.workspaceIdAsUUID} order by #$idCol"
sql"select #$allColumns from #$tableName where #${workspaceIdCol.value} = ${workspace.workspaceIdAsUUID} order by #${idCol.value}"
.as[MultiregionalBucketMigration]
.map(_.toList)

final def selectMigrationsWhere(conditions: SQLActionBuilder): ReadAction[Vector[MultiregionalBucketMigration]] = {
val startingSql = sql"select #$allColumns from #$tableName where #$finishedCol is null and "
concatSqlActions(startingSql, conditions, sql" order by #$updatedCol asc limit 1")
val startingSql = sql"select #$allColumns from #$tableName where #${finishedCol.value} is null and "
concatSqlActions(startingSql, conditions, sql" order by #${updatedCol.value} asc limit 1")
.as[MultiregionalBucketMigration]
}

final def getAttempt(workspaceUuid: UUID) = OptionT[ReadWriteAction, MultiregionalBucketMigration] {
sql"select #$allColumns from #$tableName where #$workspaceIdCol = $workspaceUuid order by #$idCol desc limit 1"
sql"select #$allColumns from #$tableName where #${workspaceIdCol.value} = $workspaceUuid order by #${idCol.value} desc limit 1"
.as[MultiregionalBucketMigration]
.headOption
}

final def getAttempt(migrationId: Long) = OptionT[ReadWriteAction, MultiregionalBucketMigration] {
sql"select #$allColumns from #$tableName where #$idCol = $migrationId".as[MultiregionalBucketMigration].headOption
sql"select #$allColumns from #$tableName where #${idCol.value} = $migrationId"
.as[MultiregionalBucketMigration]
.headOption
}

// Resource-limited migrations are those requiring new google projects and storage transfer jobs
final def getNumActiveResourceLimitedMigrations: ReadWriteAction[Int] =
sql"""
select count(*) from #$tableName m
join (select id, namespace from WORKSPACE) as w on (w.id = m.#$workspaceIdCol)
where m.#$startedCol is not null and m.#$finishedCol is null
join (select id, namespace from WORKSPACE) as w on (w.id = m.#${workspaceIdCol.value})
where m.#${startedCol.value} is not null and m.#${finishedCol.value} is null
""".as[Int].head

// The following query uses raw parameters. In this particular case it's safe to do as the
Expand All @@ -460,59 +462,59 @@ trait MultiregionalBucketMigrationHistory extends DriverComponent with RawSqlQue
final def nextMigration() = OptionT[ReadWriteAction, (Long, UUID, String)] {
concatSqlActions(
sql"""
select m.#$idCol, m.#$workspaceIdCol, CONCAT_WS("/", w.namespace, w.name) from #$tableName m
join (select id, namespace, name, is_locked from WORKSPACE) as w on (w.id = m.#$workspaceIdCol)
where m.#$startedCol is null
select m.#${idCol.value}, m.#${workspaceIdCol.value}, CONCAT_WS("/", w.namespace, w.name) from #$tableName m
join (select id, namespace, name, is_locked from WORKSPACE) as w on (w.id = m.#${workspaceIdCol.value})
where m.#${startedCol.value} is null
/* exclude workspaces with active submissions */
and not exists (
select workspace_id, status from SUBMISSION
where status in #${SubmissionStatuses.activeStatuses.mkString("('", "','", "')")}
and workspace_id = m.workspace_id
)
""",
sql"order by m.#$idCol limit 1"
sql"order by m.#${idCol.value} limit 1"
).as[(Long, UUID, String)].headOption
}

def getWorkspaceName(migrationId: Long) = OptionT[ReadWriteAction, WorkspaceName] {
sql"""
select w.namespace, w.name from WORKSPACE w
where w.id in (select #$workspaceIdCol from #$tableName m where m.id = $migrationId)
where w.id in (select #${workspaceIdCol.value} from #$tableName m where m.id = $migrationId)
""".as[(String, String)].headOption.map(_.map(WorkspaceName.tupled))
}

val removeWorkspaceBucketIamCondition = selectMigrationsWhere(
sql"#$startedCol is not null and #$workspaceBucketIamRemovedCol is null"
sql"#${startedCol.value} is not null and #${workspaceBucketIamRemovedCol.value} is null"
)
val createTempBucketConditionCondition = selectMigrationsWhere(
sql"#$workspaceBucketIamRemovedCol is not null and #$tmpBucketCreatedCol is null"
sql"#${workspaceBucketIamRemovedCol.value} is not null and #${tmpBucketCreatedCol.value} is null"
)
val configureWorkspaceBucketTransferIam = selectMigrationsWhere(
sql"#$tmpBucketCreatedCol is not null and #$workspaceBucketTransferIamConfiguredCol is null"
sql"#${tmpBucketCreatedCol.value} is not null and #${workspaceBucketTransferIamConfiguredCol.value} is null"
)
val issueTransferJobToTmpBucketCondition = selectMigrationsWhere(
sql"#$workspaceBucketTransferIamConfiguredCol is not null and #$workspaceBucketTransferJobIssuedCol is null"
sql"#${workspaceBucketTransferIamConfiguredCol.value} is not null and #${workspaceBucketTransferJobIssuedCol.value} is null"
)
val deleteWorkspaceBucketCondition = selectMigrationsWhere(
sql"#$workspaceBucketTransferredCol is not null and #$workspaceBucketDeletedCol is null"
sql"#${workspaceBucketTransferredCol.value} is not null and #${workspaceBucketDeletedCol.value} is null"
)
val createFinalWorkspaceBucketCondition = selectMigrationsWhere(
sql"#$workspaceBucketDeletedCol is not null and #$finalBucketCreatedCol is null"
sql"#${workspaceBucketDeletedCol.value} is not null and #${finalBucketCreatedCol.value} is null"
)
val configureTmpBucketTransferIam = selectMigrationsWhere(
sql"#$finalBucketCreatedCol is not null and #$tmpBucketTransferIamConfiguredCol is null"
sql"#${finalBucketCreatedCol.value} is not null and #${tmpBucketTransferIamConfiguredCol.value} is null"
)
val issueTransferJobToFinalWorkspaceBucketCondition = selectMigrationsWhere(
sql"#$tmpBucketTransferIamConfiguredCol is not null and #$tmpBucketTransferJobIssuedCol is null"
sql"#${tmpBucketTransferIamConfiguredCol.value} is not null and #${tmpBucketTransferJobIssuedCol.value} is null"
)
val deleteTemporaryBucketCondition = selectMigrationsWhere(
sql"#$tmpBucketTransferredCol is not null and #$tmpBucketDeletedCol is null"
sql"#${tmpBucketTransferredCol.value} is not null and #${tmpBucketDeletedCol.value} is null"
)
val restoreIamPoliciesCondition = selectMigrationsWhere(
sql"#$tmpBucketDeletedCol is not null"
sql"#${tmpBucketDeletedCol.value} is not null"
)

def withMigrationId(migrationId: Long) = selectMigrationsWhere(sql"#$idCol = $migrationId")
def withMigrationId(migrationId: Long) = selectMigrationsWhere(sql"#${idCol.value} = $migrationId")
}

object multiregionalBucketMigrationRetryQuery
Expand Down Expand Up @@ -552,7 +554,7 @@ trait MultiregionalBucketMigrationHistory extends DriverComponent with RawSqlQue

def notExceededMaxRetries = (migration: String) => sql"""not exists (
select null from #$tableName
where #$migrationIdCol = #$migration.id and #$retriesCol >= $maxRetries
where #${migrationIdCol.value} = #$migration.id and #${retriesCol.value} >= $maxRetries
)"""

reduceSqlActionsWithDelim(
Expand All @@ -576,8 +578,8 @@ trait MultiregionalBucketMigrationHistory extends DriverComponent with RawSqlQue
}

final def getOrCreate(migrationId: Long): ReadWriteAction[MultiregionalBucketMigrationRetry] =
sqlu"insert ignore into #$tableName (#$migrationIdCol) values ($migrationId)" >>
sql"select #$allColumns from #$tableName where #$migrationIdCol = $migrationId"
sqlu"insert ignore into #$tableName (#${migrationIdCol.value}) values ($migrationId)" >>
sql"select #$allColumns from #$tableName where #${migrationIdCol.value} = $migrationId"
.as[MultiregionalBucketMigrationRetry]
.head
}
Expand All @@ -589,7 +591,7 @@ trait MultiregionalBucketMigrationHistory extends DriverComponent with RawSqlQue
def update[A](key: PrimaryKey, columnName: ColumnName[A], value: A)(implicit
setA: SetParameter[A]
): ReadWriteAction[Int] =
sqlu"update #$tableName set #$columnName = $value where #$primaryKey = $key"
sqlu"update #$tableName set #${columnName.value} = $value where #${primaryKey.value} = $key"

def update2[A, B](key: PrimaryKey, columnName1: ColumnName[A], value1: A, columnName2: ColumnName[B], value2: B)(
implicit
Expand All @@ -598,8 +600,8 @@ trait MultiregionalBucketMigrationHistory extends DriverComponent with RawSqlQue
): ReadWriteAction[Int] =
sqlu"""
update #$tableName
set #$columnName1 = $value1, #$columnName2 = $value2
where #$primaryKey = $key
set #${columnName1.value} = $value1, #${columnName2.value} = $value2
where #${primaryKey.value} = $key
"""

def update3[A, B, C](key: PrimaryKey,
Expand All @@ -616,8 +618,8 @@ trait MultiregionalBucketMigrationHistory extends DriverComponent with RawSqlQue
): ReadWriteAction[Int] =
sqlu"""
update #$tableName
set #$columnName1 = $value1, #$columnName2 = $value2, #$columnName3 = $value3
where #$primaryKey = $key
set #${columnName1.value} = $value1, #${columnName2.value} = $value2, #${columnName3.value} = $value3
where #${primaryKey.value} = $key
"""

def update5[A, B, C, D, E](key: PrimaryKey,
Expand All @@ -640,12 +642,12 @@ trait MultiregionalBucketMigrationHistory extends DriverComponent with RawSqlQue
): ReadWriteAction[Int] =
sqlu"""
update #$tableName
set #$columnName1 = $value1,
#$columnName2 = $value2,
#$columnName3 = $value3,
#$columnName4 = $value4,
#$columnName5 = $value5
where #$primaryKey = $key
set #${columnName1.value} = $value1,
#${columnName2.value} = $value2,
#${columnName3.value} = $value3,
#${columnName4.value} = $value4,
#${columnName5.value} = $value5
where #${primaryKey.value} = $key
"""

def update10[A, B, C, D, E, F, G, H, I, J](key: PrimaryKey,
Expand Down Expand Up @@ -683,17 +685,17 @@ trait MultiregionalBucketMigrationHistory extends DriverComponent with RawSqlQue
): ReadWriteAction[Int] =
sqlu"""
update #$tableName
set #$columnName1 = $value1,
#$columnName2 = $value2,
#$columnName3 = $value3,
#$columnName4 = $value4,
#$columnName5 = $value5,
#$columnName6 = $value6,
#$columnName7 = $value7,
#$columnName8 = $value8,
#$columnName9 = $value9,
#$columnName10 = $value10
where #$primaryKey = $key
set #${columnName1.value} = $value1,
#${columnName2.value} = $value2,
#${columnName3.value} = $value3,
#${columnName4.value} = $value4,
#${columnName5.value} = $value5,
#${columnName6.value} = $value6,
#${columnName7.value} = $value7,
#${columnName8.value} = $value8,
#${columnName9.value} = $value9,
#${columnName10.value} = $value10
where #${primaryKey.value} = $key
"""

def delete: WriteAction[Int] =
Expand Down
2 changes: 1 addition & 1 deletion project/Dependencies.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import sbt._
object Dependencies {
val akkaV = "2.6.20"
val akkaHttpV = "10.2.10"
val slickV = "3.4.1"
val slickV = "3.5.1"

val googleV = "2.0.0"

Expand Down

0 comments on commit b182a1c

Please sign in to comment.