From 969d2e79072d3f84a496220c96d61364f184e6c8 Mon Sep 17 00:00:00 2001 From: Frank Thomas Date: Tue, 21 Jan 2025 09:24:37 +0100 Subject: [PATCH] Skip abandoned repos --- .../main/resources/default.scala-steward.conf | 2 + .../scalasteward/core/git/FileGitAlg.scala | 8 +++- .../org/scalasteward/core/git/GenGitAlg.scala | 6 +++ .../core/repocache/RepoCache.scala | 2 + .../core/repocache/RepoCacheAlg.scala | 21 +++++++++- .../core/repoconfig/RepoConfig.scala | 11 +++++- .../scalasteward/core/util/Timestamp.scala | 3 ++ .../org/scalasteward/core/util/dateTime.scala | 7 ++++ .../org/scalasteward/core/TestInstances.scala | 6 ++- .../core/git/FileGitAlgTest.scala | 16 +++++++- .../scalasteward/core/io/processTest.scala | 5 ++- .../core/repocache/RepoCacheAlgTest.scala | 38 +++++++++++++++++-- .../core/update/PruningAlgTest.scala | 5 +++ 13 files changed, 118 insertions(+), 12 deletions(-) diff --git a/modules/core/src/main/resources/default.scala-steward.conf b/modules/core/src/main/resources/default.scala-steward.conf index 263bf980be..5e0ef0c22b 100644 --- a/modules/core/src/main/resources/default.scala-steward.conf +++ b/modules/core/src/main/resources/default.scala-steward.conf @@ -4,6 +4,8 @@ // Changes to this file are therefore immediately visible to all // Scala Steward instances. +lastCommitMaxAge = "540 days" + postUpdateHooks = [ { groupId = "com.github.liancheng", diff --git a/modules/core/src/main/scala/org/scalasteward/core/git/FileGitAlg.scala b/modules/core/src/main/scala/org/scalasteward/core/git/FileGitAlg.scala index 8552d2391f..928afb3c86 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/git/FileGitAlg.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/git/FileGitAlg.scala @@ -25,7 +25,8 @@ import org.scalasteward.core.forge.ForgeType.* import org.scalasteward.core.git.FileGitAlg.{dotdot, gitCmd} import org.scalasteward.core.io.process.{ProcessFailedException, SlurpOptions} import org.scalasteward.core.io.{FileAlg, ProcessAlg, WorkspaceAlg} -import org.scalasteward.core.util.Nel +import org.scalasteward.core.util.{Nel, Timestamp} +import scala.util.Try final class FileGitAlg[F[_]](config: Config)(implicit fileAlg: FileAlg[F], @@ -102,6 +103,11 @@ final class FileGitAlg[F[_]](config: Config)(implicit .handleError(_ => List.empty[String]) .map(_.filter(_.nonEmpty)) + override def getCommitDate(repo: File, sha1: Sha1): F[Timestamp] = + git("show", "--no-patch", "--format=%ct", sha1.value.value)(repo) + .flatMap(out => F.fromTry(Try(out.mkString.trim.toLong))) + .map(Timestamp.fromEpochSecond) + override def hasConflicts(repo: File, branch: Branch, base: Branch): F[Boolean] = { val tryMerge = git_("merge", "--no-commit", "--no-ff", branch.name)(repo) val abortMerge = git_("merge", "--abort")(repo).attempt.void diff --git a/modules/core/src/main/scala/org/scalasteward/core/git/GenGitAlg.scala b/modules/core/src/main/scala/org/scalasteward/core/git/GenGitAlg.scala index c4240622fb..f89130c0d5 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/git/GenGitAlg.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/git/GenGitAlg.scala @@ -22,6 +22,7 @@ import cats.{FlatMap, Monad} import org.http4s.Uri import org.scalasteward.core.application.Config import org.scalasteward.core.io.{FileAlg, ProcessAlg, WorkspaceAlg} +import org.scalasteward.core.util.Timestamp trait GenGitAlg[F[_], Repo] { def add(repo: Repo, file: String): F[Unit] @@ -57,6 +58,8 @@ trait GenGitAlg[F[_], Repo] { def findFilesContaining(repo: Repo, string: String): F[List[String]] + def getCommitDate(repo: Repo, sha1: Sha1): F[Timestamp] + /** Returns `true` if merging `branch` into `base` results in merge conflicts. */ def hasConflicts(repo: Repo, branch: Branch, base: Branch): F[Boolean] @@ -144,6 +147,9 @@ trait GenGitAlg[F[_], Repo] { override def findFilesContaining(repo: A, string: String): F[List[String]] = f(repo).flatMap(self.findFilesContaining(_, string)) + override def getCommitDate(repo: A, sha1: Sha1): F[Timestamp] = + f(repo).flatMap(self.getCommitDate(_, sha1)) + override def hasConflicts(repo: A, branch: Branch, base: Branch): F[Boolean] = f(repo).flatMap(self.hasConflicts(_, branch, base)) diff --git a/modules/core/src/main/scala/org/scalasteward/core/repocache/RepoCache.scala b/modules/core/src/main/scala/org/scalasteward/core/repocache/RepoCache.scala index e1d098db43..c62d8e79e2 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/repocache/RepoCache.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/repocache/RepoCache.scala @@ -22,9 +22,11 @@ import io.circe.generic.semiauto.* import org.scalasteward.core.data.{ArtifactId, DependencyInfo, GroupId, Scope} import org.scalasteward.core.git.Sha1 import org.scalasteward.core.repoconfig.RepoConfig +import org.scalasteward.core.util.Timestamp final case class RepoCache( sha1: Sha1, + commitDate: Timestamp, dependencyInfos: List[Scope[List[DependencyInfo]]], maybeRepoConfig: Option[RepoConfig], maybeRepoConfigParsingError: Option[String] diff --git a/modules/core/src/main/scala/org/scalasteward/core/repocache/RepoCacheAlg.scala b/modules/core/src/main/scala/org/scalasteward/core/repocache/RepoCacheAlg.scala index cf760bce6b..d89bea0c08 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/repocache/RepoCacheAlg.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/repocache/RepoCacheAlg.scala @@ -25,10 +25,13 @@ import org.scalasteward.core.forge.data.RepoOut import org.scalasteward.core.forge.{ForgeApiAlg, ForgeRepoAlg} import org.scalasteward.core.git.GitAlg import org.scalasteward.core.repoconfig.RepoConfigAlg +import org.scalasteward.core.util.{dateTime, DateTimeAlg} import org.typelevel.log4cats.Logger +import scala.util.control.NoStackTrace final class RepoCacheAlg[F[_]](config: Config)(implicit buildToolDispatcher: BuildToolDispatcher[F], + dateTimeAlg: DateTimeAlg[F], forgeApiAlg: ForgeApiAlg[F], forgeRepoAlg: ForgeRepoAlg[F], gitAlg: GitAlg[F], @@ -50,6 +53,7 @@ final class RepoCacheAlg[F[_]](config: Config)(implicit data <- maybeCache .filter(_.sha1 === latestSha1) .fold(cloneAndRefreshCache(repo, repoOut))(supplementCache(repo, _).pure[F]) + _ <- throwIfAbandoned(data) } yield (data, repoOut) private def supplementCache(repo: Repo, cache: RepoCache): RepoData = @@ -68,7 +72,8 @@ final class RepoCacheAlg[F[_]](config: Config)(implicit private def computeCache(repo: Repo): F[RepoData] = for { branch <- gitAlg.currentBranch(repo) - latestSha1 <- gitAlg.latestSha1(repo, branch) + sha1 <- gitAlg.latestSha1(repo, branch) + commitDate <- gitAlg.getCommitDate(repo, sha1) configParsingResult <- repoConfigAlg.readRepoConfig(repo) maybeConfig = configParsingResult.maybeRepoConfig maybeConfigParsingError = configParsingResult.maybeParsingError.map(_.getMessage) @@ -77,9 +82,21 @@ final class RepoCacheAlg[F[_]](config: Config)(implicit dependencyInfos <- dependencies.traverse(_.traverse(_.traverse(gatherDependencyInfo(repo, _)))) _ <- gitAlg.discardChanges(repo) - cache = RepoCache(latestSha1, dependencyInfos, maybeConfig, maybeConfigParsingError) + cache = RepoCache(sha1, commitDate, dependencyInfos, maybeConfig, maybeConfigParsingError) } yield RepoData(repo, cache, config) private def gatherDependencyInfo(repo: Repo, dependency: Dependency): F[DependencyInfo] = gitAlg.findFilesContaining(repo, dependency.version.value).map(DependencyInfo(dependency, _)) + + private[repocache] def throwIfAbandoned(data: RepoData): F[Unit] = + data.config.lastCommitMaxAge.traverse_ { maxAge => + dateTimeAlg.currentTimestamp.flatMap { now => + val sinceLastCommit = data.cache.commitDate.until(now) + val isAbandoned = sinceLastCommit > maxAge + F.raiseWhen(isAbandoned) { + val msg = s"Skipping because last commit is older than ${dateTime.showDuration(maxAge)}" + new Throwable(msg) with NoStackTrace + } + } + } } diff --git a/modules/core/src/main/scala/org/scalasteward/core/repoconfig/RepoConfig.scala b/modules/core/src/main/scala/org/scalasteward/core/repoconfig/RepoConfig.scala index 708d87aa51..1f1166bd5d 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/repoconfig/RepoConfig.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/repoconfig/RepoConfig.scala @@ -25,6 +25,9 @@ import org.scalasteward.core.buildtool.BuildRoot import org.scalasteward.core.data.Repo import org.scalasteward.core.edit.hooks.PostUpdateHook import org.scalasteward.core.repoconfig.RepoConfig.defaultBuildRoots +import org.scalasteward.core.util.dateTime.* +import org.scalasteward.core.util.{combineOptions, intellijThisImportIsUsed} +import scala.concurrent.duration.FiniteDuration final case class RepoConfig( private val commits: Option[CommitsConfig] = None, @@ -37,7 +40,8 @@ final case class RepoConfig( private val assignees: Option[List[String]] = None, private val reviewers: Option[List[String]] = None, private val dependencyOverrides: Option[List[GroupRepoConfig]] = None, - signoffCommits: Option[Boolean] = None + signoffCommits: Option[Boolean] = None, + lastCommitMaxAge: Option[FiniteDuration] = None ) { def commitsOrDefault: CommitsConfig = commits.getOrElse(CommitsConfig()) @@ -107,8 +111,11 @@ object RepoConfig { assignees = x.assignees |+| y.assignees, reviewers = x.reviewers |+| y.reviewers, dependencyOverrides = x.dependencyOverrides |+| y.dependencyOverrides, - signoffCommits = x.signoffCommits.orElse(y.signoffCommits) + signoffCommits = x.signoffCommits.orElse(y.signoffCommits), + lastCommitMaxAge = combineOptions(x.lastCommitMaxAge, y.lastCommitMaxAge)(_ max _) ) } ) + + intellijThisImportIsUsed(finiteDurationEncoder) } diff --git a/modules/core/src/main/scala/org/scalasteward/core/util/Timestamp.scala b/modules/core/src/main/scala/org/scalasteward/core/util/Timestamp.scala index 6e3513c53b..44aac04554 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/util/Timestamp.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/util/Timestamp.scala @@ -34,6 +34,9 @@ final case class Timestamp(millis: Long) { } object Timestamp { + def fromEpochSecond(seconds: Long): Timestamp = + Timestamp(seconds * 1000L) + def fromLocalDateTime(ldt: LocalDateTime): Timestamp = Timestamp(ldt.toInstant(ZoneOffset.UTC).toEpochMilli) diff --git a/modules/core/src/main/scala/org/scalasteward/core/util/dateTime.scala b/modules/core/src/main/scala/org/scalasteward/core/util/dateTime.scala index 88948db260..9fc86f68c4 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/util/dateTime.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/util/dateTime.scala @@ -17,6 +17,7 @@ package org.scalasteward.core.util import cats.syntax.all.* +import io.circe.{Decoder, Encoder} import java.util.concurrent.TimeUnit import scala.annotation.tailrec import scala.concurrent.duration.* @@ -31,6 +32,12 @@ object dateTime { def renderFiniteDuration(fd: FiniteDuration): String = fd.toString.filterNot(_.isSpaceChar) + implicit val finiteDurationDecoder: Decoder[FiniteDuration] = + Decoder[String].emap(parseFiniteDuration(_).leftMap(_.getMessage)) + + implicit val finiteDurationEncoder: Encoder[FiniteDuration] = + Encoder[String].contramap(renderFiniteDuration) + def showDuration(d: FiniteDuration): String = { def symbol(unit: TimeUnit): String = unit match { diff --git a/modules/core/src/test/scala/org/scalasteward/core/TestInstances.scala b/modules/core/src/test/scala/org/scalasteward/core/TestInstances.scala index 0e98d94511..a2fa638f99 100644 --- a/modules/core/src/test/scala/org/scalasteward/core/TestInstances.scala +++ b/modules/core/src/test/scala/org/scalasteward/core/TestInstances.scala @@ -10,6 +10,7 @@ import org.scalasteward.core.git.Sha1 import org.scalasteward.core.repocache.RepoCache import org.scalasteward.core.repoconfig.* import org.scalasteward.core.repoconfig.PullRequestFrequency.{Asap, Timespan} +import org.scalasteward.core.util.{DateTimeAlg, Timestamp} import org.typelevel.log4cats.Logger import org.typelevel.log4cats.slf4j.Slf4jLogger import scala.concurrent.duration.FiniteDuration @@ -19,11 +20,14 @@ object TestInstances { Sha1.unsafeFrom("da39a3ee5e6b4b0d3255bfef95601890afd80709") val dummyRepoCache: RepoCache = - RepoCache(dummySha1, List.empty, Option.empty, Option.empty) + RepoCache(dummySha1, Timestamp(0L), List.empty, Option.empty, Option.empty) val dummyRepoCacheWithParsingError: RepoCache = dummyRepoCache.copy(maybeRepoConfigParsingError = Some("Failed to parse .scala-steward.conf")) + val ioDateTimeAlg: DateTimeAlg[IO] = + DateTimeAlg.create[IO] + implicit val ioLogger: Logger[IO] = Slf4jLogger.getLogger[IO] diff --git a/modules/core/src/test/scala/org/scalasteward/core/git/FileGitAlgTest.scala b/modules/core/src/test/scala/org/scalasteward/core/git/FileGitAlgTest.scala index 1105727787..45cfb40929 100644 --- a/modules/core/src/test/scala/org/scalasteward/core/git/FileGitAlgTest.scala +++ b/modules/core/src/test/scala/org/scalasteward/core/git/FileGitAlgTest.scala @@ -5,7 +5,7 @@ import cats.Monad import cats.effect.IO import cats.syntax.all.* import munit.CatsEffectSuite -import org.scalasteward.core.TestInstances.ioLogger +import org.scalasteward.core.TestInstances.{ioDateTimeAlg, ioLogger} import org.scalasteward.core.git.FileGitAlgTest.{ conflictsNo, conflictsYes, @@ -18,6 +18,7 @@ import org.scalasteward.core.io.ProcessAlgTest.ioProcessAlg import org.scalasteward.core.io.{FileAlg, ProcessAlg, WorkspaceAlg} import org.scalasteward.core.mock.MockConfig.{config, mockRoot} import org.scalasteward.core.util.Nel +import scala.concurrent.duration.DurationInt class FileGitAlgTest extends CatsEffectSuite { private val rootDir = mockRoot / "git-tests" @@ -158,6 +159,19 @@ class FileGitAlgTest extends CatsEffectSuite { } yield () } + test("getCommitDate") { + val repo = rootDir / "getCommitDate" + for { + _ <- ioAuxGitAlg.createRepo(repo) + sha1 <- ioGitAlg.latestSha1(repo, master) + commitDate <- ioGitAlg.getCommitDate(repo, sha1) + now <- ioDateTimeAlg.currentTimestamp + diff = commitDate.until(now) + maxDrift = 2.hours + _ = assert(diff > -maxDrift && diff < maxDrift, clue((commitDate, now))) + } yield () + } + test("hasConflicts") { val repo = rootDir / "hasConflicts" for { diff --git a/modules/core/src/test/scala/org/scalasteward/core/io/processTest.scala b/modules/core/src/test/scala/org/scalasteward/core/io/processTest.scala index ef9d8b008c..34f09420bc 100644 --- a/modules/core/src/test/scala/org/scalasteward/core/io/processTest.scala +++ b/modules/core/src/test/scala/org/scalasteward/core/io/processTest.scala @@ -3,8 +3,9 @@ package org.scalasteward.core.io import cats.effect.IO import cats.effect.unsafe.implicits.global import munit.FunSuite +import org.scalasteward.core.TestInstances.ioDateTimeAlg import org.scalasteward.core.io.process.* -import org.scalasteward.core.util.{DateTimeAlg, Nel} +import org.scalasteward.core.util.Nel import scala.concurrent.duration.* class processTest extends FunSuite { @@ -66,7 +67,7 @@ class processTest extends FunSuite { val timeout = 500.milliseconds val sleep = timeout * 2 val p = slurp2(Nel.of("sleep", sleep.toSeconds.toInt.toString), timeout).attempt - val (Left(t), fd) = DateTimeAlg.create[IO].timed(p).unsafeRunSync(): @unchecked + val (Left(t), fd) = ioDateTimeAlg.timed(p).unsafeRunSync(): @unchecked assert(clue(t).isInstanceOf[ProcessTimedOutException]) assert(clue(fd) > timeout) diff --git a/modules/core/src/test/scala/org/scalasteward/core/repocache/RepoCacheAlgTest.scala b/modules/core/src/test/scala/org/scalasteward/core/repocache/RepoCacheAlgTest.scala index 26f79bbcfd..b800c3eb2f 100644 --- a/modules/core/src/test/scala/org/scalasteward/core/repocache/RepoCacheAlgTest.scala +++ b/modules/core/src/test/scala/org/scalasteward/core/repocache/RepoCacheAlgTest.scala @@ -1,7 +1,8 @@ package org.scalasteward.core.repocache -import cats.implicits.toSemigroupKOps +import cats.syntax.all.* import io.circe.syntax.* +import java.time.LocalDateTime import munit.CatsEffectSuite import org.http4s.HttpApp import org.http4s.circe.* @@ -14,7 +15,9 @@ import org.scalasteward.core.forge.github.Repository import org.scalasteward.core.git.Branch import org.scalasteward.core.mock.MockContext.context.{repoCacheAlg, repoConfigAlg, workspaceAlg} import org.scalasteward.core.mock.{GitHubAuth, MockEff, MockEffOps, MockState} -import org.scalasteward.core.util.intellijThisImportIsUsed +import org.scalasteward.core.repoconfig.RepoConfig +import org.scalasteward.core.util.{intellijThisImportIsUsed, Timestamp} +import scala.concurrent.duration.* class RepoCacheAlgTest extends CatsEffectSuite with Http4sDsl[MockEff] { intellijThisImportIsUsed(encodeUri) @@ -36,7 +39,7 @@ class RepoCacheAlgTest extends CatsEffectSuite with Http4sDsl[MockEff] { uri"https://github.com/scala-steward/cats-effect.git", Branch("main") ) - val repoCache = RepoCache(dummySha1, Nil, None, None) + val repoCache = RepoCache(dummySha1, Timestamp(0L), Nil, None, None) val workspace = workspaceAlg.rootDir.unsafeRunSync() val httpApp = HttpApp[MockEff] { case POST -> Root / "repos" / "typelevel" / "cats-effect" / "forks" => @@ -55,4 +58,33 @@ class RepoCacheAlgTest extends CatsEffectSuite with Http4sDsl[MockEff] { val expected = (RepoData(repo, repoCache, repoConfigAlg.mergeWithGlobal(None)), repoOut) assertIO(obtained, expected) } + + test("throwIfAbandoned: no maxAge") { + val repo = Repo("repo-cache-alg", "test-1") + val cache = RepoCache(dummySha1, Timestamp(0L), Nil, None, None) + val config = RepoConfig.empty + val data = RepoData(repo, cache, config) + val obtained = repoCacheAlg.throwIfAbandoned(data).runA(MockState.empty).attempt + assertIO(obtained, Right(())) + } + + test("throwIfAbandoned: lastCommit < maxAge") { + val repo = Repo("repo-cache-alg", "test-2") + val commitDate = Timestamp.fromLocalDateTime(LocalDateTime.now()) + val cache = RepoCache(dummySha1, commitDate, Nil, None, None) + val config = RepoConfig(lastCommitMaxAge = Some(1.day)) + val data = RepoData(repo, cache, config) + val obtained = repoCacheAlg.throwIfAbandoned(data).runA(MockState.empty).attempt + assertIO(obtained, Right(())) + } + + test("throwIfAbandoned: lastCommit > maxAge") { + val repo = Repo("repo-cache-alg", "test-3") + val cache = RepoCache(dummySha1, Timestamp(0L), Nil, None, None) + val config = RepoConfig(lastCommitMaxAge = Some(1.day)) + val data = RepoData(repo, cache, config) + val obtained = + repoCacheAlg.throwIfAbandoned(data).runA(MockState.empty).attempt.map(_.leftMap(_.getMessage)) + assertIO(obtained, Left("Skipping because last commit is older than 1d")) + } } diff --git a/modules/core/src/test/scala/org/scalasteward/core/update/PruningAlgTest.scala b/modules/core/src/test/scala/org/scalasteward/core/update/PruningAlgTest.scala index 3ac0ad3292..f11806153e 100644 --- a/modules/core/src/test/scala/org/scalasteward/core/update/PruningAlgTest.scala +++ b/modules/core/src/test/scala/org/scalasteward/core/update/PruningAlgTest.scala @@ -21,6 +21,7 @@ class PruningAlgTest extends FunSuite { val Right(repoCache) = decode[RepoCache]( s"""|{ | "sha1": "12def27a837ba6dc9e17406cbbe342fba3527c14", + | "commitDate": 0, | "dependencyInfos": [], | "maybeRepoConfig": { | "pullRequests": { @@ -79,6 +80,7 @@ class PruningAlgTest extends FunSuite { val Right(repoCache) = decode[RepoCache]( s"""|{ | "sha1": "12def27a837ba6dc9e17406cbbe342fba3527c14", + | "commitDate": 0, | "dependencyInfos" : [ | { | "value" : [ @@ -218,6 +220,7 @@ class PruningAlgTest extends FunSuite { val Right(repoCache) = decode[RepoCache]( s"""|{ | "sha1": "12def27a837ba6dc9e17406cbbe342fba3527c14", + | "commitDate": 0, | "dependencyInfos" : [ | { | "value" : [ @@ -330,6 +333,7 @@ class PruningAlgTest extends FunSuite { val Right(repoCache) = decode[RepoCache]( s"""|{ | "sha1": "12def27a837ba6dc9e17406cbbe342fba3527c14", + | "commitDate": 0, | "dependencyInfos" : [ | { | "value" : [ @@ -441,6 +445,7 @@ class PruningAlgTest extends FunSuite { val Right(repoCache) = decode[RepoCache]( s"""|{ | "sha1": "12def27a837ba6dc9e17406cbbe342fba3527c14", + | "commitDate": 0, | "dependencyInfos" : [ | { | "value" : [