diff --git a/modules/core/src/main/scala/org/scalasteward/core/application/Context.scala b/modules/core/src/main/scala/org/scalasteward/core/application/Context.scala index da7572addb..90d90fedd5 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/application/Context.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/application/Context.scala @@ -33,7 +33,7 @@ import org.scalasteward.core.repoconfig.RepoConfigAlg import org.scalasteward.core.sbt.SbtAlg import org.scalasteward.core.scalafix.MigrationAlg import org.scalasteward.core.scalafmt.ScalafmtAlg -import org.scalasteward.core.update.{FilterAlg, PruningAlg, UpdateAlg} +import org.scalasteward.core.update.{FilterAlg, PruningAlg, UpdateAlg, VersionsCacheAlg} import org.scalasteward.core.util._ import org.scalasteward.core.vcs.data.AuthenticatedUser import org.scalasteward.core.vcs.{VCSApiAlg, VCSExtraAlg, VCSRepoAlg, VCSSelection} @@ -50,7 +50,7 @@ object Context { implicit0(logger: Logger[F]) <- Resource.liftF(Slf4jLogger.create[F]) implicit0(httpExistenceClient: HttpExistenceClient[F]) <- HttpExistenceClient.create[F] implicit0(user: AuthenticatedUser) <- Resource.liftF(config.vcsUser[F]) - updateAlgRateLimiter <- RateLimiter.create[F] + rateLimiter <- Resource.liftF(RateLimiter.create[F]) } yield { implicit val dateTimeAlg: DateTimeAlg[F] = DateTimeAlg.create[F] implicit val fileAlg: FileAlg[F] = FileAlg.create[F] @@ -61,20 +61,22 @@ object Context { implicit val gitAlg: GitAlg[F] = GitAlg.create[F] implicit val httpJsonClient: HttpJsonClient[F] = new HttpJsonClient[F] implicit val repoCacheRepository: RepoCacheRepository[F] = - new RepoCacheRepository[F](new JsonKeyValueStore("repos", "9")) + new RepoCacheRepository[F](new JsonKeyValueStore("repo_cache", "1")) implicit val selfCheckAlg: SelfCheckAlg[F] = new SelfCheckAlg[F] val vcsSelection = new VCSSelection[F] implicit val vcsApiAlg: VCSApiAlg[F] = vcsSelection.getAlg(config) implicit val vcsRepoAlg: VCSRepoAlg[F] = VCSRepoAlg.create[F](config, gitAlg) implicit val vcsExtraAlg: VCSExtraAlg[F] = VCSExtraAlg.create[F] implicit val pullRequestRepository: PullRequestRepository[F] = - new PullRequestRepository[F](new JsonKeyValueStore("prs", "5")) + new PullRequestRepository[F](new JsonKeyValueStore("pull_requests", "1")) implicit val scalafmtAlg: ScalafmtAlg[F] = ScalafmtAlg.create[F] implicit val coursierAlg: CoursierAlg[F] = CoursierAlg.create - implicit val updateAlg: UpdateAlg[F] = new UpdateAlg[F](updateAlgRateLimiter) + implicit val versionsCacheAlg: VersionsCacheAlg[F] = + new VersionsCacheAlg[F](new JsonKeyValueStore("versions", "1"), rateLimiter) + implicit val updateAlg: UpdateAlg[F] = new UpdateAlg[F] implicit val sbtAlg: SbtAlg[F] = SbtAlg.create[F] implicit val refreshErrorAlg: RefreshErrorAlg[F] = - new RefreshErrorAlg[F](new JsonKeyValueStore("repos_refresh_errors", "1")) + new RefreshErrorAlg[F](new JsonKeyValueStore("refresh_error", "1")) implicit val repoCacheAlg: RepoCacheAlg[F] = new RepoCacheAlg[F] implicit val migrationAlg: MigrationAlg[F] = MigrationAlg.create[F] implicit val editAlg: EditAlg[F] = new EditAlg[F] diff --git a/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala b/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala index 18f58c7029..6832507aa4 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala @@ -33,7 +33,7 @@ import org.scalasteward.core.data.{Dependency, Version} trait CoursierAlg[F[_]] { def getArtifactUrl(dependency: Dependency): F[Option[Uri]] - def getNewerVersions(dependency: Dependency): F[List[Version]] + def getVersions(dependency: Dependency): F[List[Version]] final def getArtifactIdUrlMapping(dependencies: List[Dependency])( implicit F: Applicative[F] @@ -80,13 +80,12 @@ object CoursierAlg { } } - override def getNewerVersions(dependency: Dependency): F[List[Version]] = { + override def getVersions(dependency: Dependency): F[List[Version]] = { val module = toCoursierModule(dependency) - val version = Version(dependency.version) versions .withModule(module) .versions() - .map(_.available.map(Version.apply).filter(_ > version).sorted) + .map(_.available.map(Version.apply).sorted) .handleErrorWith { throwable => logger.error(throwable)(s"Failed to get newer versions of $module").as(List.empty) } diff --git a/modules/core/src/main/scala/org/scalasteward/core/data/Version.scala b/modules/core/src/main/scala/org/scalasteward/core/data/Version.scala index 2e14c1e0f2..5c6e6d69a1 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/data/Version.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/data/Version.scala @@ -19,6 +19,8 @@ package org.scalasteward.core.data import cats.Order import cats.implicits._ import eu.timepit.refined.types.numeric.NonNegInt +import io.circe.Codec +import io.circe.generic.extras.semiauto.deriveUnwrappedCodec import scala.annotation.tailrec final case class Version(value: String) { @@ -90,6 +92,9 @@ final case class Version(value: String) { } object Version { + implicit val versionCodec: Codec[Version] = + deriveUnwrappedCodec + implicit val versionOrder: Order[Version] = Order.from[Version] { (v1, v2) => val (c1, c2) = padToSameLength(v1.alnumComponents, v2.alnumComponents, Component.Empty) diff --git a/modules/core/src/main/scala/org/scalasteward/core/persistence/JsonKeyValueStore.scala b/modules/core/src/main/scala/org/scalasteward/core/persistence/JsonKeyValueStore.scala index 552d40387f..3bf3d9e3d8 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/persistence/JsonKeyValueStore.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/persistence/JsonKeyValueStore.scala @@ -20,43 +20,36 @@ import better.files.File import cats.implicits._ import io.circe.parser.decode import io.circe.syntax._ -import io.circe.{Decoder, Encoder, KeyDecoder, KeyEncoder} +import io.circe.{Decoder, Encoder, KeyEncoder} import org.scalasteward.core.io.{FileAlg, WorkspaceAlg} import org.scalasteward.core.util.MonadThrowable final class JsonKeyValueStore[F[_], K, V](name: String, schemaVersion: String)( implicit fileAlg: FileAlg[F], - workspaceAlg: WorkspaceAlg[F], - F: MonadThrowable[F], - keyDecoder: KeyDecoder[K], keyEncoder: KeyEncoder[K], valueDecoder: Decoder[V], - valueEncoder: Encoder[V] + valueEncoder: Encoder[V], + workspaceAlg: WorkspaceAlg[F], + F: MonadThrowable[F] ) extends KeyValueStore[F, K, V] { override def get(key: K): F[Option[V]] = - read.map(_.get(key)) - - override def modifyF(key: K)(f: Option[V] => F[Option[V]]): F[Option[V]] = - read.flatMap { store => - f(store.get(key)).flatMap { - case res @ Some(updated) => write(store.updated(key, updated)).as(res) - case None => write(store - key).as(None) - } + jsonFile(key).flatMap(fileAlg.readFile).flatMap { + case Some(content) => F.fromEither(decode[Option[V]](content)) + case None => F.pure(Option.empty[V]) } - private val filename = - s"${name}_v${schemaVersion}.json" + override def put(key: K, value: V): F[Unit] = + write(key, Some(value)) - private val jsonFile: F[File] = - workspaceAlg.rootDir.map(_ / filename) + override def modifyF(key: K)(f: Option[V] => F[Option[V]]): F[Option[V]] = + get(key).flatMap(maybeValue => f(maybeValue).flatTap(write(key, _))) - private def read: F[Map[K, V]] = - jsonFile.flatMap(fileAlg.readFile).flatMap { - case Some(content) => F.fromEither(decode[Map[K, V]](content)) - case None => F.pure(Map.empty[K, V]) - } + private def jsonFile(key: K): F[File] = + workspaceAlg.rootDir.map( + _ / "store" / s"${name}_v${schemaVersion}" / keyEncoder(key) / s"$name.json" + ) - private def write(store: Map[K, V]): F[Unit] = - jsonFile.flatMap(fileAlg.writeFile(_, store.asJson.toString)) + private def write(key: K, value: Option[V]): F[Unit] = + jsonFile(key).flatMap(fileAlg.writeFile(_, value.asJson.toString)) } diff --git a/modules/core/src/main/scala/org/scalasteward/core/persistence/KeyValueStore.scala b/modules/core/src/main/scala/org/scalasteward/core/persistence/KeyValueStore.scala index c7f7f6a9be..50c60ce7f2 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/persistence/KeyValueStore.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/persistence/KeyValueStore.scala @@ -22,14 +22,13 @@ import cats.implicits._ trait KeyValueStore[F[_], K, V] { def get(key: K): F[Option[V]] + def put(key: K, value: V): F[Unit] + def modifyF(key: K)(f: Option[V] => F[Option[V]]): F[Option[V]] final def modify(key: K)(f: Option[V] => Option[V])(implicit F: Applicative[F]): F[Option[V]] = modifyF(key)(f.andThen(F.pure)) - final def put(key: K, value: V)(implicit F: Applicative[F]): F[Unit] = - modify(key)(_ => Some(value)).void - final def update(key: K)(f: Option[V] => V)(implicit F: Applicative[F]): F[Unit] = modify(key)(f.andThen(Some.apply)).void } diff --git a/modules/core/src/main/scala/org/scalasteward/core/update/UpdateAlg.scala b/modules/core/src/main/scala/org/scalasteward/core/update/UpdateAlg.scala index 7f54bf863d..a612eb6307 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/update/UpdateAlg.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/update/UpdateAlg.scala @@ -19,34 +19,28 @@ package org.scalasteward.core.update import cats.implicits._ import cats.{Monad, Parallel} import io.chrisdavenport.log4cats.Logger -import org.scalasteward.core.coursier.CoursierAlg import org.scalasteward.core.data._ import org.scalasteward.core.repoconfig.RepoConfig import org.scalasteward.core.util -import org.scalasteward.core.util.{Nel, RateLimiter} +import org.scalasteward.core.util.Nel -final class UpdateAlg[F[_]](rateLimiter: RateLimiter[F])( +final class UpdateAlg[F[_]]( implicit - coursierAlg: CoursierAlg[F], filterAlg: FilterAlg[F], logger: Logger[F], parallel: Parallel[F], + versionsCacheAlg: VersionsCacheAlg[F], F: Monad[F] ) { def findUpdate(dependency: Dependency): F[Option[Update.Single]] = for { - newerVersions0 <- getNewerVersions(dependency) + newerVersions0 <- versionsCacheAlg.getNewerVersions(dependency) maybeUpdate0 = Nel.fromList(newerVersions0).map { newerVersions1 => Update.Single(CrossDependency(dependency), newerVersions1.map(_.value)) } maybeUpdate1 = maybeUpdate0.orElse(UpdateAlg.findUpdateWithNewerGroupId(dependency)) } yield maybeUpdate1 - private def getNewerVersions(dependency: Dependency): F[List[Version]] = { - val key = s" ${dependency.groupId.value}:${dependency.artifactId.crossName}" - rateLimiter.limitUnseen(key)(coursierAlg.getNewerVersions(dependency)) - } - def findUpdates(dependencies: List[Dependency], repoConfig: RepoConfig): F[List[Update.Single]] = for { _ <- logger.info(s"Find updates") diff --git a/modules/core/src/main/scala/org/scalasteward/core/update/VersionsCacheAlg.scala b/modules/core/src/main/scala/org/scalasteward/core/update/VersionsCacheAlg.scala new file mode 100644 index 0000000000..e24a78cf6c --- /dev/null +++ b/modules/core/src/main/scala/org/scalasteward/core/update/VersionsCacheAlg.scala @@ -0,0 +1,82 @@ +/* + * Copyright 2018-2019 Scala Steward contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.scalasteward.core.update + +import cats.Monad +import cats.implicits._ +import io.circe.generic.semiauto.deriveCodec +import io.circe.{Codec, KeyEncoder} +import java.util.concurrent.TimeUnit +import org.scalasteward.core.application.Config +import org.scalasteward.core.coursier.CoursierAlg +import org.scalasteward.core.data.{Dependency, Version} +import org.scalasteward.core.persistence.KeyValueStore +import org.scalasteward.core.update.VersionsCacheAlg.{Entry, Module} +import org.scalasteward.core.util.{DateTimeAlg, RateLimiter} +import scala.concurrent.duration.FiniteDuration + +final class VersionsCacheAlg[F[_]]( + kvStore: KeyValueStore[F, Module, Entry], + rateLimiter: RateLimiter[F] +)( + implicit + config: Config, + coursierAlg: CoursierAlg[F], + dateTimeAlg: DateTimeAlg[F], + F: Monad[F] +) { + def getVersions(dependency: Dependency): F[List[Version]] = { + val module = Module(dependency) + dateTimeAlg.currentTimeMillis.flatMap { now => + kvStore.get(module).flatMap { + case Some(entry) if entry.age(now) <= config.cacheTtl => F.pure(entry.versions.sorted) + case _ => + rateLimiter + .limit(coursierAlg.getVersions(dependency)) + .flatTap(versions => kvStore.put(module, Entry(now, versions))) + } + } + } + + def getNewerVersions(dependency: Dependency): F[List[Version]] = { + val current = Version(dependency.version) + getVersions(dependency).map(_.filter(_ > current)) + } +} + +object VersionsCacheAlg { + final case class Module(dependency: Dependency) + + object Module { + implicit val moduleKeyEncoder: KeyEncoder[Module] = + KeyEncoder.instance { m => + m.dependency.groupId.value + "/" + m.dependency.artifactId.crossName + + m.dependency.scalaVersion.fold("")("_" + _.value) + + m.dependency.sbtVersion.fold("")("_" + _.value) + } + } + + final case class Entry(updatedAt: Long, versions: List[Version]) { + def age(now: Long): FiniteDuration = + FiniteDuration(now - updatedAt, TimeUnit.MILLISECONDS) + } + + object Entry { + implicit val entryCodec: Codec[Entry] = + deriveCodec + } +} diff --git a/modules/core/src/main/scala/org/scalasteward/core/util/RateLimiter.scala b/modules/core/src/main/scala/org/scalasteward/core/util/RateLimiter.scala index 91d7b0fe9a..42f3d33c23 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/util/RateLimiter.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/util/RateLimiter.scala @@ -17,30 +17,20 @@ package org.scalasteward.core.util import cats.effect.concurrent.Semaphore -import cats.effect.{Concurrent, Resource, Timer} +import cats.effect.{Concurrent, Timer} import cats.implicits._ -import com.github.benmanes.caffeine.cache.Caffeine import scala.concurrent.duration._ -import scalacache.CatsEffect.modes._ -import scalacache.Entry -import scalacache.caffeine.CaffeineCache trait RateLimiter[F[_]] { - def limitUnseen[A](key: String)(fa: F[A]): F[A] + def limit[A](fa: F[A]): F[A] } object RateLimiter { - def create[F[_]](implicit timer: Timer[F], F: Concurrent[F]): Resource[F, RateLimiter[F]] = - for { - cache <- Resource.make(F.delay { - CaffeineCache(Caffeine.newBuilder().maximumSize(65536L).build[String, Entry[Unit]]()) - })(_.close().void) - semaphore <- Resource.liftF(Semaphore(1)) - } yield new RateLimiter[F] { - override def limitUnseen[A](key: String)(fa: F[A]): F[A] = - cache.get(key).flatMap { - case Some(_) => fa - case None => semaphore.withPermit(timer.sleep(250.millis) *> fa <* cache.put(key)(())) - } + def create[F[_]](implicit timer: Timer[F], F: Concurrent[F]): F[RateLimiter[F]] = + Semaphore(1).map { semaphore => + new RateLimiter[F] { + override def limit[A](fa: F[A]): F[A] = + semaphore.withPermit(timer.sleep(250.millis) >> fa) + } } } diff --git a/modules/core/src/test/scala/org/scalasteward/core/mock/MockContext.scala b/modules/core/src/test/scala/org/scalasteward/core/mock/MockContext.scala index 054b59bacc..19018f7bf1 100644 --- a/modules/core/src/test/scala/org/scalasteward/core/mock/MockContext.scala +++ b/modules/core/src/test/scala/org/scalasteward/core/mock/MockContext.scala @@ -18,7 +18,7 @@ import org.scalasteward.core.repoconfig.RepoConfigAlg import org.scalasteward.core.sbt.SbtAlg import org.scalasteward.core.scalafix.MigrationAlg import org.scalasteward.core.scalafmt.ScalafmtAlg -import org.scalasteward.core.update.{FilterAlg, PruningAlg, UpdateAlg} +import org.scalasteward.core.update.{FilterAlg, PruningAlg, UpdateAlg, VersionsCacheAlg} import org.scalasteward.core.util.{BracketThrowable, DateTimeAlg, RateLimiter} import org.scalasteward.core.vcs.VCSRepoAlg import org.scalasteward.core.vcs.data.AuthenticatedUser @@ -50,7 +50,7 @@ object MockContext { ) val nopLimiter: RateLimiter[MockEff] = new RateLimiter[MockEff] { - override def limitUnseen[A](key: String)(fa: MockEff[A]): MockEff[A] = fa + override def limit[A](fa: MockEff[A]): MockEff[A] = fa } implicit val mockEffBracketThrowable: BracketThrowable[MockEff] = Sync[MockEff] @@ -69,13 +69,15 @@ object MockContext { implicit val scalafmtAlg: ScalafmtAlg[MockEff] = ScalafmtAlg.create implicit val migrationAlg: MigrationAlg[MockEff] = MigrationAlg.create implicit val cacheRepository: RepoCacheRepository[MockEff] = - new RepoCacheRepository[MockEff](new JsonKeyValueStore("repos", "6")) + new RepoCacheRepository[MockEff](new JsonKeyValueStore("repo_cache", "1")) implicit val filterAlg: FilterAlg[MockEff] = new FilterAlg[MockEff] - implicit val updateAlg: UpdateAlg[MockEff] = new UpdateAlg[MockEff](nopLimiter) + implicit val versionsCacheAlg: VersionsCacheAlg[MockEff] = + new VersionsCacheAlg[MockEff](new JsonKeyValueStore("versions", "1"), nopLimiter) + implicit val updateAlg: UpdateAlg[MockEff] = new UpdateAlg[MockEff] implicit val sbtAlg: SbtAlg[MockEff] = SbtAlg.create implicit val editAlg: EditAlg[MockEff] = new EditAlg[MockEff] implicit val repoConfigAlg: RepoConfigAlg[MockEff] = new RepoConfigAlg[MockEff] implicit val prRepo: PullRequestRepository[MockEff] = - new PullRequestRepository[MockEff](new JsonKeyValueStore("pullrequests", "9")) + new PullRequestRepository[MockEff](new JsonKeyValueStore("pull_requests", "1")) implicit val pruningAlg: PruningAlg[MockEff] = new PruningAlg[MockEff] } diff --git a/modules/core/src/test/scala/org/scalasteward/core/persistence/JsonKeyValueStoreTest.scala b/modules/core/src/test/scala/org/scalasteward/core/persistence/JsonKeyValueStoreTest.scala index 8ad939f005..63dd0d1114 100644 --- a/modules/core/src/test/scala/org/scalasteward/core/persistence/JsonKeyValueStoreTest.scala +++ b/modules/core/src/test/scala/org/scalasteward/core/persistence/JsonKeyValueStoreTest.scala @@ -6,7 +6,7 @@ import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers class JsonKeyValueStoreTest extends AnyFunSuite with Matchers { - test("put, get, getMany, delete") { + test("put, get") { val kvStore = new JsonKeyValueStore[MockEff, String, String]("test", "0") val p = for { _ <- kvStore.put("k1", "v1") @@ -16,22 +16,20 @@ class JsonKeyValueStoreTest extends AnyFunSuite with Matchers { } yield (v1, v3) val (state, value) = p.run(MockState.empty).unsafeRunSync() - val file = config.workspace / "test_v0.json" + val k1File = config.workspace / "store" / "test_v0" / "k1" / "test.json" + val k2File = config.workspace / "store" / "test_v0" / "k2" / "test.json" + val k3File = config.workspace / "store" / "test_v0" / "k3" / "test.json" value shouldBe (Some("v1") -> None) state shouldBe MockState.empty.copy( commands = Vector( - List("read", file.toString), - List("write", file.toString), - List("read", file.toString), - List("read", file.toString), - List("write", file.toString), - List("read", file.toString) + List("write", k1File.toString), + List("read", k1File.toString), + List("write", k2File.toString), + List("read", k3File.toString) ), files = Map( - file -> """|{ - | "k1" : "v1", - | "k2" : "v2" - |}""".stripMargin.trim + k1File -> """"v1"""", + k2File -> """"v2"""" ) ) } diff --git a/modules/core/src/test/scala/org/scalasteward/core/sbt/SbtAlgTest.scala b/modules/core/src/test/scala/org/scalasteward/core/sbt/SbtAlgTest.scala index fd950cb7a3..2b422a11b4 100644 --- a/modules/core/src/test/scala/org/scalasteward/core/sbt/SbtAlgTest.scala +++ b/modules/core/src/test/scala/org/scalasteward/core/sbt/SbtAlgTest.scala @@ -73,7 +73,7 @@ class SbtAlgTest extends AnyFunSuite with Matchers { ) val initialState = MockState.empty.copy(files = files) val state = sbtAlg.getUpdates(repo).runS(initialState).unsafeRunSync() - state shouldBe initialState.copy( + state.copy(files = files) shouldBe initialState.copy( commands = Vector( List( "TEST_VAR=GREAT", @@ -87,7 +87,17 @@ class SbtAlgTest extends AnyFunSuite with Matchers { s";$crossStewardDependencies;$crossStewardUpdates;$reloadPlugins;$stewardDependencies;$stewardUpdates" ), List("read", s"$repoDir/project/build.properties"), - List("read", s"$repoDir/.scalafmt.conf") + List("read", s"$repoDir/.scalafmt.conf"), + List("read", s"${config.workspace}/store/versions_v1/org.scala-sbt/sbt/versions.json"), + List("write", s"${config.workspace}/store/versions_v1/org.scala-sbt/sbt/versions.json"), + List( + "read", + s"${config.workspace}/store/versions_v1/org.scalameta/scalafmt-core_2.13/versions.json" + ), + List( + "write", + s"${config.workspace}/store/versions_v1/org.scalameta/scalafmt-core_2.13/versions.json" + ) ) ) }