Add our own versions cache

modules/core/src/main/scala/org/scalasteward/core/application/Context.scala

@@ -33,7 +33,7 @@ import org.scalasteward.core.repoconfig.RepoConfigAlg
 import org.scalasteward.core.sbt.SbtAlg
 import org.scalasteward.core.scalafix.MigrationAlg
 import org.scalasteward.core.scalafmt.ScalafmtAlg
-import org.scalasteward.core.update.{FilterAlg, PruningAlg, UpdateAlg}
+import org.scalasteward.core.update.{FilterAlg, PruningAlg, UpdateAlg, VersionsCacheAlg}
 import org.scalasteward.core.util._
 import org.scalasteward.core.vcs.data.AuthenticatedUser
 import org.scalasteward.core.vcs.{VCSApiAlg, VCSExtraAlg, VCSRepoAlg, VCSSelection}
@@ -50,7 +50,7 @@ object Context {
       implicit0(logger: Logger[F]) <- Resource.liftF(Slf4jLogger.create[F])
       implicit0(httpExistenceClient: HttpExistenceClient[F]) <- HttpExistenceClient.create[F]
       implicit0(user: AuthenticatedUser) <- Resource.liftF(config.vcsUser[F])
-      updateAlgRateLimiter <- RateLimiter.create[F]
+      rateLimiter <- Resource.liftF(RateLimiter.create[F])
     } yield {
       implicit val dateTimeAlg: DateTimeAlg[F] = DateTimeAlg.create[F]
       implicit val fileAlg: FileAlg[F] = FileAlg.create[F]
@@ -61,20 +61,22 @@ object Context {
       implicit val gitAlg: GitAlg[F] = GitAlg.create[F]
       implicit val httpJsonClient: HttpJsonClient[F] = new HttpJsonClient[F]
       implicit val repoCacheRepository: RepoCacheRepository[F] =
-        new RepoCacheRepository[F](new JsonKeyValueStore("repos", "9"))
+        new RepoCacheRepository[F](new JsonKeyValueStore("repo_cache", "1"))
       implicit val selfCheckAlg: SelfCheckAlg[F] = new SelfCheckAlg[F]
       val vcsSelection = new VCSSelection[F]
       implicit val vcsApiAlg: VCSApiAlg[F] = vcsSelection.getAlg(config)
       implicit val vcsRepoAlg: VCSRepoAlg[F] = VCSRepoAlg.create[F](config, gitAlg)
       implicit val vcsExtraAlg: VCSExtraAlg[F] = VCSExtraAlg.create[F]
       implicit val pullRequestRepository: PullRequestRepository[F] =
-        new PullRequestRepository[F](new JsonKeyValueStore("prs", "5"))
+        new PullRequestRepository[F](new JsonKeyValueStore("pull_requests", "1"))
       implicit val scalafmtAlg: ScalafmtAlg[F] = ScalafmtAlg.create[F]
       implicit val coursierAlg: CoursierAlg[F] = CoursierAlg.create
-      implicit val updateAlg: UpdateAlg[F] = new UpdateAlg[F](updateAlgRateLimiter)
+      implicit val versionsCacheAlg: VersionsCacheAlg[F] =
+        new VersionsCacheAlg[F](new JsonKeyValueStore("versions", "1"), rateLimiter)
+      implicit val updateAlg: UpdateAlg[F] = new UpdateAlg[F]
       implicit val sbtAlg: SbtAlg[F] = SbtAlg.create[F]
       implicit val refreshErrorAlg: RefreshErrorAlg[F] =
-        new RefreshErrorAlg[F](new JsonKeyValueStore("repos_refresh_errors", "1"))
+        new RefreshErrorAlg[F](new JsonKeyValueStore("refresh_error", "1"))
       implicit val repoCacheAlg: RepoCacheAlg[F] = new RepoCacheAlg[F]
       implicit val migrationAlg: MigrationAlg[F] = MigrationAlg.create[F]
       implicit val editAlg: EditAlg[F] = new EditAlg[F]

modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala

@@ -33,7 +33,7 @@ import org.scalasteward.core.data.{Dependency, Version}
 trait CoursierAlg[F[_]] {
   def getArtifactUrl(dependency: Dependency): F[Option[Uri]]
 
-  def getNewerVersions(dependency: Dependency): F[List[Version]]
+  def getVersions(dependency: Dependency): F[List[Version]]
 
   final def getArtifactIdUrlMapping(dependencies: List[Dependency])(
       implicit F: Applicative[F]
@@ -80,13 +80,12 @@ object CoursierAlg {
         }
       }
 
-      override def getNewerVersions(dependency: Dependency): F[List[Version]] = {
+      override def getVersions(dependency: Dependency): F[List[Version]] = {
         val module = toCoursierModule(dependency)
-        val version = Version(dependency.version)
         versions
           .withModule(module)
           .versions()
-          .map(_.available.map(Version.apply).filter(_ > version).sorted)
+          .map(_.available.map(Version.apply).sorted)
           .handleErrorWith { throwable =>
             logger.error(throwable)(s"Failed to get newer versions of $module").as(List.empty)
           }

modules/core/src/main/scala/org/scalasteward/core/data/Version.scala

@@ -19,6 +19,8 @@ package org.scalasteward.core.data
 import cats.Order
 import cats.implicits._
 import eu.timepit.refined.types.numeric.NonNegInt
+import io.circe.Codec
+import io.circe.generic.extras.semiauto.deriveUnwrappedCodec
 import scala.annotation.tailrec
 
 final case class Version(value: String) {
@@ -90,6 +92,9 @@ final case class Version(value: String) {
 }
 
 object Version {
+  implicit val versionCodec: Codec[Version] =
+    deriveUnwrappedCodec
+
   implicit val versionOrder: Order[Version] =
     Order.from[Version] { (v1, v2) =>
       val (c1, c2) = padToSameLength(v1.alnumComponents, v2.alnumComponents, Component.Empty)

modules/core/src/main/scala/org/scalasteward/core/persistence/JsonKeyValueStore.scala

@@ -20,43 +20,36 @@ import better.files.File
 import cats.implicits._
 import io.circe.parser.decode
 import io.circe.syntax._
-import io.circe.{Decoder, Encoder, KeyDecoder, KeyEncoder}
+import io.circe.{Decoder, Encoder, KeyEncoder}
 import org.scalasteward.core.io.{FileAlg, WorkspaceAlg}
 import org.scalasteward.core.util.MonadThrowable
 
 final class JsonKeyValueStore[F[_], K, V](name: String, schemaVersion: String)(
     implicit
     fileAlg: FileAlg[F],
-    workspaceAlg: WorkspaceAlg[F],
-    F: MonadThrowable[F],
-    keyDecoder: KeyDecoder[K],
     keyEncoder: KeyEncoder[K],
     valueDecoder: Decoder[V],
-    valueEncoder: Encoder[V]
+    valueEncoder: Encoder[V],
+    workspaceAlg: WorkspaceAlg[F],
+    F: MonadThrowable[F]
 ) extends KeyValueStore[F, K, V] {
   override def get(key: K): F[Option[V]] =
-    read.map(_.get(key))
-
-  override def modifyF(key: K)(f: Option[V] => F[Option[V]]): F[Option[V]] =
-    read.flatMap { store =>
-      f(store.get(key)).flatMap {
-        case res @ Some(updated) => write(store.updated(key, updated)).as(res)
-        case None                => write(store - key).as(None)
-      }
+    jsonFile(key).flatMap(fileAlg.readFile).flatMap {
+      case Some(content) => F.fromEither(decode[Option[V]](content))
+      case None          => F.pure(Option.empty[V])
     }
 
-  private val filename =
-    s"${name}_v${schemaVersion}.json"
+  override def put(key: K, value: V): F[Unit] =
+    write(key, Some(value))
 
-  private val jsonFile: F[File] =
-    workspaceAlg.rootDir.map(_ / filename)
+  override def modifyF(key: K)(f: Option[V] => F[Option[V]]): F[Option[V]] =
+    get(key).flatMap(maybeValue => f(maybeValue).flatTap(write(key, _)))
 
-  private def read: F[Map[K, V]] =
-    jsonFile.flatMap(fileAlg.readFile).flatMap {
-      case Some(content) => F.fromEither(decode[Map[K, V]](content))
-      case None          => F.pure(Map.empty[K, V])
-    }
+  private def jsonFile(key: K): F[File] =
+    workspaceAlg.rootDir.map(
+      _ / "store" / s"${name}_v${schemaVersion}" / keyEncoder(key) / s"$name.json"
+    )
 
-  private def write(store: Map[K, V]): F[Unit] =
-    jsonFile.flatMap(fileAlg.writeFile(_, store.asJson.toString))
+  private def write(key: K, value: Option[V]): F[Unit] =
+    jsonFile(key).flatMap(fileAlg.writeFile(_, value.asJson.toString))
 }

modules/core/src/main/scala/org/scalasteward/core/persistence/KeyValueStore.scala

@@ -22,14 +22,13 @@ import cats.implicits._
 trait KeyValueStore[F[_], K, V] {
   def get(key: K): F[Option[V]]
 
+  def put(key: K, value: V): F[Unit]
+
   def modifyF(key: K)(f: Option[V] => F[Option[V]]): F[Option[V]]
 
   final def modify(key: K)(f: Option[V] => Option[V])(implicit F: Applicative[F]): F[Option[V]] =
     modifyF(key)(f.andThen(F.pure))
 
-  final def put(key: K, value: V)(implicit F: Applicative[F]): F[Unit] =
-    modify(key)(_ => Some(value)).void
-
   final def update(key: K)(f: Option[V] => V)(implicit F: Applicative[F]): F[Unit] =
     modify(key)(f.andThen(Some.apply)).void
 }

modules/core/src/main/scala/org/scalasteward/core/update/UpdateAlg.scala

@@ -19,34 +19,28 @@ package org.scalasteward.core.update
 import cats.implicits._
 import cats.{Monad, Parallel}
 import io.chrisdavenport.log4cats.Logger
-import org.scalasteward.core.coursier.CoursierAlg
 import org.scalasteward.core.data._
 import org.scalasteward.core.repoconfig.RepoConfig
 import org.scalasteward.core.util
-import org.scalasteward.core.util.{Nel, RateLimiter}
+import org.scalasteward.core.util.Nel
 
-final class UpdateAlg[F[_]](rateLimiter: RateLimiter[F])(
+final class UpdateAlg[F[_]](
     implicit
-    coursierAlg: CoursierAlg[F],
     filterAlg: FilterAlg[F],
     logger: Logger[F],
     parallel: Parallel[F],
+    versionsCacheAlg: VersionsCacheAlg[F],
     F: Monad[F]
 ) {
   def findUpdate(dependency: Dependency): F[Option[Update.Single]] =
     for {
-      newerVersions0 <- getNewerVersions(dependency)
+      newerVersions0 <- versionsCacheAlg.getNewerVersions(dependency)
       maybeUpdate0 = Nel.fromList(newerVersions0).map { newerVersions1 =>
         Update.Single(CrossDependency(dependency), newerVersions1.map(_.value))
       }
       maybeUpdate1 = maybeUpdate0.orElse(UpdateAlg.findUpdateWithNewerGroupId(dependency))
     } yield maybeUpdate1
 
-  private def getNewerVersions(dependency: Dependency): F[List[Version]] = {
-    val key = s" ${dependency.groupId.value}:${dependency.artifactId.crossName}"
-    rateLimiter.limitUnseen(key)(coursierAlg.getNewerVersions(dependency))
-  }
-
   def findUpdates(dependencies: List[Dependency], repoConfig: RepoConfig): F[List[Update.Single]] =
     for {
       _ <- logger.info(s"Find updates")

modules/core/src/main/scala/org/scalasteward/core/update/VersionsCacheAlg.scala

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2018-2019 Scala Steward contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.scalasteward.core.update
+
+import cats.Monad
+import cats.implicits._
+import io.circe.generic.semiauto.deriveCodec
+import io.circe.{Codec, KeyEncoder}
+import java.util.concurrent.TimeUnit
+import org.scalasteward.core.application.Config
+import org.scalasteward.core.coursier.CoursierAlg
+import org.scalasteward.core.data.{Dependency, Version}
+import org.scalasteward.core.persistence.KeyValueStore
+import org.scalasteward.core.update.VersionsCacheAlg.{Entry, Module}
+import org.scalasteward.core.util.{DateTimeAlg, RateLimiter}
+import scala.concurrent.duration.FiniteDuration
+
+final class VersionsCacheAlg[F[_]](
+    kvStore: KeyValueStore[F, Module, Entry],
+    rateLimiter: RateLimiter[F]
+)(
+    implicit
+    config: Config,
+    coursierAlg: CoursierAlg[F],
+    dateTimeAlg: DateTimeAlg[F],
+    F: Monad[F]
+) {
+  def getVersions(dependency: Dependency): F[List[Version]] = {
+    val module = Module(dependency)
+    dateTimeAlg.currentTimeMillis.flatMap { now =>
+      kvStore.get(module).flatMap {
+        case Some(entry) if entry.age(now) <= config.cacheTtl => F.pure(entry.versions.sorted)
+        case _ =>
+          rateLimiter
+            .limit(coursierAlg.getVersions(dependency))
+            .flatTap(versions => kvStore.put(module, Entry(now, versions)))
+      }
+    }
+  }
+
+  def getNewerVersions(dependency: Dependency): F[List[Version]] = {
+    val current = Version(dependency.version)
+    getVersions(dependency).map(_.filter(_ > current))
+  }
+}
+
+object VersionsCacheAlg {
+  final case class Module(dependency: Dependency)
+
+  object Module {
+    implicit val moduleKeyEncoder: KeyEncoder[Module] =
+      KeyEncoder.instance { m =>
+        m.dependency.groupId.value + "/" + m.dependency.artifactId.crossName +
+          m.dependency.scalaVersion.fold("")("_" + _.value) +
+          m.dependency.sbtVersion.fold("")("_" + _.value)
+      }
+  }
+
+  final case class Entry(updatedAt: Long, versions: List[Version]) {
+    def age(now: Long): FiniteDuration =
+      FiniteDuration(now - updatedAt, TimeUnit.MILLISECONDS)
+  }
+
+  object Entry {
+    implicit val entryCodec: Codec[Entry] =
+      deriveCodec
+  }
+}

modules/core/src/main/scala/org/scalasteward/core/util/RateLimiter.scala

@@ -17,30 +17,20 @@
 package org.scalasteward.core.util
 
 import cats.effect.concurrent.Semaphore
-import cats.effect.{Concurrent, Resource, Timer}
+import cats.effect.{Concurrent, Timer}
 import cats.implicits._
-import com.github.benmanes.caffeine.cache.Caffeine
 import scala.concurrent.duration._
-import scalacache.CatsEffect.modes._
-import scalacache.Entry
-import scalacache.caffeine.CaffeineCache
 
 trait RateLimiter[F[_]] {
-  def limitUnseen[A](key: String)(fa: F[A]): F[A]
+  def limit[A](fa: F[A]): F[A]
 }
 
 object RateLimiter {
-  def create[F[_]](implicit timer: Timer[F], F: Concurrent[F]): Resource[F, RateLimiter[F]] =
-    for {
-      cache <- Resource.make(F.delay {
-        CaffeineCache(Caffeine.newBuilder().maximumSize(65536L).build[String, Entry[Unit]]())
-      })(_.close().void)
-      semaphore <- Resource.liftF(Semaphore(1))
-    } yield new RateLimiter[F] {
-      override def limitUnseen[A](key: String)(fa: F[A]): F[A] =
-        cache.get(key).flatMap {
-          case Some(_) => fa
-          case None    => semaphore.withPermit(timer.sleep(250.millis) *> fa <* cache.put(key)(()))
-        }
+  def create[F[_]](implicit timer: Timer[F], F: Concurrent[F]): F[RateLimiter[F]] =
+    Semaphore(1).map { semaphore =>
+      new RateLimiter[F] {
+        override def limit[A](fa: F[A]): F[A] =
+          semaphore.withPermit(timer.sleep(250.millis) >> fa)
+      }
     }
 }

modules/core/src/test/scala/org/scalasteward/core/mock/MockContext.scala

@@ -18,7 +18,7 @@ import org.scalasteward.core.repoconfig.RepoConfigAlg
 import org.scalasteward.core.sbt.SbtAlg
 import org.scalasteward.core.scalafix.MigrationAlg
 import org.scalasteward.core.scalafmt.ScalafmtAlg
-import org.scalasteward.core.update.{FilterAlg, PruningAlg, UpdateAlg}
+import org.scalasteward.core.update.{FilterAlg, PruningAlg, UpdateAlg, VersionsCacheAlg}
 import org.scalasteward.core.util.{BracketThrowable, DateTimeAlg, RateLimiter}
 import org.scalasteward.core.vcs.VCSRepoAlg
 import org.scalasteward.core.vcs.data.AuthenticatedUser
@@ -50,7 +50,7 @@ object MockContext {
   )
 
   val nopLimiter: RateLimiter[MockEff] = new RateLimiter[MockEff] {
-    override def limitUnseen[A](key: String)(fa: MockEff[A]): MockEff[A] = fa
+    override def limit[A](fa: MockEff[A]): MockEff[A] = fa
   }
 
   implicit val mockEffBracketThrowable: BracketThrowable[MockEff] = Sync[MockEff]
@@ -69,13 +69,15 @@ object MockContext {
   implicit val scalafmtAlg: ScalafmtAlg[MockEff] = ScalafmtAlg.create
   implicit val migrationAlg: MigrationAlg[MockEff] = MigrationAlg.create
   implicit val cacheRepository: RepoCacheRepository[MockEff] =
-    new RepoCacheRepository[MockEff](new JsonKeyValueStore("repos", "6"))
+    new RepoCacheRepository[MockEff](new JsonKeyValueStore("repo_cache", "1"))
   implicit val filterAlg: FilterAlg[MockEff] = new FilterAlg[MockEff]
-  implicit val updateAlg: UpdateAlg[MockEff] = new UpdateAlg[MockEff](nopLimiter)
+  implicit val versionsCacheAlg: VersionsCacheAlg[MockEff] =
+    new VersionsCacheAlg[MockEff](new JsonKeyValueStore("versions", "1"), nopLimiter)
+  implicit val updateAlg: UpdateAlg[MockEff] = new UpdateAlg[MockEff]
   implicit val sbtAlg: SbtAlg[MockEff] = SbtAlg.create
   implicit val editAlg: EditAlg[MockEff] = new EditAlg[MockEff]
   implicit val repoConfigAlg: RepoConfigAlg[MockEff] = new RepoConfigAlg[MockEff]
   implicit val prRepo: PullRequestRepository[MockEff] =
-    new PullRequestRepository[MockEff](new JsonKeyValueStore("pullrequests", "9"))
+    new PullRequestRepository[MockEff](new JsonKeyValueStore("pull_requests", "1"))
   implicit val pruningAlg: PruningAlg[MockEff] = new PruningAlg[MockEff]
 }