Skip to content

Commit

Permalink
Update metrics scheduler
Browse files Browse the repository at this point in the history
  • Loading branch information
adpi2 committed Dec 5, 2022
1 parent 3a14b5b commit 5de2dc7
Show file tree
Hide file tree
Showing 6 changed files with 48 additions and 73 deletions.
7 changes: 7 additions & 0 deletions modules/infra/src/main/scala/scaladex/infra/SqlDatabase.scala
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package scaladex.infra

import java.time.Instant
import java.time.OffsetDateTime
import java.time.ZoneOffset
import java.util.UUID

import scala.concurrent.ExecutionContext.Implicits.global
Expand Down Expand Up @@ -150,6 +152,11 @@ class SqlDatabase(datasource: HikariDataSource, xa: doobie.Transactor[IO]) exten
def countProjects(): Future[Long] =
run(ProjectTable.countProjects.unique)

def countProjects(year: Int): Future[Long] = {
val instant = OffsetDateTime.of(year + 1, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant
run(ProjectTable.countProjectsUntil.unique(instant))
}

override def countArtifacts(): Future[Long] =
run(ArtifactTable.count.unique)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,13 @@ object ProjectTable {
val countProjects: Query0[Long] =
selectRequest(table, Seq("count(*)"))

val countProjectsUntil: Query[Instant, Long] =
selectRequest1(
table,
Seq("count(*)"),
where = Seq("creation_date < ?", "github_status!='Moved'", "github_status!='NotFound'")
)

val selectByReference: Query[Project.Reference, Project] =
selectRequest(fullTable, allFields, referenceFields.map(f => s"p.$f"))

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,15 @@ import scaladex.core.model.Project
import scaladex.core.model.Project.Settings
import scaladex.core.model.UserState
import scaladex.core.service.GithubClient
import scaladex.core.service.SchedulerDatabase
import scaladex.core.service.SearchEngine
import scaladex.core.util.ScalaExtensions._
import scaladex.infra.SqlDatabase
import scaladex.view.Job
import scaladex.view.Task

class AdminService(
env: Env,
database: SchedulerDatabase,
database: SqlDatabase,
searchEngine: SearchEngine,
githubClientOpt: Option[GithubClient],
sonatypeSynchronizer: SonatypeService
Expand All @@ -39,7 +39,8 @@ class AdminService(
new JobScheduler(Job.projectDependencies, projectDependenciesUpdater.updateAll),
new JobScheduler(Job.projectCreationDates, updateProjectCreationDate),
new JobScheduler(Job.moveArtifacts, artifactsService.moveAll),
new JobScheduler(Job.userSessions, userSessionService.updateAll)
new JobScheduler(Job.userSessions, userSessionService.updateAll),
new JobScheduler(Job.metrics, (new Metrics(database)).run)
) ++
githubClientOpt.map { client =>
val githubUpdater = new GithubUpdater(database, client)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package scaladex.server.service

import scala.concurrent.ExecutionContext
import scala.concurrent.Future

import cats.implicits.toTraverseOps
import com.typesafe.scalalogging.LazyLogging
import scaladex.infra.SqlDatabase

class Metrics(db: SqlDatabase)(implicit ec: ExecutionContext) extends LazyLogging {
def run(): Future[String] = {
val years: Seq[Int] = Range.inclusive(2013, 2022)
for {
countByYear <- years.traverse(db.countProjects)
projects <- db.getAllProjects()
} yield {
years.zip(countByYear).foreach { case (year, count) => logger.info(s"$year: $count") }
val filteredProjects = projects.filter(p => !p.githubStatus.isMoved && !p.githubStatus.isNotFound)
logger.info(s"total projects: ${filteredProjects.size}")
val contributors = filteredProjects.flatMap(_.githubInfo).flatMap(_.contributors).map(_.login).distinct.size
logger.info(s"total contributors: $contributors")
"Success"
}
}
}
5 changes: 5 additions & 0 deletions modules/template/src/main/scala/scaladex/view/Job.scala
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,11 @@ object Job {
"Find missing artifacts in Maven Central of the known group IDs.",
24.hours
)
val metrics: Job = Job(
"metrics",
"Print regular metrics into the logs",
5.minutes
)

case class Status(state: State, results: Seq[Result], progress: Option[Progress]) {
def isStarted: Boolean = state.isInstanceOf[Started]
Expand Down
70 changes: 0 additions & 70 deletions server/src/main/scala/scaladex/server/service/Metrics.scala

This file was deleted.

0 comments on commit 5de2dc7

Please sign in to comment.