POC retrieve transitive dependencies with sbt.Classpaths$.interSort

extractor/src/main/scala/org/jetbrains/sbt/CreateTasks.scala

@@ -12,8 +12,6 @@ object CreateTasks extends (State => State) with SbtStateOps {
 
   lazy val globalSettings: Seq[Setting[_]] = Seq[Setting[_]](
     Keys.commands += UtilityTasks.preferScala2,
-    Keys.commands += UtilityTasks.internalDependencyClasspath2,
-    Keys.commands += UtilityTasks.exportedProductsNoTracking2,
     StructureKeys.sbtStructureOpts := StructureKeys.sbtStructureOptions.apply(Options.readFromString).value,
     StructureKeys.dumpStructure := UtilityTasks.dumpStructure.value,
     StructureKeys.acceptedProjects := UtilityTasks.acceptedProjects.value,
@@ -34,10 +32,10 @@ object CreateTasks extends (State => State) with SbtStateOps {
     StructureKeys.extractBuild := BuildExtractor.taskDef.value,
     StructureKeys.extractDependencies := DependenciesExtractor.taskDef.value,
     StructureKeys.extractProject := ProjectExtractor.taskDef.value,
-
+    StructureKeys.allSourceConfigurations := KeysExtractor.allSourceConfigurations.value,
+    StructureKeys.allTestConfigurations := KeysExtractor.allTestConfigurations.value,
     StructureKeys.allKeys := KeysExtractor.allKeys.value,
     StructureKeys.taskData := KeysExtractor.taskData.value,
-    StructureKeys.allSourceConfigs := KeysExtractor.allSourceConfigs.value,
     StructureKeys.settingData := KeysExtractor.settingData.value,
     StructureKeys.commandData := KeysExtractor.commandData.value,
 

extractor/src/main/scala/org/jetbrains/sbt/StructureKeys.scala

@@ -14,6 +14,8 @@ object StructureKeys {
   val testConfigurations: SettingKey[Seq[Configuration]] = SettingKey("ssTestConfigurations", rank = Invisible)
   val acceptedProjects: TaskKey[Seq[ProjectRef]] = TaskKey("ssAcceptedProjects", rank = Invisible)
 
+  val allSourceConfigurations: TaskKey[Seq[Configuration]] = TaskKey("ssAllSourceConfigurations", rank = Invisible)
+  val allTestConfigurations: TaskKey[Seq[Configuration]] = TaskKey("ssAllTestConfigurations", rank = Invisible)
   val extractPlay2: TaskKey[Option[Play2Data]] = TaskKey("ssExtractPlay2", rank = Invisible)
   val extractBuild: TaskKey[BuildData] = TaskKey("ssExtractBuild", rank = Invisible)
   val extractBuilds: TaskKey[Seq[BuildData]] = TaskKey("ssExtractBuilds", rank = Invisible)
@@ -25,7 +27,6 @@ object StructureKeys {
 
   val settingData: TaskKey[Seq[SettingData]] = TaskKey("settingData", rank = Invisible)
   val taskData: TaskKey[Seq[TaskData]] = TaskKey("taskData", rank = Invisible)
-  val allSourceConfigs : TaskKey[Seq[Configuration]] = TaskKey("allSourceConfigs", rank = Invisible)
   val commandData: TaskKey[Seq[CommandData]] = TaskKey("commandData", rank = Invisible)
   val localCachePath: TaskKey[Option[File]] = TaskKey("localCachePath", rank = Invisible)
   val allKeys:TaskKey[Seq[AttributeKey[_]]] = TaskKey("allKeys", rank = Invisible)

extractor/src/main/scala/org/jetbrains/sbt/extractors/DependenciesExtractor.scala

@@ -19,8 +19,6 @@ class DependenciesExtractor(projectRef: ProjectRef,
                             externalDependencyClasspath: Option[sbt.Configuration => Keys.Classpath],
                             dependencyConfigurations: Seq[sbt.Configuration],
                             testConfigurations: Seq[sbt.Configuration],
-                            internalDependencyClasspath: String => Keys.Classpath,
-                            exportedClasspathToProjectMapping: Map[File, ProjectRef],
                             insertProjectTransitiveDependencies: Boolean,
                             projectToConfigurations: Map[ProjectRef, Seq[jb.Configuration]])
   extends ModulesOps {
@@ -32,75 +30,13 @@ class DependenciesExtractor(projectRef: ProjectRef,
     DependencyData(projectDependencies, moduleDependencies, jarDependencies)
   }
 
-  private def transitiveProjectDependencies2 = {
+  private def transitiveProjectDependencies: Dependencies[ProjectDependencyData] = {
     val dependencies = projectToConfigurations.map { case(project, configurations) =>
-      ProjectDependencyData(project.id, Some(project.build), configurations)
+      ProjectDependencyData(project.id, Some(project.build), mapConfigurations(configurations))
     }.toSeq
     Dependencies(dependencies, Nil)
   }
 
-  private def processDependenciesMap[D, F](dependenciesList: Seq[(D, Seq[jb.Configuration])])
-                                          (mapToTargetType: ((D, Seq[jb.Configuration])) => F): Dependencies[F] = {
-    val productionDependencies = mutable.Map.empty[D, Seq[jb.Configuration]]
-    val testDependencies = mutable.Map.empty[D, Seq[jb.Configuration]]
-
-    def updateDependenciesInProductionAndTest(project: D, configsForProduction: Seq[jb.Configuration], configsForTest: Seq[jb.Configuration]): Unit = {
-      Seq((productionDependencies, configsForProduction), (testDependencies, configsForTest))
-        .filterNot(_._2.isEmpty)
-        .foreach { case (dependencies, configs) =>
-          val existingConfigurations = dependencies.getOrElse(project, Seq.empty)
-          dependencies.update(project, existingConfigurations ++ configs)
-        }
-    }
-
-    dependenciesList.foreach { case (dependency, configurations) =>
-      val cs = mapProjectDependenciesConfigurations(configurations)
-      updateDependenciesInProductionAndTest(dependency, cs.production, cs.test)
-    }
-    Dependencies(testDependencies.toSeq.map(mapToTargetType), productionDependencies.toSeq.map(mapToTargetType))
-  }
-
-  // We have to perform this configurations mapping because we're using sbt keys, which give
-  // us only information which dependencies are visible in what configurations instead of explicitly declared configurations.
-  private def mapProjectDependenciesConfigurations(configurations: Seq[jb.Configuration]): Dependencies[jb.Configuration] = {
-    val cs = swapAllDerivativesOfTestConfigurations(configurations)
-    // TODO: after splitting production and test sources all test configurations should be changed to Compile (sbt does not distinguish between test compile & runtime)
-    val resultConfigurations: (Seq[jb.Configuration], Seq[jb.Configuration]) = {
-      if (Seq(jb.Configuration.Compile, jb.Configuration.Test, jb.Configuration.Runtime).forall(cs.contains)) { // compile configuration
-        (Seq(jb.Configuration.Compile), Seq(jb.Configuration.Compile))
-      } else if (Seq(jb.Configuration.Compile, jb.Configuration.Test).forall(cs.contains)) { // provided configuration
-        (Seq(jb.Configuration.Provided), Seq(jb.Configuration.Provided))
-      } else if (Seq(jb.Configuration.Test, jb.Configuration.Runtime).forall(cs.contains)) { // runtime configuration
-        (Seq(jb.Configuration.Runtime), Seq(jb.Configuration.Runtime))
-      } else {
-        if (!cs.exists(Seq(jb.Configuration.Compile, jb.Configuration.Runtime, jb.Configuration.Test).contains(_))) {
-          (Seq(jb.Configuration.Compile), Seq(jb.Configuration.Compile))
-        } else {
-          Seq(
-            (cs.contains(jb.Configuration.Test), (Nil, Seq(jb.Configuration.Test))),
-            (cs.contains(jb.Configuration.Compile), (Seq(jb.Configuration.Provided), Nil)),
-            (cs.contains(jb.Configuration.Runtime), (Seq(jb.Configuration.Runtime), Nil))
-          ).foldLeft((Seq.empty[jb.Configuration], Seq.empty[jb.Configuration])) { (acc, cur) =>
-            val (condition, (productionConfigs, testConfigs)) = cur
-            if (condition) (acc._1 ++ productionConfigs, acc._2 ++ testConfigs)
-            else acc
-          }
-        }
-      }
-    }
-    Dependencies(resultConfigurations._2, resultConfigurations._1)
-  }
-
-  private def projectsIn(configuration: sbt.Configuration): Seq[(String, Some[URI])] =
-    for {
-      entry <- internalDependencyClasspath(configuration.name)
-      project <- exportedClasspathToProjectMapping.get(entry.data)
-      // TODO: remove this condition when production and test sources are separated
-      if !(project == projectRef)
-    } yield {
-      (project.project, Some(project.build))
-    }
-
   private def nonTransitiveProjectDependencies: Dependencies[ProjectDependencyData] = {
     val dependencies = buildDependencies.classpath.getOrElse(projectRef, Seq.empty).map { it =>
       val configurations = it.configuration.map(jb.Configuration.fromString).getOrElse(Seq.empty)
@@ -109,12 +45,6 @@ class DependenciesExtractor(projectRef: ProjectRef,
     Dependencies(dependencies, Nil)
   }
 
-  private def transitiveProjectDependencies: Dependencies[ProjectDependencyData] = {
-    processDependenciesMap(forAllConfigurations(projectsIn)) { case ((project, uri), configs) =>
-      ProjectDependencyData(project, uri, configs)
-    }
-  }
-
   private def moduleDependencies: Seq[ModuleDependencyData] =
     forAllConfigurations(modulesIn).map { case (moduleId, configurations) =>
       ModuleDependencyData(moduleId, mapConfigurations(configurations))
@@ -175,7 +105,6 @@ object DependenciesExtractor extends SbtStateOps with TaskOps {
     val options = StructureKeys.sbtStructureOpts.value
     val dependencyConfigurations = StructureKeys.dependencyConfigurations.value
     val testConfigurations = StructureKeys.testConfigurations.value
-    val sourceConfigurations = StructureKeys.sourceConfigurations.value
     val buildDependencies = Keys.buildDependencies.value
 
     val unmanagedClasspathTask =
@@ -188,118 +117,109 @@ object DependenciesExtractor extends SbtStateOps with TaskOps {
         .map(throwExceptionIfUpdateFailed)
         .onlyIf(options.download)
 
-    val acceptedProjects = StructureKeys.acceptedProjects.value
-
-    val allSourceConfigsTask = StructureKeys.allSourceConfigs
-      .forAllProjects(state, acceptedProjects)
-
-
-//    val internalDependencyClasspathTask = sbt.Keys.internalDependencyClasspath.in(projectRef)
-//      .forAllConfigurations(state, dependencyConfigurations)
-//      .result
-//      .map(throwExceptionIfUpdateFailed)
-
-
-    val exportedProductsTask = sbt.Keys.exportedProductsNoTracking
-      .forAllProjectsAndConfigurations(state, dependencyConfigurations, acceptedProjects)
-
-    val args = dependencyConfigurations.map(_.name).mkString(",")
-    val command = "internalDependencyClasspath2" + " " + s"${projectRef.build.getPath},${projectRef.project},$args"
-    val internalDependencyClasspathState = Command.process(command, state)
-    val exportedProductsState = Command.process("exportedProductsNoTracking2 " + args, state)
+    val allSourceConfigurationsTask = StructureKeys.allSourceConfigurations.in(projectRef).get(state)
+    val allTestConfigurationsTask = StructureKeys.allTestConfigurations.in(projectRef).get(state)
 
     val classpathConfigurationTask = sbt.Keys.classpathConfiguration.in(projectRef)
       .forAllConfigurations(state, dependencyConfigurations)
 
-    val settings = Keys.settingsData.value
+   val settings = Keys.settingsData.value
 
     Def.task {
-
       (for {
         unmanagedClasspath <- unmanagedClasspathTask
         externalDependencyClasspathOpt <- externalDependencyClasspathTask
-//        classpathConfiguration <- classpathConfigurationTask
-//        allSourceConfigs <- allSourceConfigsTask
+        classpathConfiguration <- classpathConfigurationTask
+        allSourceConfigurations <- allSourceConfigurationsTask
+        allTestConfigurations <- allTestConfigurationsTask
       } yield {
-        val projectsToClasspathMapping = exportedProductsState.get(StructureKeys.projectsToClasspathMapping).get
-        val exportedClasspathToProjectMapping = projectsToClasspathMapping.flatMap { case (classpath, project) =>
-          classpath.map { attributedFile => (attributedFile.data, project) }
-        }
-//        val finalAllSourceConfigs = allSourceConfigs.values.flatten.toSeq.distinct
-//            .groupBy(_.name)
-//            .mapValues(_.flatMap(_.extendsConfigs).map(_.name))
-//            .flatMap {case(key, value) =>
-//              Seq(key) ++ value
-//            }.toSeq
+        val allSourceConfigsWithExtends = allSourceConfigurations.distinct
+          .flatMap(config => transitiveExtends(config.extendsConfigs) :+ config)
+          .distinct
+
+        val allTestConfigsWithExtends = allTestConfigurations
+          .flatMap(config => transitiveExtends(config.extendsConfigs) :+ config)
+          .distinct
+
+        val allApplicableConfigs = (allTestConfigsWithExtends ++ allSourceConfigsWithExtends).distinct.map(_.name)
 
-        val allTestConfigs = testConfigurations.map(_.name)
+        val unknownConfigReplacement = (allSourceConfigurations ++ allTestConfigurations).distinct.flatMap { config =>
+          if (scopesAvailableInIntelliJ.contains(config.name)) None
+          else Some((config.name, findClosestConfigurationApplicableInIntelliJ(config.extendsConfigs)))
+        }.toMap
 
-        //val result = getTransitiveDependenciesInConfig(allTestConfigs, finalAllSourceConfigs, classpathConfiguration, projectRef, settings, buildDependencies)
-        val internalDependencyClasspathPerConfig = internalDependencyClasspathState.get(StructureKeys.internalDependencyClasspathPerConfig)
+        val result = getTransitiveDependenciesForProject(unknownConfigReplacement, allApplicableConfigs, classpathConfiguration, projectRef, settings, buildDependencies)
         new DependenciesExtractor(projectRef,
           buildDependencies,
           unmanagedClasspath.getOrElse(_, Nil),
           externalDependencyClasspathOpt.map(it => it.getOrElse(_, Nil)),
           dependencyConfigurations, testConfigurations,
-          internalDependencyClasspathPerConfig.get.getOrElse(_, Nil),
-          exportedClasspathToProjectMapping,
           options.insertProjectTransitiveDependencies,
-          Map.empty).extract
+          result).extract
 
       }).value
     }
   }
 
-  private def getTransitiveDependenciesInConfig(allTestConfigs: Seq[String], allCompileConfigs: Seq[String], classPathConfiguration: Map[sbt.Configuration, sbt.Configuration], projectRef: ProjectRef, settings: Settings[Scope], buildDependencies: BuildDependencies): Map[jb.Configuration, Seq[(ProjectRef, jb.Configuration)]] = {
-    def getTrans(self: sbt.Configuration, config: sbt.Configuration): Seq[(ProjectRef, jb.Configuration)] = {
+  private case class Dependency(project: ProjectRef, configuration: jb.Configuration)
+
+
+  private def getTransitiveDependenciesForProject(
+    unknownConfigReplacement: Map[String, jb.Configuration],
+    allApplicableConfigurations: Seq[String],
+    classPathConfiguration: Map[sbt.Configuration, sbt.Configuration],
+    projectRef: ProjectRef,
+    settings: Settings[Scope],
+    buildDependencies: BuildDependencies
+  ): Map[ProjectRef, Seq[jb.Configuration]] = {
+
+    def retrieveAllTransitiveDependencies(config: sbt.Configuration): Seq[Dependency] = {
       Classpaths.interSort(projectRef, config, settings, buildDependencies)
-        .filter { case (dep, c) =>
-//          (dep != projectRef) || (config.name != c && self.name != c)
-          (dep != projectRef)
-        }
+        .filter { case (dep, _) => dep != projectRef }
         .map { case (projectRef, configName) =>
-          (projectRef, jb.Configuration(configName))
+          Dependency(projectRef, jb.Configuration(configName))
         }
     }
 
-    val basic = classPathConfiguration.map { case (selfConfig, config) =>
-      (jb.Configuration(selfConfig.name), getTrans(selfConfig, config))
-    }
-    val checkScopes = basic.foldLeft(Map.empty[jb.Configuration, Seq[(ProjectRef, jb.Configuration)]]) { case (aggr, (config, values)) =>
-      if (Seq(Compile, Test, Runtime, Provided).map(_.name).contains(config.name)) {
-        val mergedValues = aggr.getOrElse(config, Seq.empty) ++ values
-        aggr.updated(config, mergedValues)
+    def replaceScopesWithThoseAvailableForIntelliJ(acc: Map[jb.Configuration, Seq[Dependency]], curr: (jb.Configuration, Seq[Dependency])): Map[jb.Configuration, Seq[Dependency]] = {
+      val (config, values) = curr
+      if (scopesAvailableInIntelliJ.contains(config.name)) {
+        acc.updated(config, acc.getOrElse(config, Seq.empty) ++ values)
       } else {
-          if(allCompileConfigs.contains(config.name)) {
-            val mergedValues = aggr.getOrElse(jb.Configuration.Compile, Seq.empty) ++ values
-            aggr.updated(jb.Configuration.Compile, mergedValues)
-          } else if (allTestConfigs.contains(config.name)){
-            val mergedValues = aggr.getOrElse(jb.Configuration.Test, Seq.empty) ++ values
-            aggr.updated(jb.Configuration.Test, mergedValues)
-          } else {
-            aggr
-          }
+        val foundConfig = unknownConfigReplacement(config.name)
+        acc.updated(foundConfig, acc.getOrElse(foundConfig, Seq.empty) ++ values)
       }
     }
-    val filterWhetherItShouldBeAddedAsDeps = checkScopes
-      .mapValues(_.groupBy(_._1))
-      .mapValues { k => k.mapValues(_.map(_._2)) }
-      .mapValues(mapa =>
-        mapa.filter { case (_, configs) =>
-          configs.exists(config => (allTestConfigs ++ allCompileConfigs).contains(config.name))
-        }.keys.toSeq
-      )
-
-    val projectToConfigurations = filterWhetherItShouldBeAddedAsDeps.foldLeft(Map.empty[ProjectRef, Seq[jb.Configuration]]) { case(aggr, (config, projects)) =>
-      val mutableMap = collection.mutable.Map(aggr.toSeq: _*)
-      projects.foreach { proj =>
-        val mergedValues = mutableMap.getOrElse(proj, Seq.empty) :+ config
-        mutableMap(proj) = mergedValues
+
+    def removeDependenciesWhichShouldNotBeAdded(scopeToDependencies: Seq[Dependency]): Seq[ProjectRef] = {
+      val groupedByProject = scopeToDependencies
+        .groupBy(_.project)
+        .mapValues(_.map(_.configuration.name))
+
+      groupedByProject.filter { case (_, configs) =>
+        configs.exists(allApplicableConfigurations.contains)
+      }.keys.toSeq
+    }
+
+    val configurationToDependencies = classPathConfiguration
+      .map { case (selfConfig, config) =>
+        (jb.Configuration(selfConfig.name), retrieveAllTransitiveDependencies(config))
+      }
+
+    val withReplacedScopesApplicableInIntelliJ = configurationToDependencies
+      .foldLeft(Map.empty[jb.Configuration, Seq[Dependency]]) { replaceScopesWithThoseAvailableForIntelliJ }
+
+    val filteredDependencies = withReplacedScopesApplicableInIntelliJ
+      .mapValues { removeDependenciesWhichShouldNotBeAdded }
+
+    val projectToConfigurations = collection.mutable.Map.empty[ProjectRef, Seq[jb.Configuration]]
+    filteredDependencies.foreach { case (config, projects) =>
+      projects.foreach { project =>
+        projectToConfigurations(project) = projectToConfigurations.getOrElse(project, Seq.empty) :+ config
       }
-      mutableMap.toMap
     }
 
-    basic
+    projectToConfigurations.toMap
   }
 
   private def throwExceptionIfUpdateFailed(result: Result[Map[sbt.Configuration,Keys.Classpath]]): Map[sbt.Configuration, Keys.Classpath] =

extractor/src/main/scala/org/jetbrains/sbt/extractors/KeysExtractor.scala

@@ -1,6 +1,6 @@
 package org.jetbrains.sbt.extractors
 
-import org.jetbrains.sbt.StructureKeys
+import org.jetbrains.sbt.{SbtStateOps, StructureKeys}
 import org.jetbrains.sbt.structure.{CommandData, SettingData, TaskData}
 import sbt.jetbrains.BadCitizen
 import sbt.{AttributeKey, BuiltinCommands, Configuration, Def, Extracted, KeyRanks, Keys, Logger, Project, SettingKey, Task}
@@ -12,7 +12,7 @@ import scala.util.control.NonFatal
   * Extract setting and task keys.
   * For instance, settings could show a stringified form directly.
   */
-object KeysExtractor {
+object KeysExtractor extends SbtStateOps {
 
   /** Maximum length of toString'ed setting value exported. */
   val maxValueStringLength = 103
@@ -56,8 +56,20 @@ object KeysExtractor {
     }
   }
 
-  val allSourceConfigs: Def.Initialize[Task[Seq[Configuration]]] = Def.task {
-    StructureKeys.sourceConfigurations.value
+  val allSourceConfigurations: Def.Initialize[Task[Seq[Configuration]]] = Def.task {
+    val acceptedProjects = StructureKeys.acceptedProjects.value
+    val state = Keys.state.value
+    acceptedProjects.flatMap { project =>
+      StructureKeys.sourceConfigurations.in(project).get(state)
+    }
+  }
+
+  val allTestConfigurations: Def.Initialize[Task[Seq[Configuration]]] = Def.task {
+    val acceptedProjects = StructureKeys.acceptedProjects.value
+    val state = Keys.state.value
+    acceptedProjects.flatMap { project =>
+      StructureKeys.testConfigurations.in(project).get(state)
+    }
   }