Merge pull request #61 from JetBrains/prodTest-sources-v2

separate modules for prod/test sources
JetBrains · Jun 19, 2024 · a654990 · a654990
2 parents db92bea + dd1146b
commit a654990
Show file tree

Hide file tree

Showing 98 changed files with 38,752 additions and 441 deletions.
diff --git a/extractor/src/main/scala/org/jetbrains/sbt/Options.scala b/extractor/src/main/scala/org/jetbrains/sbt/Options.scala
@@ -5,7 +5,8 @@ final case class Options(download: Boolean = false,
                          resolveJavadocClassifiers: Boolean = false,
                          resolveSbtClassifiers: Boolean = false,
                          prettyPrint: Boolean = false,
-                         insertProjectTransitiveDependencies: Boolean = true)
+                         insertProjectTransitiveDependencies: Boolean = true,
+                         separateProdAndTestSources: Boolean = false)
 
 object Options {
 
@@ -26,7 +27,8 @@ object Options {
     resolveJavadocClassifiers = options.contains(Keys.ResolveJavadocClassifiers),
     resolveSbtClassifiers = options.contains(Keys.ResolveSbtClassifiers),
     prettyPrint = options.contains(Keys.PrettyPrint),
-    insertProjectTransitiveDependencies = options.contains(Keys.InsertProjectTransitiveDependencies)
+    insertProjectTransitiveDependencies = options.contains(Keys.InsertProjectTransitiveDependencies),
+    separateProdAndTestSources = options.contains(Keys.SeparateProdAndTestSources)
   )
 
   object Keys {
@@ -36,6 +38,7 @@ object Options {
     val ResolveSbtClassifiers = "resolveSbtClassifiers"
     val PrettyPrint = "prettyPrint"
     val InsertProjectTransitiveDependencies = "insertProjectTransitiveDependencies"
+    val SeparateProdAndTestSources = "separateProdAndTestSources"
   }
 }
 
diff --git a/extractor/src/main/scala/org/jetbrains/sbt/extractors/DependenciesExtractor.scala b/extractor/src/main/scala/org/jetbrains/sbt/extractors/DependenciesExtractor.scala
diff --git a/extractor/src/main/scala/org/jetbrains/sbt/extractors/ProjectExtractor.scala b/extractor/src/main/scala/org/jetbrains/sbt/extractors/ProjectExtractor.scala
@@ -4,7 +4,7 @@ package extractors
 import org.jetbrains.sbt.structure._
 import sbt.Def.Initialize
 import sbt.jetbrains.keysAdapterEx
-import sbt.{Def, File, Configuration => _, _}
+import sbt.{Def, File, Configuration => SbtConfiguration, _}
 
 import scala.reflect.ClassTag
 import scala.util.{Failure, Success, Try}
@@ -35,17 +35,19 @@ class ProjectExtractor(
   scalaOrganization: String,
   scalaInstance: Option[ScalaInstance],
   scalaCompilerBridgeBinaryJar: Option[File],
-  scalacOptions: Seq[String],
+  scalacOptions: Seq[CompilerOptions],
   javaHome: Option[File],
-  javacOptions: Seq[String],
+  javacOptions: Seq[CompilerOptions],
   compileOrder: CompileOrder,
   sourceConfigurations: Seq[sbt.Configuration],
   testConfigurations: Seq[sbt.Configuration],
   dependencies: DependencyData,
   play2: Option[Play2Data],
   settingData: Seq[SettingData],
   taskData: Seq[TaskData],
-  commandData: Seq[CommandData]
+  commandData: Seq[CommandData],
+  mainSourceDirectories: Seq[File],
+  testSourceDirectories: Seq[File]
 ) {
 
   private[extractors] def extract: ProjectData = {
@@ -97,7 +99,9 @@ class ProjectExtractor(
       play2,
       settingData,
       taskData,
-      commandData
+      commandData,
+      mainSourceDirectories,
+      testSourceDirectories
     )
   }
 
@@ -272,6 +276,11 @@ object ProjectExtractor extends SbtStateOps with TaskOps {
                                                 state: State) =
     key.in(projectRef, Compile).get(state)
 
+  private def taskInConfig[T](key: TaskKey[T], config: SbtConfiguration)
+    (implicit projectRef: ProjectRef, state: State) =
+    key.in(projectRef, config).get(state)
+
+
   def taskDef: Initialize[Task[ProjectData]] = Def.taskDyn {
 
     implicit val state: State = Keys.state.value
@@ -325,10 +334,26 @@ object ProjectExtractor extends SbtStateOps with TaskOps {
         taskInCompile(Keys.scalaInstance).onlyIf(options.download).value
       val scalaCompilerBridgeBinaryJar =
         keysAdapterEx.myScalaCompilerBridgeBinaryJar.value
-      val scalacOptions =
-        taskInCompile(Keys.scalacOptions).onlyIf(options.download).value
-      val javacOptions =
-        taskInCompile(Keys.javacOptions).onlyIf(options.download).value
+
+      def mapToCompilerOptions(configToOptions: Seq[(Configuration, Seq[String])]) = {
+        configToOptions.collect { case(config, options) if options.nonEmpty =>
+          CompilerOptions(config, options)
+        }
+      }
+
+      val scalacOptions = mapToCompilerOptions(
+        Seq(
+          (Configuration.Compile, taskInConfig(Keys.scalacOptions, Compile).onlyIf(options.download).value.getOrElse(Seq.empty)),
+          (Configuration.Test, taskInConfig(Keys.scalacOptions, Test).onlyIf(options.download).value.getOrElse(Seq.empty))
+        )
+      )
+
+      val javacOptions = mapToCompilerOptions(
+        Seq(
+          (Configuration.Compile, taskInConfig(Keys.javacOptions, Compile).onlyIf(options.download).value.getOrElse(Seq.empty)),
+          (Configuration.Test, taskInConfig(Keys.javacOptions, Test).onlyIf(options.download).value.getOrElse(Seq.empty))
+        )
+      )
 
       val name = Keys.name.in(projectRef, Compile).value
       val organization = Keys.organization.in(projectRef, Compile).value
@@ -338,6 +363,16 @@ object ProjectExtractor extends SbtStateOps with TaskOps {
       val javaHome = Keys.javaHome.in(projectRef, Compile).value
       val compileOrder = Keys.compileOrder.in(projectRef, Compile).value
 
+      val sourceConfigurations = StructureKeys.sourceConfigurations.value
+      val testConfigurations = StructureKeys.testConfigurations.value
+
+      val mainSourceDirectories = Keys.sourceDirectory.in(projectRef)
+        .forAllConfigurations(state, sourceConfigurations)
+        .map(_._2).distinct
+      val testSourceDirectories = Keys.sourceDirectory.in(projectRef)
+        .forAllConfigurations(state, testConfigurations)
+        .map(_._2).distinct
+
       new ProjectExtractor(
         projectRef,
         name,
@@ -358,17 +393,19 @@ object ProjectExtractor extends SbtStateOps with TaskOps {
         scalaOrganization,
         scalaInstance,
         scalaCompilerBridgeBinaryJar,
-        scalacOptions.getOrElse(Seq.empty),
+        scalacOptions,
         javaHome,
-        javacOptions.getOrElse(Seq.empty),
+        javacOptions,
         compileOrder,
         StructureKeys.sourceConfigurations.value,
         StructureKeys.testConfigurations.value,
         StructureKeys.extractDependencies.value,
         StructureKeys.extractPlay2.value,
         StructureKeys.settingData.value,
         StructureKeys.taskData.value,
-        StructureKeys.commandData.value.distinct
+        StructureKeys.commandData.value.distinct,
+        mainSourceDirectories,
+        testSourceDirectories
       ).extract
     }
   }

diff --git a/extractor/src/main/scala/org/jetbrains/sbt/extractors/UtilityTasks.scala b/extractor/src/main/scala/org/jetbrains/sbt/extractors/UtilityTasks.scala
@@ -140,9 +140,12 @@ object UtilityTasks extends SbtStateOps {
       val transitiveTest = cs.filter(c =>
         transitiveExtends(c.extendsConfigs)
           .toSet
-          .intersect(predefinedTest).nonEmpty) ++
-        predefinedTest
-      transitiveTest.distinct
+          .intersect(predefinedTest).nonEmpty
+      )
+      // note: IntegrationTest is not a predefined configuration in each sbt project. It has to be manually enabled.
+      // So returning it from testConfigurations is not necessary and it causes incorrect values to be returned from the sourceDirectory key.
+      val predefinedAvailableTest = predefinedTest.filter(cs.contains).toSeq
+      (transitiveTest ++ predefinedAvailableTest).distinct
     }
 
   def sourceConfigurations: Def.Initialize[Seq[Configuration]] = Def.setting {

diff --git a/extractor/src/main/scala/org/jetbrains/sbt/operations.scala b/extractor/src/main/scala/org/jetbrains/sbt/operations.scala
@@ -44,6 +44,10 @@ trait SbtStateOps {
     def forAllProjects(state: State, projects: Seq[ProjectRef]): Seq[(ProjectRef, T)] =
       projects.flatMap(p => key.in(p).find(state).map(it => (p, it)))
 
+    def forAllConfigurations(state: State, configurations: Seq[sbt.Configuration]): Seq[(sbt.Configuration, T)] = {
+      configurations.flatMap(c => key.in(c).get(structure(state).data).map(it => (c, it)))
+    }
+
   }
 
   implicit class `enrich TaskKey`[T](key: TaskKey[T]) {

diff --git a/extractor/src/test/data/0.13/bare/structure.xml b/extractor/src/test/data/0.13/bare/structure.xml
@@ -151,7 +151,20 @@
       <classes>$BASE/target/scala-2.10/test-classes</classes>
     </configuration>
     <dependencies>
-      <module configurations="compile" organization="org.scala-lang" name="scala-library" revision="2.10.1" artifactType="jar" classifier=""/>
+      <projects>
+        <forTest/>
+        <forProduction/>
+      </projects>
+      <modules>
+        <forTest/>
+        <forProduction>
+          <module configurations="compile" organization="org.scala-lang" name="scala-library" revision="2.10.1" artifactType="jar" classifier=""/>
+        </forProduction>
+      </modules>
+      <jars>
+        <forTest/>
+        <forProduction/>
+      </jars>
     </dependencies>
     <resolver name="public" root="https://repo1.maven.org/maven2/"/>
     <resolver name="local-preloaded" root="file:$HOME/.sbt-structure-global/0.13/preloaded"/>
@@ -1666,6 +1679,8 @@
     <command>
       <name>---</name>
     </command>
+    <testSourceDir>$BASE/src/test</testSourceDir>
+    <mainSourceDir>$BASE/src/main</mainSourceDir>
   </project>
   <repository>
     <module organization="org.scala-lang" name="scala-library" revision="2.10.1" artifactType="jar" classifier="">

diff --git a/extractor/src/test/data/0.13/classifiers/structure.xml b/extractor/src/test/data/0.13/classifiers/structure.xml
@@ -152,7 +152,20 @@
       <classes>$BASE/target/scala-2.10/test-classes</classes>
     </configuration>
     <dependencies>
-      <module configurations="compile" organization="org.scala-lang" name="scala-library" revision="2.10.7" artifactType="jar" classifier=""/>
+      <projects>
+        <forTest/>
+        <forProduction/>
+      </projects>
+      <modules>
+        <forTest/>
+        <forProduction>
+          <module configurations="compile" organization="org.scala-lang" name="scala-library" revision="2.10.7" artifactType="jar" classifier=""/>
+        </forProduction>
+      </modules>
+      <jars>
+        <forTest/>
+        <forProduction/>
+      </jars>
     </dependencies>
     <resolver name="public" root="https://repo1.maven.org/maven2/"/>
     <resolver name="local-preloaded" root="file:$HOME/.sbt-structure-global/0.13/preloaded"/>
@@ -1666,6 +1679,8 @@
     <command>
       <name>---</name>
     </command>
+    <testSourceDir>$BASE/src/test</testSourceDir>
+    <mainSourceDir>$BASE/src/main</mainSourceDir>
   </project>
   <project>
     <id>bar</id>
@@ -1708,11 +1723,24 @@
       <classes>$BASE/bar/target/scala-2.10/test-classes</classes>
     </configuration>
     <dependencies>
-      <module configurations="compile" organization="javax.activation" name="activation" revision="1.1.1" artifactType="jar" classifier=""/>
-      <module configurations="compile" organization="javax.mail" name="mail" revision="1.4.4" artifactType="jar" classifier=""/>
-      <module configurations="compile" organization="org.apache.james" name="apache-mailet-api" revision="2.5.0" artifactType="jar" classifier=""/>
-      <module configurations="compile" organization="org.apache.james" name="apache-mailet-base" revision="2.5.0" artifactType="jar" classifier=""/>
-      <module configurations="compile" organization="org.scala-lang" name="scala-library" revision="2.10.7" artifactType="jar" classifier=""/>
+      <projects>
+        <forTest/>
+        <forProduction/>
+      </projects>
+      <modules>
+        <forTest/>
+        <forProduction>
+          <module configurations="compile" organization="javax.activation" name="activation" revision="1.1.1" artifactType="jar" classifier=""/>
+          <module configurations="compile" organization="org.apache.james" name="apache-mailet-api" revision="2.5.0" artifactType="jar" classifier=""/>
+          <module configurations="compile" organization="org.apache.james" name="apache-mailet-base" revision="2.5.0" artifactType="jar" classifier=""/>
+          <module configurations="compile" organization="javax.mail" name="mail" revision="1.4.4" artifactType="jar" classifier=""/>
+          <module configurations="compile" organization="org.scala-lang" name="scala-library" revision="2.10.7" artifactType="jar" classifier=""/>
+        </forProduction>
+      </modules>
+      <jars>
+        <forTest/>
+        <forProduction/>
+      </jars>
     </dependencies>
     <resolver name="public" root="https://repo1.maven.org/maven2/"/>
     <resolver name="local-preloaded" root="file:$HOME/.sbt-structure-global/0.13/preloaded"/>
@@ -3226,6 +3254,8 @@
     <command>
       <name>---</name>
     </command>
+    <testSourceDir>$BASE/bar/src/test</testSourceDir>
+    <mainSourceDir>$BASE/bar/src/main</mainSourceDir>
   </project>
   <project>
     <id>foo</id>
@@ -3268,12 +3298,25 @@
       <classes>$BASE/foo/target/scala-2.10/test-classes</classes>
     </configuration>
     <dependencies>
-      <module configurations="compile" organization="javax.activation" name="activation" revision="1.1.1" artifactType="jar" classifier=""/>
-      <module configurations="compile" organization="javax.mail" name="mail" revision="1.4.4" artifactType="jar" classifier=""/>
-      <module configurations="compile" organization="org.apache.james" name="apache-mailet-api" revision="2.5.0" artifactType="jar" classifier=""/>
-      <module configurations="compile" organization="org.apache.james" name="apache-mailet-base" revision="2.5.0" artifactType="jar" classifier=""/>
-      <module configurations="test" organization="org.apache.james" name="apache-mailet-base" revision="2.5.0" artifactType="jar" classifier="tests"/>
-      <module configurations="compile" organization="org.scala-lang" name="scala-library" revision="2.10.7" artifactType="jar" classifier=""/>
+      <projects>
+        <forTest/>
+        <forProduction/>
+      </projects>
+      <modules>
+        <forTest/>
+        <forProduction>
+          <module configurations="compile" organization="javax.activation" name="activation" revision="1.1.1" artifactType="jar" classifier=""/>
+          <module configurations="compile" organization="org.apache.james" name="apache-mailet-api" revision="2.5.0" artifactType="jar" classifier=""/>
+          <module configurations="test" organization="org.apache.james" name="apache-mailet-base" revision="2.5.0" artifactType="jar" classifier="tests"/>
+          <module configurations="compile" organization="org.apache.james" name="apache-mailet-base" revision="2.5.0" artifactType="jar" classifier=""/>
+          <module configurations="compile" organization="javax.mail" name="mail" revision="1.4.4" artifactType="jar" classifier=""/>
+          <module configurations="compile" organization="org.scala-lang" name="scala-library" revision="2.10.7" artifactType="jar" classifier=""/>
+        </forProduction>
+      </modules>
+      <jars>
+        <forTest/>
+        <forProduction/>
+      </jars>
     </dependencies>
     <resolver name="public" root="https://repo1.maven.org/maven2/"/>
     <resolver name="local-preloaded" root="file:$HOME/.sbt-structure-global/0.13/preloaded"/>
@@ -4787,6 +4830,8 @@
     <command>
       <name>---</name>
     </command>
+    <testSourceDir>$BASE/foo/src/test</testSourceDir>
+    <mainSourceDir>$BASE/foo/src/main</mainSourceDir>
   </project>
   <repository>
     <module organization="javax.activation" name="activation" revision="1.1.1" artifactType="jar" classifier="">

diff --git a/extractor/src/test/data/0.13/custom-test-config/structure.xml b/extractor/src/test/data/0.13/custom-test-config/structure.xml
@@ -158,14 +158,27 @@
       <classes>$BASE/target/scala-2.10/fun-classes</classes>
     </configuration>
     <dependencies>
-      <module configurations="compile" organization="org.scala-lang" name="scala-library" revision="2.10.1" artifactType="jar" classifier=""/>
-      <module configurations="test" organization="org.scala-lang" name="scala-reflect" revision="2.10.1" artifactType="jar" classifier=""/>
-      <module configurations="test" organization="org.scalaz" name="scalaz-concurrent_2.10" revision="7.1.0" artifactType="jar" classifier=""/>
-      <module configurations="test" organization="org.scalaz" name="scalaz-core_2.10" revision="7.1.0" artifactType="jar" classifier=""/>
-      <module configurations="test" organization="org.scalaz" name="scalaz-effect_2.10" revision="7.1.0" artifactType="jar" classifier=""/>
-      <module configurations="test" organization="org.specs2" name="specs2-common_2.10" revision="2.4.14" artifactType="jar" classifier=""/>
-      <module configurations="test" organization="org.specs2" name="specs2-core_2.10" revision="2.4.14" artifactType="jar" classifier=""/>
-      <module configurations="test" organization="org.specs2" name="specs2-matcher_2.10" revision="2.4.14" artifactType="jar" classifier=""/>
+      <projects>
+        <forTest/>
+        <forProduction/>
+      </projects>
+      <modules>
+        <forTest/>
+        <forProduction>
+          <module configurations="compile" organization="org.scala-lang" name="scala-library" revision="2.10.1" artifactType="jar" classifier=""/>
+          <module configurations="test" organization="org.scala-lang" name="scala-reflect" revision="2.10.1" artifactType="jar" classifier=""/>
+          <module configurations="test" organization="org.scalaz" name="scalaz-concurrent_2.10" revision="7.1.0" artifactType="jar" classifier=""/>
+          <module configurations="test" organization="org.scalaz" name="scalaz-core_2.10" revision="7.1.0" artifactType="jar" classifier=""/>
+          <module configurations="test" organization="org.scalaz" name="scalaz-effect_2.10" revision="7.1.0" artifactType="jar" classifier=""/>
+          <module configurations="test" organization="org.specs2" name="specs2-common_2.10" revision="2.4.14" artifactType="jar" classifier=""/>
+          <module configurations="test" organization="org.specs2" name="specs2-core_2.10" revision="2.4.14" artifactType="jar" classifier=""/>
+          <module configurations="test" organization="org.specs2" name="specs2-matcher_2.10" revision="2.4.14" artifactType="jar" classifier=""/>
+        </forProduction>
+      </modules>
+      <jars>
+        <forTest/>
+        <forProduction/>
+      </jars>
     </dependencies>
     <resolver name="public" root="https://repo1.maven.org/maven2/"/>
     <resolver name="local-preloaded" root="file:$HOME/.sbt-structure-global/0.13/preloaded"/>
@@ -1675,6 +1688,9 @@
     <command>
       <name>---</name>
     </command>
+    <testSourceDir>$BASE/src/fun</testSourceDir>
+    <testSourceDir>$BASE/src/test</testSourceDir>
+    <mainSourceDir>$BASE/src/main</mainSourceDir>
   </project>
   <repository>
     <module organization="org.scala-lang" name="scala-library" revision="2.10.1" artifactType="jar" classifier="">