diff --git a/settings/src/main/scala/org/typelevel/sbt/TypelevelSettingsPlugin.scala b/settings/src/main/scala/org/typelevel/sbt/TypelevelSettingsPlugin.scala
index ad1a65b9..f2cddeea 100644
--- a/settings/src/main/scala/org/typelevel/sbt/TypelevelSettingsPlugin.scala
+++ b/settings/src/main/scala/org/typelevel/sbt/TypelevelSettingsPlugin.scala
@@ -195,10 +195,9 @@ object TypelevelSettingsPlugin extends AutoPlugin {
       "-encoding",
       "utf8",
       "-Xlint:all"
-    )
-  ) ++ inConfig(Compile)(perConfigSettings) ++ inConfig(Test)(perConfigSettings)
+    ),
 
-  private val perConfigSettings = Seq(
+    // TODO make these respect Compile/Test config
     scalacOptions ++= {
       if (tlFatalWarnings.value)
         Seq("-Xfatal-warnings")
@@ -237,7 +236,10 @@ object TypelevelSettingsPlugin extends AutoPlugin {
     },
     javacOptions ++= {
       withJdkRelease(tlJdkRelease.value)(Seq.empty[String])(n => Seq("--release", n.toString))
-    },
+    }
+  ) ++ inConfig(Compile)(perConfigSettings) ++ inConfig(Test)(perConfigSettings)
+
+  private val perConfigSettings = Seq(
     unmanagedSourceDirectories ++= {
       def extraDirs(suffix: String) =
         if (crossProjectPlatform.?.value.isDefined)