diff --git a/gradle/libraries.toml b/gradle/libraries.toml
index efb317cca0a..a238ebad05a 100644
--- a/gradle/libraries.toml
+++ b/gradle/libraries.toml
@@ -106,6 +106,7 @@ jetbrains-annotations = { group = "org.jetbrains", name = "annotations", version
 junit = { group = "junit", name = "junit", version.ref = "junit" }
 kotlin-allopen = { group = "org.jetbrains.kotlin", name = "kotlin-allopen", version.ref = "kotlin-plugin" }
 kotlin-compiletesting = { group = "com.github.tschuchortdev", name = "kotlin-compile-testing", version = "1.5.0" }
+graphql-java = "com.graphql-java:graphql-java:20.4"
 # The main kotlin version for build-logic and Gradle tests
 kotlin-plugin = { group = "org.jetbrains.kotlin", name = "kotlin-gradle-plugin", version.ref = "kotlin-plugin" }
 # For Gradle integration tests to make sure we stay compatible with 1.5.0
diff --git a/tests/ast-benchmark/build.gradle.kts b/tests/ast-benchmark/build.gradle.kts
index e3c0e10b3e0..8cf11fdc8a2 100644
--- a/tests/ast-benchmark/build.gradle.kts
+++ b/tests/ast-benchmark/build.gradle.kts
@@ -14,6 +14,8 @@ benchmark {
 
 dependencies {
   implementation("com.apollographql.apollo3:apollo-ast")
+  implementation(golatac.lib("graphql.java"))
+  implementation("org.jetbrains.kotlin:kotlin-test")
 
   add("jmhImplementation", golatac.lib("kotlinx.benchmark.runtime"))
   add("jmhImplementation", sourceSets.main.get().output + sourceSets.main.get().runtimeClasspath)
diff --git a/tests/ast-benchmark/src/jmh/kotlin/benchmark/Benchmark.kt b/tests/ast-benchmark/src/jmh/kotlin/benchmark/Benchmark.kt
index 06a7b16d6b0..0f82d0ce5ff 100644
--- a/tests/ast-benchmark/src/jmh/kotlin/benchmark/Benchmark.kt
+++ b/tests/ast-benchmark/src/jmh/kotlin/benchmark/Benchmark.kt
@@ -26,6 +26,13 @@ open class Benchmark {
     testFiles = File(".").resolve("../../libraries/apollo-compiler/src/test/graphql")
         .walk()
         .filter { it.extension in setOf("graphql", "graphqls") }
+        .filter {
+          when (it.parentFile.name) {
+            "empty", "__schema" -> false // contains empty document which are not spec compliant
+            "simple_fragment" -> false // contains operation/fragment descriptions which are not spec compliant
+            else -> true
+          }
+        }
         .toList()
   }
 
@@ -42,4 +49,11 @@ open class Benchmark {
       it.source().buffer().parseAsGQLDocument(options = ParserOptions(useAntlr = false)).getOrThrow().definitions.size.toDouble()
     }
   }
+
+  @Benchmark
+  fun graphqlJava(): Double {
+    return testFiles.sumOf {
+      graphql.parser.Parser.parse(it.readText()).definitions.size.toDouble()
+    }
+  }
 }
\ No newline at end of file