Initial E2E tests infrastructure (#29)

* stage

* stage

* E2E controlled startup and traffic simulators

* setup to bootstrap UI tests to services overview page with current service listed

* minimal tests

* minimal documentation for running tests

* update license headers

* fix license header in DotSettings

* junit and github actions logger dont gel well together

* allow unit/integration/e2e test to be split and called in isolation

* Rename .IntegrationTests to EndToEndTests

* Move to Nullean.Xunit.Partitions and simplify test logging

* introduce --test-suite to better control what tests need to run

* allow equal assignment on flags

* pass env's more explicitly

* ensure we can write to console out early if e2e is not configured properly

* possible null handling in StartedConfirationHandler

* ensure validation only happens if we run e2e tests

* Update to latest partitions xunit that keeps track of the last test exception

* fix vault lookup

* ci: use pro secret

* temporarily disable e2e tests

* restore deleted development files

* rename GlobalSetup to EndToEndOptions

---------

Co-authored-by: Victor Martinez <[email protected]>

.github/workflows/bootstrap/action.yml

@@ -32,7 +32,6 @@ runs:
           dotnet-version: |
             6.0.x
             8.0.x
-            6.0.x
     - id: dotnet
       shell: bash
       run: |

.github/workflows/ci.yml

@@ -35,7 +35,7 @@ jobs:
         id: bootstrap
         uses: ./.github/workflows/bootstrap
 
-      - run: build.bat test
+      - run: build.bat test --test-suite=skip-e2e
         shell: cmd
         name: Test
 
@@ -48,5 +48,5 @@ jobs:
         uses: ./.github/workflows/bootstrap
 
       # We still run the full release build on pull-requests this ensures packages are validated ahead of time
-      - run: ./build.sh release
+      - run: ./build.sh release --test-suite=skip-e2e
         name: Release

.github/workflows/e2e.yml

@@ -24,16 +24,17 @@ concurrency:
 env:
   # (keep_serverless-staging-oblt, keep_serverless-qa-oblt or serverless-production-oblt)
   SERVERLESS_PROJECT: serverless-production-oblt
-  # (staging, qa or production)
-  VAULT_SECRET_SUFFIX: production
+  # (staging, qa or pro)
+  VAULT_SECRET_SUFFIX: pro
 
 # NOTE: if you add a new job and it's a mandatory check then
 #       update e2e-docs.yml
 jobs:
   test:
-    if: |
-      github.event_name != 'pull_request' || 
-      (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false)
+    if: false()
+    #if: |
+    # github.event_name != 'pull_request' || 
+    # (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false)
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -55,10 +56,6 @@ jobs:
           vault-role-id: ${{ secrets.VAULT_ROLE_ID }}
           vault-secret-id: ${{ secrets.VAULT_SECRET_ID }}
 
-      # TODO: run the e2e targeting the required endpoint.
-      #       those values can be found in https://github.com/elastic/apm-pipeline-library/tree/main/.github/actions/oblt-cli-cluster-credentials#outputs
-      - run: curl -X GET "${ELASTICSEARCH_HOST}/_cat/indices?v" -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}
-
       - name: Get the browser email and password from Vault
         uses: hashicorp/[email protected]
         with:
@@ -67,5 +64,10 @@ jobs:
           secretId: ${{ secrets.VAULT_SECRET_ID }}
           method: approle
           secrets: |
-            secret/observability-team/ci/elastic-cloud/observability-team-${{ env.VAULT_SECRET_SUFFIX }} username | E2E__BROWSEREMAIL
+            secret/observability-team/ci/elastic-cloud/observability-team-${{ env.VAULT_SECRET_SUFFIX }} username | E2E__BROWSEREMAIL ;
             secret/observability-team/ci/elastic-cloud/observability-team-${{ env.VAULT_SECRET_SUFFIX }} password | E2E__BROWSERPASSWORD
+        
+      - run: ./build.sh test --test-suite=e2e
+        env: 
+          E2E__ENDPOINT: "${{env.ELASTIC_APM_SERVER_URL}}"
+          E2E__AUTHORIZATION: "Authentication=ApiKey ${{env.ELASTIC_APM_API_KEY}}"

.github/workflows/prerelease.yml

@@ -21,7 +21,7 @@ jobs:
         id: bootstrap
         uses: ./.github/workflows/bootstrap
 
-      - run: ./build.sh release
+      - run: ./build.sh release --test-suite=skip-e2e
         name: Release
 
       - name: publish canary packages github package repository

.github/workflows/release.yml

@@ -24,7 +24,7 @@ jobs:
       id: bootstrap
       uses: ./.github/workflows/bootstrap
 
-    - run: ./build.sh release --skiptests
+    - run: ./build.sh release --test-suite=skip-all
       name: Release
 
     - name: Prepare Nuget

Elastic.OpenTelemetry.sln

@@ -22,6 +22,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{AAD39891
 	ProjectSection(SolutionItems) = preProject
 		tests\Directory.Build.props = tests\Directory.Build.props
 		tests\xunit.runner.json = tests\xunit.runner.json
+		tests\.runsettings = tests\.runsettings
 	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elastic.OpenTelemetry.Tests", "tests\Elastic.OpenTelemetry.Tests\Elastic.OpenTelemetry.Tests.csproj", "{22BF9223-3A6D-4197-8527-3E4E43A98A81}"
@@ -34,6 +35,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Example.Elastic.OpenTelemet
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elastic.OpenTelemetry.AspNetCore", "src\Elastic.OpenTelemetry.AspNetCore\Elastic.OpenTelemetry.AspNetCore.csproj", "{2139F902-B10D-475D-8A38-F78962CEBFD3}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.OpenTelemetry.EndToEndTests", "tests\Elastic.OpenTelemetry.EndToEndTests\Elastic.OpenTelemetry.EndToEndTests.csproj", "{B970DBE1-6A04-4014-A285-6A9F36421025}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -76,6 +79,10 @@ Global
 		{2139F902-B10D-475D-8A38-F78962CEBFD3}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{2139F902-B10D-475D-8A38-F78962CEBFD3}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{2139F902-B10D-475D-8A38-F78962CEBFD3}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B970DBE1-6A04-4014-A285-6A9F36421025}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B970DBE1-6A04-4014-A285-6A9F36421025}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B970DBE1-6A04-4014-A285-6A9F36421025}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B970DBE1-6A04-4014-A285-6A9F36421025}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -87,6 +94,7 @@ Global
 		{EC81FA30-C765-4F04-8679-86F16DA3CC65} = {4E95C87B-655B-4BC3-8F2A-DF06B7AAB7E9}
 		{4377A059-16E0-4D5D-AC03-44C09BCE5BC4} = {4E95C87B-655B-4BC3-8F2A-DF06B7AAB7E9}
 		{2139F902-B10D-475D-8A38-F78962CEBFD3} = {E622CFF2-C6C4-40FB-BE42-7C4F2B38B75A}
+		{B970DBE1-6A04-4014-A285-6A9F36421025} = {AAD39891-0B70-47FA-A212-43E1AAE5DF56}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {573B2B5F-8CBB-4D52-A55A-4E65E282AAFB}

Elastic.OpenTelemetry.sln.DotSettings

@@ -84,8 +84,7 @@
     <Reformat>false</Reformat>
   </Language>
 &amp;lt;/profile&amp;gt;&lt;/RIDER_SETTINGS&gt;&lt;/Profile&gt;</s:String>
-	<s:String x:Key="/Default/CodeStyle/FileHeader/FileHeaderText/@EntryValue">Licensed to Elasticsearch B.V under 
-one or more agreements.
+	<s:String x:Key="/Default/CodeStyle/FileHeader/FileHeaderText/@EntryValue">Licensed to Elasticsearch B.V under one or more agreements.
 Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 See the LICENSE file in the project root for more information</s:String>
 	<s:Boolean x:Key="/Default/CodeEditing/GenerateMemberBody/PlaceBackingFieldAboveProperty/@EntryValue">True</s:Boolean>
@@ -564,6 +563,7 @@ See the LICENSE file in the project root for more information</s:String>
 	<s:Boolean x:Key="/Default/CodeStyle/IntroduceVariableUseVar/UseVarForIntroduceVariableRefactoring/@EntryValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/CodeStyle/LiveTemplatesUseVar/UseVar/@EntryValue">False</s:Boolean>
 	<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=KM/@EntryIndexedValue">KM</s:String>
+	<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=UI/@EntryIndexedValue">UI</s:String>
 	<s:Boolean x:Key="/Default/CodeStyle/Naming/CSharpNaming/ApplyAutoDetectedRules/@EntryValue">False</s:Boolean>
 	<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EFeature_002EServices_002ECodeCleanup_002EFileHeader_002EFileHeaderSettingsMigrate/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ECSharpFileLayoutPatternsUpgrade/@EntryIndexedValue">True</s:Boolean>

build/build.fsproj

@@ -1,31 +1,33 @@
 <Project Sdk="Microsoft.NET.Sdk">
-    <PropertyGroup>
-        <TargetFramework>net8.0</TargetFramework>
-        <OutputType>Exe</OutputType>
-        <NoWarn>$(NoWarn);NU1701</NoWarn>
-        <IsPackable>false</IsPackable>
-    </PropertyGroup>
-    <ItemGroup>
-        <PackageReference Include="Argu" Version="6.1.4" />
-        <PackageReference Include="Bullseye" Version="4.2.1" />
-        <PackageReference Include="Proc.Fs" Version="0.7.2" />
-        <PackageReference Include="Fake.Tools.Git" Version="5.20.3"/>
-    </ItemGroup>
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <OutputType>Exe</OutputType>
+    <NoWarn>$(NoWarn);NU1701</NoWarn>
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Argu" Version="6.1.4"/>
+    <PackageReference Include="Bullseye" Version="4.2.1"/>
+    <PackageReference Include="Proc.Fs" Version="0.8.1"/>
+    <PackageReference Include="Fake.Tools.Git" Version="5.20.3"/>
+    <PackageReference Remove="FSharp.Core"/>
+    <PackageReference Include="FSharp.Core" Version="8.0.101"/>
+  </ItemGroup>
 
-    <ItemGroup>
-        <None Include="..\*" LinkBase="_root"/>
-    </ItemGroup>
+  <ItemGroup>
+    <None Include="..\*" LinkBase="_root"/>
+  </ItemGroup>
 
-    <ItemGroup>
-        <Compile Include="scripts\BuildInformation.fs" />
-        <Compile Include="scripts\CommandLine.fs"/>
-        <Compile Include="scripts\Targets.fs"/>
-        <Compile Include="scripts\Program.fs"/>
-        <None Include="**\*" />
-        <None Remove="bin\**" />
-        <None Remove="obj\**" />
-        <None Remove="scripts\**" />
-        <None Remove="output\**" />
-    </ItemGroup>
+  <ItemGroup>
+    <Compile Include="scripts\BuildInformation.fs"/>
+    <Compile Include="scripts\CommandLine.fs"/>
+    <Compile Include="scripts\Targets.fs"/>
+    <Compile Include="scripts\Program.fs"/>
+    <None Include="**\*"/>
+    <None Remove="bin\**"/>
+    <None Remove="obj\**"/>
+    <None Remove="scripts\**"/>
+    <None Remove="output\**"/>
+  </ItemGroup>
 
 </Project>

build/scripts/CommandLine.fs

@@ -9,12 +9,21 @@ open Microsoft.FSharp.Reflection
 open System
 open Bullseye
 
+type TestSuite = All | Unit | Integration | E2E | Skip_All | Skip_E2E
+    with 
+    member this.SuitName =
+        match FSharpValue.GetUnionFields(this, typeof<TestSuite>) with
+        | case, _ -> case.Name.ToLowerInvariant()
+
 type Build =
     | [<CliPrefix(CliPrefix.None);SubCommand>] Clean
     | [<CliPrefix(CliPrefix.None);SubCommand>] Version
     | [<CliPrefix(CliPrefix.None);SubCommand>] Build
     | [<CliPrefix(CliPrefix.None);SubCommand>] Test
 
+    | [<CliPrefix(CliPrefix.None);SubCommand>] Unit_Test
+    | [<CliPrefix(CliPrefix.None);SubCommand>] End_To_End
+
     | [<CliPrefix(CliPrefix.None);Hidden;SubCommand>] PristineCheck 
     | [<CliPrefix(CliPrefix.None);Hidden;SubCommand>] GeneratePackages
     | [<CliPrefix(CliPrefix.None);Hidden;SubCommand>] ValidateLicenses 
@@ -23,10 +32,10 @@ type Build =
     | [<CliPrefix(CliPrefix.None);Hidden;SubCommand>] GenerateApiChanges 
     | [<CliPrefix(CliPrefix.None);SubCommand>] Release
 
-    | [<Inherit;AltCommandLine("-s")>] SingleTarget
+    | [<Inherit;AltCommandLine("-s")>] Single_Target
     | [<Inherit>] Token of string 
-    | [<Inherit;AltCommandLine("-c")>] SkipDirtyCheck
-    | [<Inherit;>] SkipTests
+    | [<Inherit;AltCommandLine("-c")>] Skip_Dirty_Check
+    | [<Inherit;EqualsAssignment>] Test_Suite of TestSuite 
 with
     interface IArgParserTemplate with
         member this.Usage =
@@ -35,7 +44,10 @@ with
             | Clean -> "clean known output locations"
             | Version -> "print version information"
             | Build -> "Run build"
-            | Test -> "Runs build then tests"
+
+            | Unit_Test -> "alias to providing: test --test-suite=unit"
+            | End_To_End -> "alias to providing: test --test-suite=e2e"
+            | Test -> "runs a clean build and then runs all the tests unless --test-suite is provided"
             | Release -> "runs build, tests, and create and validates the packages shy of publishing them"
 
             // steps
@@ -47,10 +59,11 @@ with
             | GenerateApiChanges -> "Undocumented, dependent target"
 
             // flags
-            | SingleTarget -> "Runs the provided sub command without running their dependencies"
+            | Single_Target -> "Runs the provided sub command without running their dependencies"
             | Token _ -> "Token to be used to authenticate with github"
-            | SkipDirtyCheck -> "Skip the clean checkout check that guards the release/publish targets"
-            | SkipTests -> "Skips running tests"
+            | Skip_Dirty_Check -> "Skip the clean checkout check that guards the release/publish targets"
+            | Test_Suite _ -> "Specify the test suite to run, defaults to all"
+
 
     member this.StepName =
         match FSharpValue.GetUnionFields(this, typeof<Build>) with
@@ -70,7 +83,7 @@ with
         Targets.Target(target.StepName, Action(fun _ -> action(parsed)))
 
     static member Cmd (dependsOn: Build list) (composedOf: Build list) action (target: Build) (parsed: ParseResults<Build>) =
-        let singleTarget = parsed.TryGetResult SingleTarget |> Option.isSome
+        let singleTarget = parsed.TryGetResult Single_Target |> Option.isSome
         let dependsOn = if singleTarget then [] else dependsOn
 
         let steps = dependsOn @ composedOf |> List.map (_.StepName)

build/scripts/Targets.fs

@@ -34,31 +34,53 @@ let private version _ =
 let private generatePackages _ = exec { run "dotnet" "pack" }
 
 let private pristineCheck (arguments:ParseResults<Build>) =
-    let skipCheck = arguments.TryGetResult SkipDirtyCheck |> Option.isSome
+    let skipCheck = arguments.TryGetResult Skip_Dirty_Check |> Option.isSome
     match skipCheck, Information.isCleanWorkingCopy "." with
     | true, _ -> printfn "Checkout is dirty but -c was specified to ignore this"
     | _, true  -> printfn "The checkout folder does not have pending changes, proceeding"
     | _ -> failwithf "The checkout folder has pending changes, aborting. Specify -c to ./build.sh to skip this check"
 
-let private runTests _ =
-    let testOutputPath = Paths.ArtifactPath "tests"
-    let junitOutput = Path.Combine(testOutputPath.FullName, "junit-{assembly}-{framework}-test-results.xml")
-    let loggerPathArgs = $"LogFilePath=%s{junitOutput}"
-    let loggerArg = $"--logger:\"junit;%s{loggerPathArgs}\""
-    let githubActionsLogger = $"--logger:\"GitHubActions;summary.includePassedTests=true\""
+let private runTests suite _ =
+    let logger =
+        // use junit xml logging locally, github actions logs using console out formats
+        match BuildServer.isGitHubActionsBuild with
+        | true -> "--logger:\"GitHubActions;summary.includePassedTests=false\""
+        | false ->
+            let testOutputPath = Paths.ArtifactPath "tests"
+            let junitOutput = Path.Combine(testOutputPath.FullName, "junit-{assembly}-{framework}-test-results.xml")
+            let loggerPathArgs = $"LogFilePath=%s{junitOutput}"
+            $"--logger:\"junit;%s{loggerPathArgs}\""
+    let filterArgs =
+        match suite with
+        | All -> []
+        | Skip_All -> ["--filter"; "FullyQualifiedName~.SKIPPING.ALL.TESTS"]
+        | Unit ->  [ "--filter"; "FullyQualifiedName~.Tests" ]
+        | Integration -> [ "--filter"; "FullyQualifiedName~.IntegrationTests" ]
+        | E2E -> [ "--filter"; "FullyQualifiedName~.EndToEndTests" ]
+        | Skip_E2E -> [ "--filter"; "FullyQualifiedName!~.EndToEndTests" ]
+
+
+    let settingsArg = ["-s"; "tests/.runsettings"]
     let tfmArgs = if OS.Current = OS.Windows then [] else ["-f"; "net8.0"]
     exec {
+        env (Map ["TEST_SUITE", suite.SuitName])
         run "dotnet" (
-            ["test"; "-c"; "release"; loggerArg; githubActionsLogger]
+            ["test"; "-c"; "release"; "--no-restore"; "--no-build"; logger]
+            @ settingsArg
+            @ filterArgs
             @ tfmArgs
             @ ["--"; "RunConfiguration.CollectSourceInformation=true"]
         )
     }
 
 let private test (arguments:ParseResults<Build>) =
-    match arguments.TryGetResult SkipTests with
-    | Some _ -> runTests arguments
-    | None -> printfn "Skipping tests because --skiptests was provided"
+    let arg = arguments.TryGetResult Test_Suite
+    match arg with
+    | None -> runTests TestSuite.All arguments 
+    | Some suite ->
+        match suite with
+        | Skip_All -> printfn "Skipping tests because --test-suite skip was provided"
+        | _ -> runTests suite arguments   
 
 let private validateLicenses _ =
     let args = ["-u"; "-t"; "-i"; "Elastic.OpenTelemetry.sln"; "--use-project-assets-json"
@@ -131,7 +153,11 @@ let Setup (parsed:ParseResults<Build>) =
         | Version -> Build.Step version
         | Clean -> Build.Cmd [Version] [] clean
         | Build -> Build.Cmd [Clean] [] build
-        | Test -> Build.Cmd [Build] [] test
+
+        | End_To_End -> Build.Cmd [] [Build] <| runTests E2E
+        | Unit_Test -> Build.Cmd [] [Build] <| runTests Unit
+        | Test -> Build.Cmd [] [Build] test
+
         | Release -> 
             Build.Cmd 
                 [PristineCheck; Test]
@@ -147,10 +173,10 @@ let Setup (parsed:ParseResults<Build>) =
         | GenerateApiChanges -> Build.Step generateApiChanges
 
         // flags
-        | SingleTarget
-        | SkipTests
+        | Single_Target
+        | Test_Suite _
         | Token _
-        | SkipDirtyCheck -> Build.Ignore
+        | Skip_Dirty_Check -> Build.Ignore
 
     for target in Build.Targets do
         let setup = wireCommandLine target