Skip to content

Commit

Permalink
Add End to End testing (matrix-org#868)
Browse files Browse the repository at this point in the history
* Ensure connection state always explicitly states all keys, even if some are undefined.

* changelog

* Fix type

* fix test types

* Add support for E2E testing

* Add CI job for e2e test

* Ensure integration test only runs when regular tests complete

* Add homerunner image

* Disallow concurrent runs

* Add concurrency to other expensive steps

* changelog

* Fix mq test

* Cache rust deps

* Drop only

* Use a shared key
  • Loading branch information
Half-Shot authored Dec 28, 2023
1 parent 46d198c commit 8e115b4
Show file tree
Hide file tree
Showing 14 changed files with 1,927 additions and 78 deletions.
5 changes: 5 additions & 0 deletions .github/workflows/docker-hub-latest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,11 @@ on:
- changelog.d/**'
merge_group:


concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

env:
DOCKER_NAMESPACE: halfshot
PLATFORMS: linux/amd64
Expand Down
4 changes: 4 additions & 0 deletions .github/workflows/docker-hub-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@ env:
DOCKER_NAMESPACE: halfshot
PLATFORMS: linux/amd64,linux/arm64

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

jobs:
docker-release:
runs-on: ubuntu-latest
Expand Down
5 changes: 5 additions & 0 deletions .github/workflows/docs-latest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,11 @@ on:
push:
paths-ignore:
- changelog.d/**'

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

jobs:
deploy:
runs-on: ubuntu-20.04
Expand Down
4 changes: 4 additions & 0 deletions .github/workflows/docs-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@ on:
release:
types: [published]

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

jobs:
deploy:
runs-on: ubuntu-20.04
Expand Down
83 changes: 83 additions & 0 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,11 @@ on:
workflow_dispatch:
merge_group:


concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

jobs:
lint-node:
runs-on: ubuntu-latest
Expand Down Expand Up @@ -75,5 +80,83 @@ jobs:
with:
toolchain: stable
profile: minimal
- uses: Swatinem/rust-cache@v2
with:
shared-key: rust-cache
- run: yarn
- run: yarn test:cover

build-homerunner:
runs-on: ubuntu-latest
outputs:
homerunnersha: ${{ steps.gitsha.outputs.sha }}
steps:
- name: Checkout matrix-org/complement
uses: actions/checkout@v3
with:
repository: matrix-org/complement
- name: Get complement git sha
id: gitsha
run: echo sha=`git rev-parse --short HEAD` >> "$GITHUB_OUTPUT"
- name: Cache homerunner
id: cached
uses: actions/cache@v3
with:
path: homerunner
key: ${{ runner.os }}-homerunner-${{ steps.gitsha.outputs.sha }}
- name: "Set Go Version"
if: ${{ steps.cached.outputs.cache-hit != 'true' }}
run: |
echo "$GOROOT_1_18_X64/bin" >> $GITHUB_PATH
echo "~/go/bin" >> $GITHUB_PATH
# Build and install homerunner
- name: Install Complement Dependencies
if: ${{ steps.cached.outputs.cache-hit != 'true' }}
run: |
sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
- name: Build homerunner
if: ${{ steps.cached.outputs.cache-hit != 'true' }}
run: |
go build ./cmd/homerunner
integration-test:
runs-on: ubuntu-latest
timeout-minutes: 30
needs:
- test
- build-homerunner
steps:
- name: Install Complement Dependencies
run: |
sudo apt-get update && sudo apt-get install -y libolm3
- name: Load cached homerunner bin
uses: actions/cache@v3
with:
path: homerunner
key: ${{ runner.os }}-homerunner-${{ needs.build-synapse.outputs.homerunnersha }}
fail-on-cache-miss: true # Shouldn't happen, we build this in the needs step.
- name: Checkout matrix-hookshot
uses: actions/checkout@v3
with:
path: matrix-hookshot
# Setup node & run tests
- name: Use Node.js
uses: actions/setup-node@v3
with:
node-version-file: matrix-hookshot/.node-version
- uses: Swatinem/rust-cache@v2
with:
workspaces: matrix-hookshot
shared-key: rust-cache
- name: Run Homerunner tests
timeout-minutes: 10
env:
HOMERUNNER_SPAWN_HS_TIMEOUT_SECS: 100
HOMERUNNER_IMAGE: ghcr.io/element-hq/synapse/complement-synapse:latest
NODE_OPTIONS: --dns-result-order ipv4first
run: |
docker pull $HOMERUNNER_IMAGE
cd matrix-hookshot
yarn --strict-semver --frozen-lockfile
../homerunner &
bash -ic 'yarn test:e2e'
1 change: 1 addition & 0 deletions changelog.d/869.misc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Integrate end to end testing.
21 changes: 21 additions & 0 deletions jest.config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
/**
* For a detailed explanation regarding each configuration property, visit:
* https://jestjs.io/docs/configuration
*/

import type {Config} from 'jest';

const config: Config = {
// The root directory that Jest should scan for tests and modules within
rootDir: "spec",
testTimeout: 60000,
transform: {
'^.+\\.tsx?$': [
'ts-jest',
{
},
],
},
};

export default config;
10 changes: 7 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
"start:matrixsender": "node --require source-map-support/register lib/App/MatrixSenderApp.js",
"start:resetcrypto": "node --require source-map-support/register lib/App/ResetCryptoStore.js",
"test": "mocha -r ts-node/register tests/init.ts tests/*.ts tests/**/*.ts",
"test:e2e": "mocha -r ts-node/register tests/init.ts tests/*.ts tests/**/*.ts",
"test:e2e": "yarn node --experimental-vm-modules $(yarn bin jest)",
"test:cover": "nyc --reporter=lcov --reporter=text yarn test",
"lint": "yarn run lint:js && yarn run lint:rs",
"lint:js": "eslint -c .eslintrc.js 'src/**/*.ts' 'tests/**/*.ts' 'web/**/*.ts' 'web/**/*.tsx'",
Expand Down Expand Up @@ -60,7 +60,7 @@
"jira-client": "^8.2.2",
"markdown-it": "^14.0.0",
"matrix-appservice-bridge": "^9.0.1",
"matrix-bot-sdk": "npm:@vector-im/matrix-bot-sdk@^0.6.7-element.1",
"matrix-bot-sdk": "npm:@vector-im/matrix-bot-sdk@^0.7.0-specific-device-2",
"matrix-widget-api": "^1.6.0",
"micromatch": "^4.0.5",
"mime": "^4.0.1",
Expand Down Expand Up @@ -88,6 +88,7 @@
"@types/chai": "^4.2.22",
"@types/cors": "^2.8.12",
"@types/express": "^4.17.14",
"@types/jest": "^29.5.11",
"@types/jira-client": "^7.1.0",
"@types/markdown-it": "^13.0.7",
"@types/micromatch": "^4.0.1",
Expand All @@ -102,14 +103,17 @@
"eslint": "^8.49.0",
"eslint-config-preact": "^1.3.0",
"eslint-plugin-mocha": "^10.1.0",
"homerunner-client": "^1.0.0",
"jest": "^29.7.0",
"mini.css": "^3.0.1",
"mocha": "^10.2.0",
"nyc": "^15.1.0",
"preact": "^10.5.15",
"rimraf": "^5.0.5",
"sass": "^1.51.0",
"ts-jest": "^29.1.1",
"ts-node": "^10.9.1",
"typescript": "^5.1.3",
"typescript": "^5.3.3",
"vite": "^5.0.10"
}
}
62 changes: 62 additions & 0 deletions spec/basic.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
import { MessageEventContent } from "matrix-bot-sdk";
import { E2ESetupTestTimeout, E2ETestEnv } from "./util/e2e-test";
import { describe, it, beforeEach, afterEach } from "@jest/globals";
import { expect } from "chai";

describe('Basic test setup', () => {
let testEnv: E2ETestEnv;

beforeEach(async () => {
testEnv = await E2ETestEnv.createTestEnv({matrixLocalparts: ['user']});
await testEnv.setUp();
}, E2ESetupTestTimeout);

afterEach(() => {
return testEnv?.tearDown();
});

it('should be able to invite the bot to a room', async () => {
const user = testEnv.getUser('user');
const roomId = await user.createRoom({ name: 'Test room', invite:[testEnv.botMxid] });
await user.waitForRoomJoin({sender: testEnv.botMxid, roomId });
await user.sendText(roomId, "!hookshot help");
const msg = await user.waitForRoomEvent<MessageEventContent>({
eventType: 'm.room.message', sender: testEnv.botMxid, roomId
});
// Expect help text.
expect(msg.data.content.body).to.include('!hookshot help` - This help text\n');
});

// TODO: Move test to it's own generic connections file.
it('should be able to setup a webhook', async () => {
const user = testEnv.getUser('user');
const testRoomId = await user.createRoom({ name: 'Test room', invite:[testEnv.botMxid] });
await user.waitForRoomJoin({sender: testEnv.botMxid, roomId: testRoomId });
await user.setUserPowerLevel(testEnv.botMxid, testRoomId, 50);
await user.sendText(testRoomId, "!hookshot webhook test-webhook");
const inviteResponse = await user.waitForRoomInvite({sender: testEnv.botMxid});
await user.waitForRoomEvent<MessageEventContent>({
eventType: 'm.room.message', sender: testEnv.botMxid, roomId: testRoomId,
body: 'Room configured to bridge webhooks. See admin room for secret url.'
});
const webhookUrlMessage = user.waitForRoomEvent<MessageEventContent>({
eventType: 'm.room.message', sender: testEnv.botMxid, roomId: inviteResponse.roomId
});
await user.joinRoom(inviteResponse.roomId);
const msgData = (await webhookUrlMessage).data.content.body;
const webhookUrl = msgData.split('\n')[2];
const webhookNotice = user.waitForRoomEvent<MessageEventContent>({
eventType: 'm.room.message', sender: testEnv.botMxid, roomId: testRoomId, body: 'Hello world!'
});

// Send a webhook
await fetch(webhookUrl, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({text: 'Hello world!'})
});

// And await the notice.
await webhookNotice;
});
});
Loading

0 comments on commit 8e115b4

Please sign in to comment.