diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.md b/.github/ISSUE_TEMPLATE/BUG_REPORT.md index a3b8c0754a..e18968baf1 100644 --- a/.github/ISSUE_TEMPLATE/BUG_REPORT.md +++ b/.github/ISSUE_TEMPLATE/BUG_REPORT.md @@ -62,6 +62,6 @@ If you can identify any relevant log snippets from server logs, please include those (please be careful to remove any personal or private data). Please surround them with ``` (three backticks, on a line on their own), so that they are formatted legibly. -Alternatively, please send logs to @kegan:matrix.org or @neilalexander:matrix.org +Alternatively, please send logs to @kegan:matrix.org, @s7evink:matrix.org or @devonh:one.ems.host with a link to the respective Github issue, thanks! --> diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index 772b45cb24..ac40f06b00 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -440,7 +440,7 @@ jobs: # Run Complement - run: | set -o pipefail && - go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt -hide all + go test -v -json -tags dendrite_blacklist ./tests ./tests/csapi 2>&1 | gotestfmt -hide all shell: bash name: Run Complement Tests env: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8448d8e23c..8d3a8d674d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -32,10 +32,6 @@ jobs: if: github.event_name == 'release' # Only for GitHub releases run: | echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV - BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) - [ ${BRANCH} == "main" ] && BRANCH="" - echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx @@ -60,7 +56,6 @@ jobs: cache-from: type=registry,ref=ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:buildcache cache-to: type=registry,ref=ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:buildcache,mode=max context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} platforms: ${{ env.PLATFORMS }} push: true tags: | @@ -75,7 +70,6 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} platforms: ${{ env.PLATFORMS }} push: true tags: | @@ -109,10 +103,6 @@ jobs: if: github.event_name == 'release' # Only for GitHub releases run: | echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV - BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) - [ ${BRANCH} == "main" ] && BRANCH="" - echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx @@ -137,7 +127,6 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} file: ./build/docker/Dockerfile.demo-pinecone platforms: ${{ env.PLATFORMS }} push: true @@ -153,7 +142,6 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} file: ./build/docker/Dockerfile.demo-pinecone platforms: ${{ env.PLATFORMS }} push: true @@ -176,10 +164,6 @@ jobs: if: github.event_name == 'release' # Only for GitHub releases run: | echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV - BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) - [ ${BRANCH} == "main" ] && BRANCH="" - echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx @@ -204,7 +188,6 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} file: ./build/docker/Dockerfile.demo-yggdrasil platforms: ${{ env.PLATFORMS }} push: true @@ -220,7 +203,6 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} file: ./build/docker/Dockerfile.demo-yggdrasil platforms: ${{ env.PLATFORMS }} push: true diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index bf62a1c199..9a5eb2b62f 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -32,7 +32,7 @@ jobs: version: v3.10.0 - name: Run chart-releaser - uses: helm/chart-releaser-action@v1.4.1 + uses: helm/chart-releaser-action@ed43eb303604cbc0eeec8390544f7748dc6c790d # specific commit, since `mark_as_latest` is not yet in a release env: CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" with: diff --git a/.golangci.yml b/.golangci.yml index bb8d38a8bd..5bee0a8858 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -180,7 +180,6 @@ linters-settings: linters: enable: - errcheck - - goconst - gocyclo - goimports # Does everything gofmt does - gosimple @@ -211,6 +210,7 @@ linters: - stylecheck - typecheck # Should turn back on soon - unconvert # Should turn back on soon + - goconst # Slightly annoying, as it reports "issues" in SQL statements disable-all: false presets: fast: false diff --git a/CHANGES.md b/CHANGES.md index f4a8145661..57e3a3d4f0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,42 @@ # Changelog +## Dendrite 0.13.4 (2023-10-25) + +Upgrading to this version is **highly** recommended, as it fixes a long-standing bug in the state resolution +algorithm. + +### Fixes: + +- The "device list updater" now de-duplicates the servers to fetch devices from on startup. (This also + avoids spamming the logs when shutting down.) +- A bug in the state resolution algorithm has been fixed. This bug could result in users "being reset" + out of rooms and other missing state events due to calculating the wrong state. +- A bug when setting notifications from Element Android has been fixed by implementing MSC3987 + +### Features + +- Updated dependencies + - Internal NATS Server has been updated from v2.9.19 to v2.9.23 + +## Dendrite 0.13.3 (2023-09-28) + +### Fixes: + +- The `user_id` query parameter when authenticating is now used correctly (contributed by [tulir](https://github.com/tulir)) +- Invitations are now correctly pushed to devices +- A bug which could result in the corruption of `m.direct` account data has been fixed + +### Features + +- [Sliding Sync proxy](https://github.com/matrix-org/sliding-sync) can be configured in the `/.well-known/matrix/client` response +- Room version 11 is now supported +- Clients can request the `federation` `event_format` when creating filters +- Many under the hood improvements for [MSC4014: Pseudonymous Identities](https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/pseudo-ids/proposals/4014-pseudonymous-identities.md) + +### Other + +- Dendrite now requires Go 1.20 if building from source + ## Dendrite 0.13.2 (2023-08-23) ### Fixes: diff --git a/Dockerfile b/Dockerfile index 4ee20933a9..8c8f1588ff 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ # # base installs required dependencies and runs go mod download to cache dependencies # -FROM --platform=${BUILDPLATFORM} docker.io/golang:1.20-alpine AS base +FROM --platform=${BUILDPLATFORM} docker.io/golang:1.21-alpine AS base RUN apk --update --no-cache add bash build-base curl git # @@ -13,7 +13,6 @@ FROM --platform=${BUILDPLATFORM} base AS build WORKDIR /src ARG TARGETOS ARG TARGETARCH -ARG FLAGS RUN --mount=target=. \ --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ @@ -21,7 +20,7 @@ RUN --mount=target=. \ GOARCH="$TARGETARCH" \ GOOS="linux" \ CGO_ENABLED=$([ "$TARGETARCH" = "$USERARCH" ] && echo "1" || echo "0") \ - go build -v -ldflags="${FLAGS}" -trimpath -o /out/ ./cmd/... + go build -v -trimpath -o /out/ ./cmd/... # diff --git a/README.md b/README.md index 34604eff93..bde19b07ee 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ If you have further questions, please take a look at [our FAQ](docs/FAQ.md) or j See the [Planning your Installation](https://matrix-org.github.io/dendrite/installation/planning) page for more information on requirements. -To build Dendrite, you will need Go 1.18 or later. +To build Dendrite, you will need Go 1.20 or later. For a usable federating Dendrite deployment, you will also need: diff --git a/appservice/appservice_test.go b/appservice/appservice_test.go index ddc24477bf..eca63371d1 100644 --- a/appservice/appservice_test.go +++ b/appservice/appservice_test.go @@ -14,6 +14,7 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/stretchr/testify/assert" "github.com/matrix-org/dendrite/appservice" @@ -32,6 +33,10 @@ import ( "github.com/matrix-org/dendrite/test/testrig" ) +var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) { + return &statistics.ServerStatistics{}, nil +} + func TestAppserviceInternalAPI(t *testing.T) { // Set expected results @@ -144,7 +149,7 @@ func TestAppserviceInternalAPI(t *testing.T) { cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil) + usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI) runCases(t, asAPI) @@ -239,7 +244,7 @@ func TestAppserviceInternalAPI_UnixSocket_Simple(t *testing.T) { cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil) + usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI) t.Run("UserIDExists", func(t *testing.T) { @@ -378,7 +383,7 @@ func TestRoomserverConsumerOneInvite(t *testing.T) { // Create required internal APIs rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil) + usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // start the consumer appservice.NewInternalAPI(processCtx, cfg, natsInstance, usrAPI, rsAPI) diff --git a/are-we-synapse-yet.list b/are-we-synapse-yet.list index 585374738d..80c0dbff6d 100644 --- a/are-we-synapse-yet.list +++ b/are-we-synapse-yet.list @@ -944,4 +944,12 @@ rmv remote user can join room with version 10 rmv User can invite remote user to room with version 10 rmv Remote user can backfill in a room with version 10 rmv Can reject invites over federation for rooms with version 10 -rmv Can receive redactions from regular users over federation in room version 10 \ No newline at end of file +rmv Can receive redactions from regular users over federation in room version 10 +rmv User can create and send/receive messages in a room with version 11 +rmv local user can join room with version 11 +rmv User can invite local user to room with version 11 +rmv remote user can join room with version 11 +rmv User can invite remote user to room with version 11 +rmv Remote user can backfill in a room with version 11 +rmv Can reject invites over federation for rooms with version 11 +rmv Can receive redactions from regular users over federation in room version 11 \ No newline at end of file diff --git a/build/dendritejs-pinecone/main.go b/build/dendritejs-pinecone/main.go index 61baed902a..6acc93c7b2 100644 --- a/build/dendritejs-pinecone/main.go +++ b/build/dendritejs-pinecone/main.go @@ -38,6 +38,7 @@ import ( "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/userapi" + "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/gomatrixserverlib" @@ -190,13 +191,13 @@ func startup() { serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation) + fedSenderAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics, fedSenderAPI.IsBlacklistedOrBackingOff) asQuery := appservice.NewInternalAPI( processCtx, cfg, &natsInstance, userAPI, rsAPI, ) rsAPI.SetAppserviceAPI(asQuery) - fedSenderAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true) rsAPI.SetFederationAPI(fedSenderAPI, keyRing) monolith := setup.Monolith{ diff --git a/build/docker/Dockerfile.demo-yggdrasil b/build/docker/Dockerfile.demo-yggdrasil index efae5496c0..b9e387666a 100644 --- a/build/docker/Dockerfile.demo-yggdrasil +++ b/build/docker/Dockerfile.demo-yggdrasil @@ -1,4 +1,4 @@ -FROM docker.io/golang:1.19-alpine AS base +FROM docker.io/golang:1.21-alpine AS base # # Needs to be separate from the main Dockerfile for OpenShift, diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go index 720ce37eb0..2b227d3736 100644 --- a/build/gobind-yggdrasil/monolith.go +++ b/build/gobind-yggdrasil/monolith.go @@ -216,7 +216,7 @@ func (m *DendriteMonolith) Start() { processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true, ) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics, fsAPI.IsBlacklistedOrBackingOff) asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) diff --git a/build/scripts/complement.sh b/build/scripts/complement.sh index 29feff304f..8608d8fa56 100755 --- a/build/scripts/complement.sh +++ b/build/scripts/complement.sh @@ -15,5 +15,5 @@ tar -xzf master.tar.gz # Run the tests! cd complement-master -COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -v -count=1 ./tests +COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -v -count=1 ./tests ./tests/csapi diff --git a/clientapi/admin_test.go b/clientapi/admin_test.go index 66667b03ca..f0e5f004d2 100644 --- a/clientapi/admin_test.go +++ b/clientapi/admin_test.go @@ -45,7 +45,7 @@ func TestAdminCreateToken(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ aliceAdmin: {}, @@ -196,7 +196,7 @@ func TestAdminListRegistrationTokens(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ aliceAdmin: {}, @@ -314,7 +314,7 @@ func TestAdminGetRegistrationToken(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ aliceAdmin: {}, @@ -415,7 +415,7 @@ func TestAdminDeleteRegistrationToken(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ aliceAdmin: {}, @@ -509,7 +509,7 @@ func TestAdminUpdateRegistrationToken(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ aliceAdmin: {}, @@ -693,7 +693,7 @@ func TestAdminResetPassword(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) // Needed for changing the password/login - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the userAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -791,7 +791,7 @@ func TestPurgeRoom(t *testing.T) { fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics) // Create the room @@ -863,7 +863,7 @@ func TestAdminEvacuateRoom(t *testing.T) { fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // Create the room if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil { @@ -964,7 +964,7 @@ func TestAdminEvacuateUser(t *testing.T) { fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, basepkg.CreateFederationClient(cfg, nil), rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // Create the room if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil { @@ -1055,7 +1055,7 @@ func TestAdminMarkAsStale(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) diff --git a/clientapi/auth/login.go b/clientapi/auth/login.go index 77835614ea..58a27e5932 100644 --- a/clientapi/auth/login.go +++ b/clientapi/auth/login.go @@ -15,7 +15,6 @@ package auth import ( - "context" "encoding/json" "io" "net/http" @@ -32,8 +31,13 @@ import ( // called after authorization has completed, with the result of the authorization. // If the final return value is non-nil, an error occurred and the cleanup function // is nil. -func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.UserLoginAPI, userAPI UserInternalAPIForLogin, cfg *config.ClientAPI) (*Login, LoginCleanupFunc, *util.JSONResponse) { - reqBytes, err := io.ReadAll(r) +func LoginFromJSONReader( + req *http.Request, + useraccountAPI uapi.UserLoginAPI, + userAPI UserInternalAPIForLogin, + cfg *config.ClientAPI, +) (*Login, LoginCleanupFunc, *util.JSONResponse) { + reqBytes, err := io.ReadAll(req.Body) if err != nil { err := &util.JSONResponse{ Code: http.StatusBadRequest, @@ -65,6 +69,20 @@ func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.U UserAPI: userAPI, Config: cfg, } + case authtypes.LoginTypeApplicationService: + token, err := ExtractAccessToken(req) + if err != nil { + err := &util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.MissingToken(err.Error()), + } + return nil, nil, err + } + + typ = &LoginTypeApplicationService{ + Config: cfg, + Token: token, + } default: err := util.JSONResponse{ Code: http.StatusBadRequest, @@ -73,7 +91,7 @@ func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.U return nil, nil, &err } - return typ.LoginFromJSON(ctx, reqBytes) + return typ.LoginFromJSON(req.Context(), reqBytes) } // UserInternalAPIForLogin contains the aspects of UserAPI required for logging in. diff --git a/clientapi/auth/login_application_service.go b/clientapi/auth/login_application_service.go new file mode 100644 index 0000000000..dd4a9cbb48 --- /dev/null +++ b/clientapi/auth/login_application_service.go @@ -0,0 +1,55 @@ +// Copyright 2023 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package auth + +import ( + "context" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/util" +) + +// LoginTypeApplicationService describes how to authenticate as an +// application service +type LoginTypeApplicationService struct { + Config *config.ClientAPI + Token string +} + +// Name implements Type +func (t *LoginTypeApplicationService) Name() string { + return authtypes.LoginTypeApplicationService +} + +// LoginFromJSON implements Type +func (t *LoginTypeApplicationService) LoginFromJSON( + ctx context.Context, reqBytes []byte, +) (*Login, LoginCleanupFunc, *util.JSONResponse) { + var r Login + if err := httputil.UnmarshalJSON(reqBytes, &r); err != nil { + return nil, nil, err + } + + _, err := internal.ValidateApplicationServiceRequest(t.Config, r.Identifier.User, t.Token) + if err != nil { + return nil, nil, err + } + + cleanup := func(ctx context.Context, j *util.JSONResponse) {} + return &r, cleanup, nil +} diff --git a/clientapi/auth/login_test.go b/clientapi/auth/login_test.go index 93d3e2713a..a2c2a719c1 100644 --- a/clientapi/auth/login_test.go +++ b/clientapi/auth/login_test.go @@ -17,7 +17,9 @@ package auth import ( "context" "net/http" + "net/http/httptest" "reflect" + "regexp" "strings" "testing" @@ -33,8 +35,9 @@ func TestLoginFromJSONReader(t *testing.T) { ctx := context.Background() tsts := []struct { - Name string - Body string + Name string + Body string + Token string WantUsername string WantDeviceID string @@ -62,6 +65,30 @@ func TestLoginFromJSONReader(t *testing.T) { WantDeviceID: "adevice", WantDeletedTokens: []string{"atoken"}, }, + { + Name: "appServiceWorksUserID", + Body: `{ + "type": "m.login.application_service", + "identifier": { "type": "m.id.user", "user": "@alice:example.com" }, + "device_id": "adevice" + }`, + Token: "astoken", + + WantUsername: "@alice:example.com", + WantDeviceID: "adevice", + }, + { + Name: "appServiceWorksLocalpart", + Body: `{ + "type": "m.login.application_service", + "identifier": { "type": "m.id.user", "user": "alice" }, + "device_id": "adevice" + }`, + Token: "astoken", + + WantUsername: "alice", + WantDeviceID: "adevice", + }, } for _, tst := range tsts { t.Run(tst.Name, func(t *testing.T) { @@ -72,11 +99,35 @@ func TestLoginFromJSONReader(t *testing.T) { ServerName: serverName, }, }, + Derived: &config.Derived{ + ApplicationServices: []config.ApplicationService{ + { + ID: "anapplicationservice", + ASToken: "astoken", + NamespaceMap: map[string][]config.ApplicationServiceNamespace{ + "users": { + { + Exclusive: true, + Regex: "@alice:example.com", + RegexpObject: regexp.MustCompile("@alice:example.com"), + }, + }, + }, + }, + }, + }, } - login, cleanup, err := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg) - if err != nil { - t.Fatalf("LoginFromJSONReader failed: %+v", err) + + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(tst.Body)) + if tst.Token != "" { + req.Header.Add("Authorization", "Bearer "+tst.Token) } + + login, cleanup, jsonErr := LoginFromJSONReader(req, &userAPI, &userAPI, cfg) + if jsonErr != nil { + t.Fatalf("LoginFromJSONReader failed: %+v", jsonErr) + } + cleanup(ctx, &util.JSONResponse{Code: http.StatusOK}) if login.Username() != tst.WantUsername { @@ -104,8 +155,9 @@ func TestBadLoginFromJSONReader(t *testing.T) { ctx := context.Background() tsts := []struct { - Name string - Body string + Name string + Body string + Token string WantErrCode spec.MatrixErrorCode }{ @@ -142,6 +194,45 @@ func TestBadLoginFromJSONReader(t *testing.T) { }`, WantErrCode: spec.ErrorInvalidParam, }, + { + Name: "noASToken", + Body: `{ + "type": "m.login.application_service", + "identifier": { "type": "m.id.user", "user": "@alice:example.com" }, + "device_id": "adevice" + }`, + WantErrCode: "M_MISSING_TOKEN", + }, + { + Name: "badASToken", + Token: "badastoken", + Body: `{ + "type": "m.login.application_service", + "identifier": { "type": "m.id.user", "user": "@alice:example.com" }, + "device_id": "adevice" + }`, + WantErrCode: "M_UNKNOWN_TOKEN", + }, + { + Name: "badASNamespace", + Token: "astoken", + Body: `{ + "type": "m.login.application_service", + "identifier": { "type": "m.id.user", "user": "@bob:example.com" }, + "device_id": "adevice" + }`, + WantErrCode: "M_EXCLUSIVE", + }, + { + Name: "badASUserID", + Token: "astoken", + Body: `{ + "type": "m.login.application_service", + "identifier": { "type": "m.id.user", "user": "@alice:wrong.example.com" }, + "device_id": "adevice" + }`, + WantErrCode: "M_INVALID_USERNAME", + }, } for _, tst := range tsts { t.Run(tst.Name, func(t *testing.T) { @@ -152,8 +243,30 @@ func TestBadLoginFromJSONReader(t *testing.T) { ServerName: serverName, }, }, + Derived: &config.Derived{ + ApplicationServices: []config.ApplicationService{ + { + ID: "anapplicationservice", + ASToken: "astoken", + NamespaceMap: map[string][]config.ApplicationServiceNamespace{ + "users": { + { + Exclusive: true, + Regex: "@alice:example.com", + RegexpObject: regexp.MustCompile("@alice:example.com"), + }, + }, + }, + }, + }, + }, } - _, cleanup, errRes := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg) + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(tst.Body)) + if tst.Token != "" { + req.Header.Add("Authorization", "Bearer "+tst.Token) + } + + _, cleanup, errRes := LoginFromJSONReader(req, &userAPI, &userAPI, cfg) if errRes == nil { cleanup(ctx, nil) t.Fatalf("LoginFromJSONReader err: got %+v, want code %q", errRes, tst.WantErrCode) diff --git a/clientapi/auth/user_interactive.go b/clientapi/auth/user_interactive.go index 92d83ad291..9831450ccd 100644 --- a/clientapi/auth/user_interactive.go +++ b/clientapi/auth/user_interactive.go @@ -55,7 +55,7 @@ type LoginCleanupFunc func(context.Context, *util.JSONResponse) // https://matrix.org/docs/spec/client_server/r0.6.1#identifier-types type LoginIdentifier struct { Type string `json:"type"` - // when type = m.id.user + // when type = m.id.user or m.id.application_service User string `json:"user"` // when type = m.id.thirdparty Medium string `json:"medium"` diff --git a/clientapi/clientapi_test.go b/clientapi/clientapi_test.go index 82ec9fea2b..2ff4b6503b 100644 --- a/clientapi/clientapi_test.go +++ b/clientapi/clientapi_test.go @@ -17,6 +17,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/routing" "github.com/matrix-org/dendrite/clientapi/threepid" + "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/pushrules" @@ -49,6 +50,10 @@ type userDevice struct { password string } +var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) { + return &statistics.ServerStatistics{}, nil +} + func TestGetPutDevices(t *testing.T) { alice := test.NewUser(t) bob := test.NewUser(t) @@ -121,7 +126,7 @@ func TestGetPutDevices(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -170,7 +175,7 @@ func TestDeleteDevice(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -275,7 +280,7 @@ func TestDeleteDevices(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -442,7 +447,7 @@ func TestSetDisplayname(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) asPI := appservice.NewInternalAPI(processCtx, cfg, natsInstance, userAPI, rsAPI) AddPublicRoutes(processCtx, routers, cfg, natsInstance, base.CreateFederationClient(cfg, nil), rsAPI, asPI, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -554,7 +559,7 @@ func TestSetAvatarURL(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) asPI := appservice.NewInternalAPI(processCtx, cfg, natsInstance, userAPI, rsAPI) AddPublicRoutes(processCtx, routers, cfg, natsInstance, base.CreateFederationClient(cfg, nil), rsAPI, asPI, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -632,7 +637,7 @@ func TestTyping(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) // Needed to create accounts - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -716,7 +721,7 @@ func TestMembership(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) // Needed to create accounts - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) rsAPI.SetUserAPI(userAPI) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -955,7 +960,7 @@ func TestCapabilities(t *testing.T) { // Needed to create accounts rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -1002,7 +1007,7 @@ func TestTurnserver(t *testing.T) { // Needed to create accounts rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) //rsAPI.SetUserAPI(userAPI) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -1100,7 +1105,7 @@ func Test3PID(t *testing.T) { // Needed to create accounts rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -1276,7 +1281,7 @@ func TestPushRules(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -1418,7 +1423,7 @@ func TestPushRules(t *testing.T) { validateFunc: func(t *testing.T, respBody *bytes.Buffer) { actions := gjson.GetBytes(respBody.Bytes(), "actions").Array() // only a basic check - assert.Equal(t, 1, len(actions)) + assert.Equal(t, 0, len(actions)) }, }, { @@ -1663,7 +1668,7 @@ func TestKeys(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -2125,7 +2130,7 @@ func TestKeyBackup(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) diff --git a/clientapi/routing/joinroom_test.go b/clientapi/routing/joinroom_test.go index 933ea8d3af..bd854efa89 100644 --- a/clientapi/routing/joinroom_test.go +++ b/clientapi/routing/joinroom_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/jetstream" @@ -21,6 +22,10 @@ import ( uapi "github.com/matrix-org/dendrite/userapi/api" ) +var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) { + return &statistics.ServerStatistics{}, nil +} + func TestJoinRoomByIDOrAlias(t *testing.T) { alice := test.NewUser(t) bob := test.NewUser(t) @@ -36,7 +41,7 @@ func TestJoinRoomByIDOrAlias(t *testing.T) { natsInstance := jetstream.NATSInstance{} rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) // Create the users in the userapi diff --git a/clientapi/routing/login.go b/clientapi/routing/login.go index bc38b83406..0f55c88165 100644 --- a/clientapi/routing/login.go +++ b/clientapi/routing/login.go @@ -19,6 +19,7 @@ import ( "net/http" "github.com/matrix-org/dendrite/clientapi/auth" + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/userutil" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -40,28 +41,25 @@ type flow struct { Type string `json:"type"` } -func passwordLogin() flows { - f := flows{} - s := flow{ - Type: "m.login.password", - } - f.Flows = append(f.Flows, s) - return f -} - // Login implements GET and POST /login func Login( req *http.Request, userAPI userapi.ClientUserAPI, cfg *config.ClientAPI, ) util.JSONResponse { if req.Method == http.MethodGet { - // TODO: support other forms of login other than password, depending on config options + loginFlows := []flow{{Type: authtypes.LoginTypePassword}} + if len(cfg.Derived.ApplicationServices) > 0 { + loginFlows = append(loginFlows, flow{Type: authtypes.LoginTypeApplicationService}) + } + // TODO: support other forms of login, depending on config options return util.JSONResponse{ Code: http.StatusOK, - JSON: passwordLogin(), + JSON: flows{ + Flows: loginFlows, + }, } } else if req.Method == http.MethodPost { - login, cleanup, authErr := auth.LoginFromJSONReader(req.Context(), req.Body, userAPI, userAPI, cfg) + login, cleanup, authErr := auth.LoginFromJSONReader(req, userAPI, userAPI, cfg) if authErr != nil { return *authErr } diff --git a/clientapi/routing/login_test.go b/clientapi/routing/login_test.go index 252017db2e..3f89344904 100644 --- a/clientapi/routing/login_test.go +++ b/clientapi/routing/login_test.go @@ -49,7 +49,7 @@ func TestLogin(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) // Needed for /login - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the userAPI for this test, so nil for other APIs/caches etc. Setup(routers, cfg, nil, nil, userAPI, nil, nil, nil, nil, nil, nil, nil, caching.DisableMetrics) @@ -114,6 +114,44 @@ func TestLogin(t *testing.T) { ctx := context.Background() + // Inject a dummy application service, so we have a "m.login.application_service" + // in the login flows + as := &config.ApplicationService{} + cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as} + + t.Run("Supported log-in flows are returned", func(t *testing.T) { + req := test.NewRequest(t, http.MethodGet, "/_matrix/client/v3/login") + rec := httptest.NewRecorder() + routers.Client.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("failed to get log-in flows: %s", rec.Body.String()) + } + + t.Logf("response: %s", rec.Body.String()) + resp := flows{} + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + + appServiceFound := false + passwordFound := false + for _, flow := range resp.Flows { + if flow.Type == "m.login.password" { + passwordFound = true + } else if flow.Type == "m.login.application_service" { + appServiceFound = true + } else { + t.Fatalf("got unknown login flow: %s", flow.Type) + } + } + if !appServiceFound { + t.Fatal("m.login.application_service missing from login flows") + } + if !passwordFound { + t.Fatal("m.login.password missing from login flows") + } + }) + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{ diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 8b8cc47bcc..06683c47d7 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -181,11 +181,19 @@ func SendKick( return *errRes } + bodyUserID, err := spec.NewUserID(body.UserID, true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("body userID is invalid"), + } + } + pl, errRes := getPowerlevels(req, rsAPI, roomID) if errRes != nil { return *errRes } - allowedToKick := pl.UserLevel(*senderID) >= pl.Kick + allowedToKick := pl.UserLevel(*senderID) >= pl.Kick || bodyUserID.String() == deviceUserID.String() if !allowedToKick { return util.JSONResponse{ Code: http.StatusForbidden, @@ -193,13 +201,6 @@ func SendKick( } } - bodyUserID, err := spec.NewUserID(body.UserID, true) - if err != nil { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: spec.BadJSON("body userID is invalid"), - } - } var queryRes roomserverAPI.QueryMembershipForUserResponse err = rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{ RoomID: roomID, diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index 564cd588aa..9959144c83 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -300,7 +300,7 @@ func updateProfile( }, e } - if err := api.SendEvents(ctx, rsAPI, api.KindNew, events, device.UserDomain(), domain, domain, nil, true); err != nil { + if err := api.SendEvents(ctx, rsAPI, api.KindNew, events, device.UserDomain(), domain, domain, nil, false); err != nil { util.GetLogger(ctx).WithError(err).Error("SendEvents failed") return util.JSONResponse{ Code: http.StatusInternalServerError, diff --git a/clientapi/routing/receipt.go b/clientapi/routing/receipt.go index be6542979f..1d7e35562d 100644 --- a/clientapi/routing/receipt.go +++ b/clientapi/routing/receipt.go @@ -23,13 +23,12 @@ import ( "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/gomatrixserverlib/spec" - "github.com/matrix-org/dendrite/userapi/api" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" "github.com/sirupsen/logrus" ) -func SetReceipt(req *http.Request, userAPI api.ClientUserAPI, syncProducer *producers.SyncAPIProducer, device *userapi.Device, roomID, receiptType, eventID string) util.JSONResponse { +func SetReceipt(req *http.Request, userAPI userapi.ClientUserAPI, syncProducer *producers.SyncAPIProducer, device *userapi.Device, roomID, receiptType, eventID string) util.JSONResponse { timestamp := spec.AsTimestamp(time.Now()) logrus.WithFields(logrus.Fields{ "roomID": roomID, @@ -54,13 +53,13 @@ func SetReceipt(req *http.Request, userAPI api.ClientUserAPI, syncProducer *prod } } - dataReq := api.InputAccountDataRequest{ + dataReq := userapi.InputAccountDataRequest{ UserID: device.UserID, DataType: "m.fully_read", RoomID: roomID, AccountData: data, } - dataRes := api.InputAccountDataResponse{} + dataRes := userapi.InputAccountDataResponse{} if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil { util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed") return util.ErrorResponse(err) diff --git a/clientapi/routing/redaction.go b/clientapi/routing/redaction.go index aa579db644..f331a73c72 100644 --- a/clientapi/routing/redaction.go +++ b/clientapi/routing/redaction.go @@ -34,7 +34,8 @@ import ( ) type redactionContent struct { - Reason string `json:"reason"` + Reason string `json:"reason"` + Redacts string `json:"redacts"` } type redactionResponse struct { @@ -151,6 +152,11 @@ func SendRedaction( Type: spec.MRoomRedaction, Redacts: eventID, } + + // Room version 11 expects the "redacts" field on the + // content field, so add it here as well + r.Redacts = eventID + err = proto.SetContent(r) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("proto.SetContent failed") diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 565c415332..558418a6f8 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -236,7 +236,7 @@ type authDict struct { // TODO: Lots of custom keys depending on the type } -// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#user-interactive-authentication-api +// https://spec.matrix.org/v1.7/client-server-api/#user-interactive-authentication-api type userInteractiveResponse struct { Flows []authtypes.Flow `json:"flows"` Completed []authtypes.LoginType `json:"completed"` @@ -256,7 +256,7 @@ func newUserInteractiveResponse( } } -// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register +// https://spec.matrix.org/v1.7/client-server-api/#post_matrixclientv3register type registerResponse struct { UserID string `json:"user_id"` AccessToken string `json:"access_token,omitempty"` @@ -462,7 +462,7 @@ func validateApplicationService( } // Register processes a /register request. -// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register +// https://spec.matrix.org/v1.7/client-server-api/#post_matrixclientv3register func Register( req *http.Request, userAPI userapi.ClientUserAPI, @@ -647,6 +647,16 @@ func handleGuestRegistration( } } +// localpartMatchesExclusiveNamespaces will check if a given username matches any +// application service's exclusive users namespace +func localpartMatchesExclusiveNamespaces( + cfg *config.ClientAPI, + localpart string, +) bool { + userID := userutil.MakeUserID(localpart, cfg.Matrix.ServerName) + return cfg.Derived.ExclusiveApplicationServicesUsernameRegexp.MatchString(userID) +} + // handleRegistrationFlow will direct and complete registration flow stages // that the client has requested. // nolint: gocyclo @@ -695,7 +705,7 @@ func handleRegistrationFlow( // If an access token is provided, ignore this check this is an appservice // request and we will validate in validateApplicationService if len(cfg.Derived.ApplicationServices) != 0 && - UsernameMatchesExclusiveNamespaces(cfg, r.Username) { + localpartMatchesExclusiveNamespaces(cfg, r.Username) { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: spec.ASExclusive("This username is reserved by an application service."), @@ -772,7 +782,7 @@ func handleApplicationServiceRegistration( // Check application service register user request is valid. // The application service's ID is returned if so. - appserviceID, err := validateApplicationService( + appserviceID, err := internal.ValidateApplicationServiceRequest( cfg, r.Username, accessToken, ) if err != nil { diff --git a/clientapi/routing/register_test.go b/clientapi/routing/register_test.go index 0a1986cf71..7fa740e7f5 100644 --- a/clientapi/routing/register_test.go +++ b/clientapi/routing/register_test.go @@ -298,25 +298,29 @@ func Test_register(t *testing.T) { guestsDisabled bool enableRecaptcha bool captchaBody string - wantResponse util.JSONResponse + // in case of an error, the expected response + wantErrorResponse util.JSONResponse + // in case of success, the expected username assigned + wantUsername string }{ { name: "disallow guests", kind: "guest", guestsDisabled: true, - wantResponse: util.JSONResponse{ + wantErrorResponse: util.JSONResponse{ Code: http.StatusForbidden, JSON: spec.Forbidden(`Guest registration is disabled on "test"`), }, }, { - name: "allow guests", - kind: "guest", + name: "allow guests", + kind: "guest", + wantUsername: "1", }, { name: "unknown login type", loginType: "im.not.known", - wantResponse: util.JSONResponse{ + wantErrorResponse: util.JSONResponse{ Code: http.StatusNotImplemented, JSON: spec.Unknown("unknown/unimplemented auth type"), }, @@ -324,25 +328,33 @@ func Test_register(t *testing.T) { { name: "disabled registration", registrationDisabled: true, - wantResponse: util.JSONResponse{ + wantErrorResponse: util.JSONResponse{ Code: http.StatusForbidden, JSON: spec.Forbidden(`Registration is disabled on "test"`), }, }, { - name: "successful registration, numeric ID", - username: "", - password: "someRandomPassword", - forceEmpty: true, + name: "successful registration, numeric ID", + username: "", + password: "someRandomPassword", + forceEmpty: true, + wantUsername: "2", }, { name: "successful registration", username: "success", }, + { + name: "successful registration, sequential numeric ID", + username: "", + password: "someRandomPassword", + forceEmpty: true, + wantUsername: "3", + }, { name: "failing registration - user already exists", username: "success", - wantResponse: util.JSONResponse{ + wantErrorResponse: util.JSONResponse{ Code: http.StatusBadRequest, JSON: spec.UserInUse("Desired user ID is already taken."), }, @@ -352,14 +364,14 @@ func Test_register(t *testing.T) { username: "LOWERCASED", // this is going to be lower-cased }, { - name: "invalid username", - username: "#totalyNotValid", - wantResponse: *internal.UsernameResponse(internal.ErrUsernameInvalid), + name: "invalid username", + username: "#totalyNotValid", + wantErrorResponse: *internal.UsernameResponse(internal.ErrUsernameInvalid), }, { name: "numeric username is forbidden", username: "1337", - wantResponse: util.JSONResponse{ + wantErrorResponse: util.JSONResponse{ Code: http.StatusBadRequest, JSON: spec.InvalidUsername("Numeric user IDs are reserved"), }, @@ -367,7 +379,7 @@ func Test_register(t *testing.T) { { name: "disabled recaptcha login", loginType: authtypes.LoginTypeRecaptcha, - wantResponse: util.JSONResponse{ + wantErrorResponse: util.JSONResponse{ Code: http.StatusForbidden, JSON: spec.Unknown(ErrCaptchaDisabled.Error()), }, @@ -376,7 +388,7 @@ func Test_register(t *testing.T) { name: "enabled recaptcha, no response defined", enableRecaptcha: true, loginType: authtypes.LoginTypeRecaptcha, - wantResponse: util.JSONResponse{ + wantErrorResponse: util.JSONResponse{ Code: http.StatusBadRequest, JSON: spec.BadJSON(ErrMissingResponse.Error()), }, @@ -386,7 +398,7 @@ func Test_register(t *testing.T) { enableRecaptcha: true, loginType: authtypes.LoginTypeRecaptcha, captchaBody: `notvalid`, - wantResponse: util.JSONResponse{ + wantErrorResponse: util.JSONResponse{ Code: http.StatusUnauthorized, JSON: spec.BadJSON(ErrInvalidCaptcha.Error()), }, @@ -398,11 +410,11 @@ func Test_register(t *testing.T) { captchaBody: `success`, }, { - name: "captcha invalid from remote", - enableRecaptcha: true, - loginType: authtypes.LoginTypeRecaptcha, - captchaBody: `i should fail for other reasons`, - wantResponse: util.JSONResponse{Code: http.StatusInternalServerError, JSON: spec.InternalServerError{}}, + name: "captcha invalid from remote", + enableRecaptcha: true, + loginType: authtypes.LoginTypeRecaptcha, + captchaBody: `i should fail for other reasons`, + wantErrorResponse: util.JSONResponse{Code: http.StatusInternalServerError, JSON: spec.InternalServerError{}}, }, } @@ -416,7 +428,7 @@ func Test_register(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -486,8 +498,8 @@ func Test_register(t *testing.T) { t.Fatalf("unexpected registration flows: %+v, want %+v", r.Flows, cfg.Derived.Registration.Flows) } case spec.MatrixError: - if !reflect.DeepEqual(tc.wantResponse, resp) { - t.Fatalf("(%s), unexpected response: %+v, want: %+v", tc.name, resp, tc.wantResponse) + if !reflect.DeepEqual(tc.wantErrorResponse, resp) { + t.Fatalf("(%s), unexpected response: %+v, want: %+v", tc.name, resp, tc.wantErrorResponse) } return case registerResponse: @@ -505,6 +517,13 @@ func Test_register(t *testing.T) { if r.DeviceID == "" { t.Fatalf("missing deviceID in response") } + // if an expected username is provided, assert that it is a match + if tc.wantUsername != "" { + wantUserID := strings.ToLower(fmt.Sprintf("@%s:%s", tc.wantUsername, "test")) + if wantUserID != r.UserID { + t.Fatalf("unexpected userID: %s, want %s", r.UserID, wantUserID) + } + } return default: t.Logf("Got response: %T", resp.JSON) @@ -541,44 +560,29 @@ func Test_register(t *testing.T) { resp = Register(req, userAPI, &cfg.ClientAPI) - switch resp.JSON.(type) { - case spec.InternalServerError: - if !reflect.DeepEqual(tc.wantResponse, resp) { - t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantResponse) + switch rr := resp.JSON.(type) { + case spec.InternalServerError, spec.MatrixError, util.JSONResponse: + if !reflect.DeepEqual(tc.wantErrorResponse, resp) { + t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantErrorResponse) } return - case spec.MatrixError: - if !reflect.DeepEqual(tc.wantResponse, resp) { - t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantResponse) + case registerResponse: + // validate the response + if tc.wantUsername != "" { + // if an expected username is provided, assert that it is a match + wantUserID := strings.ToLower(fmt.Sprintf("@%s:%s", tc.wantUsername, "test")) + if wantUserID != rr.UserID { + t.Fatalf("unexpected userID: %s, want %s", rr.UserID, wantUserID) + } } - return - case util.JSONResponse: - if !reflect.DeepEqual(tc.wantResponse, resp) { - t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantResponse) + if rr.DeviceID != *reg.DeviceID { + t.Fatalf("unexpected deviceID: %s, want %s", rr.DeviceID, *reg.DeviceID) } - return - } - - rr, ok := resp.JSON.(registerResponse) - if !ok { - t.Fatalf("expected a registerresponse, got %T", resp.JSON) - } - - // validate the response - if tc.forceEmpty { - // when not supplying a username, one will be generated. Given this _SHOULD_ be - // the second user, set the username accordingly - reg.Username = "2" - } - wantUserID := strings.ToLower(fmt.Sprintf("@%s:%s", reg.Username, "test")) - if wantUserID != rr.UserID { - t.Fatalf("unexpected userID: %s, want %s", rr.UserID, wantUserID) - } - if rr.DeviceID != *reg.DeviceID { - t.Fatalf("unexpected deviceID: %s, want %s", rr.DeviceID, *reg.DeviceID) - } - if rr.AccessToken == "" { - t.Fatalf("missing accessToken in response") + if rr.AccessToken == "" { + t.Fatalf("missing accessToken in response") + } + default: + t.Fatalf("expected one of internalservererror, matrixerror, jsonresponse, registerresponse, got %T", resp.JSON) } }) } @@ -596,7 +600,7 @@ func TestRegisterUserWithDisplayName(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) deviceName, deviceID := "deviceName", "deviceID" expectedDisplayName := "DisplayName" response := completeRegistration( @@ -637,7 +641,7 @@ func TestRegisterAdminUsingSharedSecret(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) expectedDisplayName := "rabbit" jsonStr := []byte(`{"admin":true,"mac":"24dca3bba410e43fe64b9b5c28306693bf3baa9f","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`) diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index fc9c05ba9b..44e82aed08 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -224,7 +224,7 @@ func SendEvent( req.Context(), rsAPI, api.KindNew, []*types.HeaderedEvent{ - &types.HeaderedEvent{PDU: e}, + {PDU: e}, }, device.UserDomain(), domain, @@ -263,7 +263,11 @@ func SendEvent( } func updatePowerLevels(req *http.Request, r map[string]interface{}, roomID string, rsAPI api.ClientRoomserverAPI) error { - userMap := r["users"].(map[string]interface{}) + users, ok := r["users"] + if !ok { + return nil + } + userMap := users.(map[string]interface{}) validRoomID, err := spec.NewRoomID(roomID) if err != nil { return err @@ -277,7 +281,8 @@ func updatePowerLevels(req *http.Request, r map[string]interface{}, roomID strin if err != nil { return err } else if senderID == nil { - return fmt.Errorf("sender ID not found for %s in %s", uID, *validRoomID) + util.GetLogger(req.Context()).Warnf("sender ID not found for %s in %s", uID, *validRoomID) + continue } userMap[string(*senderID)] = level delete(userMap, user) diff --git a/cmd/dendrite-demo-pinecone/monolith/monolith.go b/cmd/dendrite-demo-pinecone/monolith/monolith.go index 41af568a61..d9f44b5cc0 100644 --- a/cmd/dendrite-demo-pinecone/monolith/monolith.go +++ b/cmd/dendrite-demo-pinecone/monolith/monolith.go @@ -145,7 +145,7 @@ func (p *P2PMonolith) SetupDendrite( ) rsAPI.SetFederationAPI(fsAPI, keyRing) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, enableMetrics, fsAPI.IsBlacklistedOrBackingOff) asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) diff --git a/cmd/dendrite-demo-yggdrasil/README.md b/cmd/dendrite-demo-yggdrasil/README.md index 14fc3a2db0..23304c214a 100644 --- a/cmd/dendrite-demo-yggdrasil/README.md +++ b/cmd/dendrite-demo-yggdrasil/README.md @@ -1,6 +1,6 @@ # Yggdrasil Demo -This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.18 or later. +This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.20 or later. To run the homeserver, start at the root of the Dendrite repository and run: diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 3ec5501139..1e5186348f 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -213,14 +213,15 @@ func main() { natsInstance := jetstream.NATSInstance{} rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.EnableMetrics) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation) - - asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) - rsAPI.SetAppserviceAPI(asAPI) fsAPI := federationapi.NewInternalAPI( processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true, ) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics, fsAPI.IsBlacklistedOrBackingOff) + + asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) + rsAPI.SetAppserviceAPI(asAPI) + rsAPI.SetFederationAPI(fsAPI, keyRing) monolith := setup.Monolith{ diff --git a/cmd/dendrite-upgrade-tests/main.go b/cmd/dendrite-upgrade-tests/main.go index 68919e5252..b78c5f6055 100644 --- a/cmd/dendrite-upgrade-tests/main.go +++ b/cmd/dendrite-upgrade-tests/main.go @@ -7,7 +7,6 @@ import ( "flag" "fmt" "io" - "io/ioutil" "log" "net/http" "os" @@ -515,7 +514,7 @@ func testCreateAccount(dockerClient *client.Client, version *semver.Version, con } defer response.Close() - data, err := ioutil.ReadAll(response.Reader) + data, err := io.ReadAll(response.Reader) if err != nil { return err } diff --git a/cmd/dendrite/main.go b/cmd/dendrite/main.go index f3140b4e22..5234b75047 100644 --- a/cmd/dendrite/main.go +++ b/cmd/dendrite/main.go @@ -162,7 +162,7 @@ func main() { // dependency. Other components also need updating after their dependencies are up. rsAPI.SetFederationAPI(fsAPI, keyRing) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient, caching.EnableMetrics, fsAPI.IsBlacklistedOrBackingOff) asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) diff --git a/cmd/resolve-state/main.go b/cmd/resolve-state/main.go index 3ffcac9e6d..d6db724365 100644 --- a/cmd/resolve-state/main.go +++ b/cmd/resolve-state/main.go @@ -11,13 +11,11 @@ import ( "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup" "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" @@ -35,6 +33,19 @@ var roomVersion = flag.String("roomversion", "5", "the room version to parse eve var filterType = flag.String("filtertype", "", "the event types to filter on") var difference = flag.Bool("difference", false, "whether to calculate the difference between snapshots") +// dummyQuerier implements QuerySenderIDAPI. Does **NOT** do any "magic" for pseudoID rooms +// to avoid having to "start" a full roomserver API. +type dummyQuerier struct{} + +func (d dummyQuerier) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) { + s := spec.SenderIDFromUserID(userID) + return &s, nil +} + +func (d dummyQuerier) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return senderID.ToUserID(), nil +} + // nolint:gocyclo func main() { ctx := context.Background() @@ -56,27 +67,32 @@ func main() { } } - fmt.Println("Fetching", len(snapshotNIDs), "snapshot NIDs") - processCtx := process.NewProcessContext() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + + dbOpts := cfg.RoomServer.Database + if dbOpts.ConnectionString == "" { + dbOpts = cfg.Global.DatabaseOptions + } + + fmt.Println("Opening database") roomserverDB, err := storage.Open( - processCtx.Context(), cm, &cfg.RoomServer.Database, - caching.NewRistrettoCache(128*1024*1024, time.Hour, true), + processCtx.Context(), cm, &dbOpts, + caching.NewRistrettoCache(8*1024*1024, time.Minute*5, caching.DisableMetrics), ) if err != nil { panic(err) } - natsInstance := &jetstream.NATSInstance{} - rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, - natsInstance, caching.NewRistrettoCache(128*1024*1024, time.Hour, true), false) + rsAPI := dummyQuerier{} roomInfo := &types.RoomInfo{ RoomVersion: gomatrixserverlib.RoomVersion(*roomVersion), } stateres := state.NewStateResolution(roomserverDB, roomInfo, rsAPI) + fmt.Println("Fetching", len(snapshotNIDs), "snapshot NIDs") + if *difference { if len(snapshotNIDs) != 2 { panic("need exactly two state snapshot NIDs to calculate difference") @@ -186,12 +202,25 @@ func main() { authEvents[i] = authEventEntries[i].PDU } + // Get the roomNID + roomInfo, err = roomserverDB.RoomInfo(ctx, authEvents[0].RoomID().String()) + if err != nil { + panic(err) + } + fmt.Println("Resolving state") var resolved Events resolved, err = gomatrixserverlib.ResolveConflicts( gomatrixserverlib.RoomVersion(*roomVersion), events, authEvents, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return rsAPI.QueryUserIDForSender(ctx, roomID, senderID) }, + func(eventID string) bool { + isRejected, rejectedErr := roomserverDB.IsEventRejected(ctx, roomInfo.RoomNID, eventID) + if rejectedErr != nil { + return true + } + return isRejected + }, ) if err != nil { panic(err) diff --git a/dendrite-sample.yaml b/dendrite-sample.yaml index 8abc230115..e143a73983 100644 --- a/dendrite-sample.yaml +++ b/dendrite-sample.yaml @@ -72,6 +72,10 @@ global: # The base URL to delegate client-server communications to e.g. https://localhost well_known_client_name: "" + # The server name to delegate sliding sync communications to, with optional port. + # Requires `well_known_client_name` to also be configured. + well_known_sliding_sync_proxy: "" + # Lists of domains that the server will trust as identity servers to verify third # party identifiers such as phone numbers and email addresses. trusted_third_party_id_servers: @@ -321,6 +325,10 @@ user_api: auto_join_rooms: # - "#main:matrix.org" + # The number of workers to start for the DeviceListUpdater. Defaults to 8. + # This only needs updating if the "InputDeviceListUpdate" stream keeps growing indefinitely. + # worker_count: 8 + # Configuration for Opentracing. # See https://github.com/matrix-org/dendrite/tree/master/docs/tracing for information on # how this works and how to set it up. diff --git a/docs/FAQ.md b/docs/FAQ.md index 570ba677e9..82b1581ea7 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -24,7 +24,7 @@ No, although a good portion of the Matrix specification has been implemented. Mo Dendrite development is currently supported by a small team of developers and due to those limited resources, the majority of the effort is focused on getting Dendrite to be specification complete. If there are major features you're requesting (e.g. new administration endpoints), we'd like to strongly encourage you to join the community in supporting -the development efforts through [contributing](../development/contributing). +the development efforts through [contributing](./development/CONTRIBUTING.md). ## Is there a migration path from Synapse to Dendrite? @@ -105,7 +105,7 @@ This can be done by performing a room upgrade. Use the command `/upgraderoom /mnt/dendrite.yaml" + +# Windows equivalent: docker run --rm --entrypoint="/bin/sh" -v %cd%/config:/mnt matrixdotorg/dendrite-monolith:latest -c "/usr/bin/generate-config -dir /var/dendrite/ -db postgres://dendrite:itsasecret@postgres/dendrite?sslmode=disable -server YourDomainHere > /mnt/dendrite.yaml" ``` You can then change `config/dendrite.yaml` to your liking. diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index e148199fb1..efbfa33157 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -24,7 +24,6 @@ import ( "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/sirupsen/logrus" - "github.com/matrix-org/dendrite/federationapi/api" federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/federationapi/consumers" "github.com/matrix-org/dendrite/federationapi/internal" @@ -102,7 +101,7 @@ func NewInternalAPI( caches *caching.Caches, keyRing *gomatrixserverlib.KeyRing, resetBlacklist bool, -) api.FederationInternalAPI { +) *internal.FederationInternalAPI { cfg := &dendriteCfg.FederationAPI federationDB, err := storage.NewDatabase(processContext.Context(), cm, &cfg.Database, caches, dendriteCfg.Global.IsLocalServerName) @@ -126,7 +125,7 @@ func NewInternalAPI( queues := queue.NewOutgoingQueues( federationDB, processContext, cfg.Matrix.DisableFederation, - cfg.Matrix.ServerName, federation, rsAPI, &stats, + cfg.Matrix.ServerName, federation, &stats, signingInfo, ) diff --git a/federationapi/internal/api.go b/federationapi/internal/api.go index 3e6f39566e..67388a102c 100644 --- a/federationapi/internal/api.go +++ b/federationapi/internal/api.go @@ -112,7 +112,7 @@ func NewFederationInternalAPI( } } -func (a *FederationInternalAPI) isBlacklistedOrBackingOff(s spec.ServerName) (*statistics.ServerStatistics, error) { +func (a *FederationInternalAPI) IsBlacklistedOrBackingOff(s spec.ServerName) (*statistics.ServerStatistics, error) { stats := a.statistics.ForServer(s) if stats.Blacklisted() { return stats, &api.FederationClientError{ @@ -151,7 +151,7 @@ func failBlacklistableError(err error, stats *statistics.ServerStatistics) (unti func (a *FederationInternalAPI) doRequestIfNotBackingOffOrBlacklisted( s spec.ServerName, request func() (interface{}, error), ) (interface{}, error) { - stats, err := a.isBlacklistedOrBackingOff(s) + stats, err := a.IsBlacklistedOrBackingOff(s) if err != nil { return nil, err } diff --git a/federationapi/internal/federationclient_test.go b/federationapi/internal/federationclient_test.go index 8c562dd61e..fe8d84ffb7 100644 --- a/federationapi/internal/federationclient_test.go +++ b/federationapi/internal/federationclient_test.go @@ -65,7 +65,7 @@ func TestFederationClientQueryKeys(t *testing.T) { queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedapi := FederationInternalAPI{ @@ -96,7 +96,7 @@ func TestFederationClientQueryKeysBlacklisted(t *testing.T) { queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedapi := FederationInternalAPI{ @@ -126,7 +126,7 @@ func TestFederationClientQueryKeysFailure(t *testing.T) { queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedapi := FederationInternalAPI{ @@ -156,7 +156,7 @@ func TestFederationClientClaimKeys(t *testing.T) { queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedapi := FederationInternalAPI{ @@ -187,7 +187,7 @@ func TestFederationClientClaimKeysBlacklisted(t *testing.T) { queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedapi := FederationInternalAPI{ diff --git a/federationapi/internal/perform_test.go b/federationapi/internal/perform_test.go index 656755f968..2795a018a4 100644 --- a/federationapi/internal/perform_test.go +++ b/federationapi/internal/perform_test.go @@ -70,7 +70,7 @@ func TestPerformWakeupServers(t *testing.T) { queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedAPI := NewFederationInternalAPI( @@ -116,7 +116,7 @@ func TestQueryRelayServers(t *testing.T) { queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedAPI := NewFederationInternalAPI( @@ -157,7 +157,7 @@ func TestRemoveRelayServers(t *testing.T) { queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedAPI := NewFederationInternalAPI( @@ -197,7 +197,7 @@ func TestPerformDirectoryLookup(t *testing.T) { queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedAPI := NewFederationInternalAPI( @@ -236,7 +236,7 @@ func TestPerformDirectoryLookupRelaying(t *testing.T) { queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedAPI := NewFederationInternalAPI( diff --git a/federationapi/queue/destinationqueue.go b/federationapi/queue/destinationqueue.go index 880aee0d35..f51e849fa3 100644 --- a/federationapi/queue/destinationqueue.go +++ b/federationapi/queue/destinationqueue.go @@ -31,7 +31,6 @@ import ( "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/federationapi/storage" "github.com/matrix-org/dendrite/federationapi/storage/shared/receipt" - "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/process" ) @@ -53,7 +52,6 @@ type destinationQueue struct { db storage.Database process *process.ProcessContext signing map[spec.ServerName]*fclient.SigningIdentity - rsAPI api.FederationRoomserverAPI client fclient.FederationClient // federation client origin spec.ServerName // origin of requests destination spec.ServerName // destination of requests diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go index 24b3efd2d4..892c26a2cc 100644 --- a/federationapi/queue/queue.go +++ b/federationapi/queue/queue.go @@ -27,12 +27,10 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" - "github.com/tidwall/gjson" "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/federationapi/storage" "github.com/matrix-org/dendrite/federationapi/storage/shared/receipt" - "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/process" ) @@ -43,7 +41,6 @@ type OutgoingQueues struct { db storage.Database process *process.ProcessContext disabled bool - rsAPI api.FederationRoomserverAPI origin spec.ServerName client fclient.FederationClient statistics *statistics.Statistics @@ -90,7 +87,6 @@ func NewOutgoingQueues( disabled bool, origin spec.ServerName, client fclient.FederationClient, - rsAPI api.FederationRoomserverAPI, statistics *statistics.Statistics, signing []*fclient.SigningIdentity, ) *OutgoingQueues { @@ -98,7 +94,6 @@ func NewOutgoingQueues( disabled: disabled, process: process, db: db, - rsAPI: rsAPI, origin: origin, client: client, statistics: statistics, @@ -162,7 +157,6 @@ func (oqs *OutgoingQueues) getQueue(destination spec.ServerName) *destinationQue queues: oqs, db: oqs.db, process: oqs.process, - rsAPI: oqs.rsAPI, origin: oqs.origin, destination: destination, client: oqs.client, @@ -213,18 +207,6 @@ func (oqs *OutgoingQueues) SendEvent( delete(destmap, local) } - // Check if any of the destinations are prohibited by server ACLs. - for destination := range destmap { - if api.IsServerBannedFromRoom( - oqs.process.Context(), - oqs.rsAPI, - ev.RoomID().String(), - destination, - ) { - delete(destmap, destination) - } - } - // If there are no remaining destinations then give up. if len(destmap) == 0 { return nil @@ -303,24 +285,6 @@ func (oqs *OutgoingQueues) SendEDU( delete(destmap, local) } - // There is absolutely no guarantee that the EDU will have a room_id - // field, as it is not required by the spec. However, if it *does* - // (e.g. typing notifications) then we should try to make sure we don't - // bother sending them to servers that are prohibited by the server - // ACLs. - if result := gjson.GetBytes(e.Content, "room_id"); result.Exists() { - for destination := range destmap { - if api.IsServerBannedFromRoom( - oqs.process.Context(), - oqs.rsAPI, - result.Str, - destination, - ) { - delete(destmap, destination) - } - } - } - // If there are no remaining destinations then give up. if len(destmap) == 0 { return nil diff --git a/federationapi/queue/queue_test.go b/federationapi/queue/queue_test.go index e75615e05c..73d3b0598f 100644 --- a/federationapi/queue/queue_test.go +++ b/federationapi/queue/queue_test.go @@ -34,7 +34,6 @@ import ( "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/federationapi/storage" - rsapi "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/process" @@ -65,15 +64,6 @@ func mustCreateFederationDatabase(t *testing.T, dbType test.DBType, realDatabase } } -type stubFederationRoomServerAPI struct { - rsapi.FederationRoomserverAPI -} - -func (r *stubFederationRoomServerAPI) QueryServerBannedFromRoom(ctx context.Context, req *rsapi.QueryServerBannedFromRoomRequest, res *rsapi.QueryServerBannedFromRoomResponse) error { - res.Banned = false - return nil -} - type stubFederationClient struct { fclient.FederationClient shouldTxSucceed bool @@ -126,7 +116,6 @@ func testSetup(failuresUntilBlacklist uint32, failuresUntilAssumedOffline uint32 txCount: *atomic.NewUint32(0), txRelayCount: *atomic.NewUint32(0), } - rs := &stubFederationRoomServerAPI{} stats := statistics.NewStatistics(db, failuresUntilBlacklist, failuresUntilAssumedOffline) signingInfo := []*fclient.SigningIdentity{ @@ -136,7 +125,7 @@ func testSetup(failuresUntilBlacklist uint32, failuresUntilAssumedOffline uint32 ServerName: "localhost", }, } - queues := NewOutgoingQueues(db, processContext, false, "localhost", fc, rs, &stats, signingInfo) + queues := NewOutgoingQueues(db, processContext, false, "localhost", fc, &stats, signingInfo) return db, fc, queues, processContext, close } diff --git a/federationapi/routing/profile_test.go b/federationapi/routing/profile_test.go index a31b206c1c..ba13e07fc9 100644 --- a/federationapi/routing/profile_test.go +++ b/federationapi/routing/profile_test.go @@ -26,7 +26,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" fedAPI "github.com/matrix-org/dendrite/federationapi" - fedInternal "github.com/matrix-org/dendrite/federationapi/internal" "github.com/matrix-org/dendrite/federationapi/routing" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" @@ -67,11 +66,8 @@ func TestHandleQueryProfile(t *testing.T) { keyRing := serverKeyAPI.KeyRing() fedapi := fedAPI.NewInternalAPI(processCtx, cfg, cm, &natsInstance, &fedClient, nil, nil, keyRing, true) userapi := fakeUserAPI{} - r, ok := fedapi.(*fedInternal.FederationInternalAPI) - if !ok { - panic("This is a programming error.") - } - routing.Setup(routers, cfg, nil, r, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics) + + routing.Setup(routers, cfg, nil, fedapi, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics) handler := fedMux.Get(routing.QueryProfileRouteName).GetHandler().ServeHTTP _, sk, _ := ed25519.GenerateKey(nil) diff --git a/federationapi/routing/query_test.go b/federationapi/routing/query_test.go index bb14ab031c..fd0894d15c 100644 --- a/federationapi/routing/query_test.go +++ b/federationapi/routing/query_test.go @@ -25,7 +25,6 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" fedAPI "github.com/matrix-org/dendrite/federationapi" - fedInternal "github.com/matrix-org/dendrite/federationapi/internal" "github.com/matrix-org/dendrite/federationapi/routing" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" @@ -65,11 +64,8 @@ func TestHandleQueryDirectory(t *testing.T) { keyRing := serverKeyAPI.KeyRing() fedapi := fedAPI.NewInternalAPI(processCtx, cfg, cm, &natsInstance, &fedClient, nil, nil, keyRing, true) userapi := fakeUserAPI{} - r, ok := fedapi.(*fedInternal.FederationInternalAPI) - if !ok { - panic("This is a programming error.") - } - routing.Setup(routers, cfg, nil, r, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics) + + routing.Setup(routers, cfg, nil, fedapi, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics) handler := fedMux.Get(routing.QueryDirectoryRouteName).GetHandler().ServeHTTP _, sk, _ := ed25519.GenerateKey(nil) diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index f629479dac..ff4f7bd065 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -23,7 +23,6 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" fedAPI "github.com/matrix-org/dendrite/federationapi" - fedInternal "github.com/matrix-org/dendrite/federationapi/internal" "github.com/matrix-org/dendrite/federationapi/routing" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" @@ -62,11 +61,8 @@ func TestHandleSend(t *testing.T) { fedapi := fedAPI.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, nil, nil, nil, true) serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - r, ok := fedapi.(*fedInternal.FederationInternalAPI) - if !ok { - panic("This is a programming error.") - } - routing.Setup(routers, cfg, nil, r, keyRing, nil, nil, &cfg.MSCs, nil, caching.DisableMetrics) + + routing.Setup(routers, cfg, nil, fedapi, keyRing, nil, nil, &cfg.MSCs, nil, caching.DisableMetrics) handler := fedMux.Get(routing.SendRouteName).GetHandler().ServeHTTP _, sk, _ := ed25519.GenerateKey(nil) diff --git a/federationapi/storage/postgres/notary_server_keys_metadata_table.go b/federationapi/storage/postgres/notary_server_keys_metadata_table.go index 7a1ec41220..47aa82b48f 100644 --- a/federationapi/storage/postgres/notary_server_keys_metadata_table.go +++ b/federationapi/storage/postgres/notary_server_keys_metadata_table.go @@ -151,7 +151,7 @@ func (s *notaryServerKeysMetadataStatements) SelectKeys(ctx context.Context, txn } results = append(results, sk) } - return results, nil + return results, rows.Err() } func (s *notaryServerKeysMetadataStatements) DeleteOldJSONResponses(ctx context.Context, txn *sql.Tx) error { diff --git a/federationapi/storage/postgres/queue_json_table.go b/federationapi/storage/postgres/queue_json_table.go index 563738dd58..f92e33d5eb 100644 --- a/federationapi/storage/postgres/queue_json_table.go +++ b/federationapi/storage/postgres/queue_json_table.go @@ -109,5 +109,5 @@ func (s *queueJSONStatements) SelectQueueJSON( } blobs[nid] = blob } - return blobs, err + return blobs, rows.Err() } diff --git a/federationapi/storage/postgres/relay_servers_table.go b/federationapi/storage/postgres/relay_servers_table.go index 9e1bc5d404..1a47816e2f 100644 --- a/federationapi/storage/postgres/relay_servers_table.go +++ b/federationapi/storage/postgres/relay_servers_table.go @@ -110,7 +110,7 @@ func (s *relayServersStatements) SelectRelayServers( } result = append(result, spec.ServerName(relayServer)) } - return result, nil + return result, rows.Err() } func (s *relayServersStatements) DeleteRelayServers( diff --git a/federationapi/storage/postgres/server_key_table.go b/federationapi/storage/postgres/server_key_table.go index c62446da53..fa58f1ea2d 100644 --- a/federationapi/storage/postgres/server_key_table.go +++ b/federationapi/storage/postgres/server_key_table.go @@ -94,12 +94,14 @@ func (s *serverSigningKeyStatements) BulkSelectServerKeys( } defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectServerKeys: rows.close() failed") results := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{} + + var serverName string + var keyID string + var key string + var validUntilTS int64 + var expiredTS int64 + var vk gomatrixserverlib.VerifyKey for rows.Next() { - var serverName string - var keyID string - var key string - var validUntilTS int64 - var expiredTS int64 if err = rows.Scan(&serverName, &keyID, &validUntilTS, &expiredTS, &key); err != nil { return nil, err } @@ -107,7 +109,6 @@ func (s *serverSigningKeyStatements) BulkSelectServerKeys( ServerName: spec.ServerName(serverName), KeyID: gomatrixserverlib.KeyID(keyID), } - vk := gomatrixserverlib.VerifyKey{} err = vk.Key.Decode(key) if err != nil { return nil, err diff --git a/federationapi/storage/sqlite3/joined_hosts_table.go b/federationapi/storage/sqlite3/joined_hosts_table.go index 2412cacdb7..4181943125 100644 --- a/federationapi/storage/sqlite3/joined_hosts_table.go +++ b/federationapi/storage/sqlite3/joined_hosts_table.go @@ -216,5 +216,5 @@ func joinedHostsFromStmt( }) } - return result, nil + return result, rows.Err() } diff --git a/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go b/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go index 2fd9ef2119..d9b98fc4f3 100644 --- a/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go +++ b/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go @@ -154,7 +154,7 @@ func (s *notaryServerKeysMetadataStatements) SelectKeys(ctx context.Context, txn } results = append(results, sk) } - return results, nil + return results, rows.Err() } func (s *notaryServerKeysMetadataStatements) DeleteOldJSONResponses(ctx context.Context, txn *sql.Tx) error { diff --git a/federationapi/storage/sqlite3/queue_json_table.go b/federationapi/storage/sqlite3/queue_json_table.go index 0e2806d56d..33ae061314 100644 --- a/federationapi/storage/sqlite3/queue_json_table.go +++ b/federationapi/storage/sqlite3/queue_json_table.go @@ -135,5 +135,5 @@ func (s *queueJSONStatements) SelectQueueJSON( } blobs[nid] = blob } - return blobs, err + return blobs, rows.Err() } diff --git a/federationapi/storage/sqlite3/relay_servers_table.go b/federationapi/storage/sqlite3/relay_servers_table.go index 36cabeb4d9..232db32af8 100644 --- a/federationapi/storage/sqlite3/relay_servers_table.go +++ b/federationapi/storage/sqlite3/relay_servers_table.go @@ -109,7 +109,7 @@ func (s *relayServersStatements) SelectRelayServers( } result = append(result, spec.ServerName(relayServer)) } - return result, nil + return result, rows.Err() } func (s *relayServersStatements) DeleteRelayServers( diff --git a/federationapi/storage/sqlite3/server_key_table.go b/federationapi/storage/sqlite3/server_key_table.go index f28b899405..65a854ce12 100644 --- a/federationapi/storage/sqlite3/server_key_table.go +++ b/federationapi/storage/sqlite3/server_key_table.go @@ -98,12 +98,13 @@ func (s *serverSigningKeyStatements) BulkSelectServerKeys( err := sqlutil.RunLimitedVariablesQuery( ctx, bulkSelectServerSigningKeysSQL, s.db, iKeyIDs, sqlutil.SQLite3MaxVariables, func(rows *sql.Rows) error { + var serverName string + var keyID string + var key string + var validUntilTS int64 + var expiredTS int64 + var vk gomatrixserverlib.VerifyKey for rows.Next() { - var serverName string - var keyID string - var key string - var validUntilTS int64 - var expiredTS int64 if err := rows.Scan(&serverName, &keyID, &validUntilTS, &expiredTS, &key); err != nil { return fmt.Errorf("bulkSelectServerKeys: %v", err) } @@ -111,7 +112,6 @@ func (s *serverSigningKeyStatements) BulkSelectServerKeys( ServerName: spec.ServerName(serverName), KeyID: gomatrixserverlib.KeyID(keyID), } - vk := gomatrixserverlib.VerifyKey{} err := vk.Key.Decode(key) if err != nil { return fmt.Errorf("bulkSelectServerKeys: %v", err) diff --git a/federationapi/storage/tables/server_key_table_test.go b/federationapi/storage/tables/server_key_table_test.go new file mode 100644 index 0000000000..e79a086b88 --- /dev/null +++ b/federationapi/storage/tables/server_key_table_test.go @@ -0,0 +1,116 @@ +package tables_test + +import ( + "context" + "testing" + "time" + + "github.com/matrix-org/dendrite/federationapi/storage/postgres" + "github.com/matrix-org/dendrite/federationapi/storage/sqlite3" + "github.com/matrix-org/dendrite/federationapi/storage/tables" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/stretchr/testify/assert" +) + +func mustCreateServerKeyDB(t *testing.T, dbType test.DBType) (tables.FederationServerSigningKeys, func()) { + connStr, close := test.PrepareDBConnectionString(t, dbType) + db, err := sqlutil.Open(&config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + }, sqlutil.NewExclusiveWriter()) + if err != nil { + t.Fatalf("failed to open database: %s", err) + } + var tab tables.FederationServerSigningKeys + switch dbType { + case test.DBTypePostgres: + tab, err = postgres.NewPostgresServerSigningKeysTable(db) + case test.DBTypeSQLite: + tab, err = sqlite3.NewSQLiteServerSigningKeysTable(db) + } + if err != nil { + t.Fatalf("failed to create table: %s", err) + } + return tab, close +} + +func TestServerKeysTable(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + ctx, cancel := context.WithCancel(context.Background()) + tab, close := mustCreateServerKeyDB(t, dbType) + t.Cleanup(func() { + close() + cancel() + }) + + req := gomatrixserverlib.PublicKeyLookupRequest{ + ServerName: "localhost", + KeyID: "ed25519:test", + } + expectedTimestamp := spec.AsTimestamp(time.Now().Add(time.Hour)) + res := gomatrixserverlib.PublicKeyLookupResult{ + VerifyKey: gomatrixserverlib.VerifyKey{Key: make(spec.Base64Bytes, 0)}, + ExpiredTS: 0, + ValidUntilTS: expectedTimestamp, + } + + // Insert the key + err := tab.UpsertServerKeys(ctx, nil, req, res) + assert.NoError(t, err) + + selectKeys := map[gomatrixserverlib.PublicKeyLookupRequest]spec.Timestamp{ + req: spec.AsTimestamp(time.Now()), + } + gotKeys, err := tab.BulkSelectServerKeys(ctx, nil, selectKeys) + assert.NoError(t, err) + + // Now we should have a key for the req above + assert.NotNil(t, gotKeys[req]) + assert.Equal(t, res, gotKeys[req]) + + // "Expire" the key by setting ExpireTS to a non-zero value and ValidUntilTS to 0 + expectedTimestamp = spec.AsTimestamp(time.Now()) + res.ExpiredTS = expectedTimestamp + res.ValidUntilTS = 0 + + // Update the key + err = tab.UpsertServerKeys(ctx, nil, req, res) + assert.NoError(t, err) + + gotKeys, err = tab.BulkSelectServerKeys(ctx, nil, selectKeys) + assert.NoError(t, err) + + // The key should be expired + assert.NotNil(t, gotKeys[req]) + assert.Equal(t, res, gotKeys[req]) + + // Upsert a different key to validate querying multiple keys + req2 := gomatrixserverlib.PublicKeyLookupRequest{ + ServerName: "notlocalhost", + KeyID: "ed25519:test2", + } + expectedTimestamp2 := spec.AsTimestamp(time.Now().Add(time.Hour)) + res2 := gomatrixserverlib.PublicKeyLookupResult{ + VerifyKey: gomatrixserverlib.VerifyKey{Key: make(spec.Base64Bytes, 0)}, + ExpiredTS: 0, + ValidUntilTS: expectedTimestamp2, + } + + err = tab.UpsertServerKeys(ctx, nil, req2, res2) + assert.NoError(t, err) + + // Select multiple keys + selectKeys[req2] = spec.AsTimestamp(time.Now()) + + gotKeys, err = tab.BulkSelectServerKeys(ctx, nil, selectKeys) + assert.NoError(t, err) + + // We now should receive two keys, one of which is expired + assert.Equal(t, 2, len(gotKeys)) + assert.Equal(t, res2, gotKeys[req2]) + assert.Equal(t, res, gotKeys[req]) + }) +} diff --git a/go.mod b/go.mod index 564b4250aa..387e99d8c0 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/blevesearch/bleve/v2 v2.3.8 github.com/codeclysm/extract v2.2.0+incompatible github.com/dgraph-io/ristretto v0.1.1 - github.com/docker/docker v24.0.5+incompatible + github.com/docker/docker v24.0.7+incompatible github.com/docker/go-connections v0.4.0 github.com/getsentry/sentry-go v0.14.0 github.com/gologme/log v1.3.0 @@ -22,12 +22,12 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 - github.com/matrix-org/gomatrixserverlib v0.0.0-20230915142004-095d10f3a87a + github.com/matrix-org/gomatrixserverlib v0.0.0-20231122130434-2beadf141c98 github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 github.com/mattn/go-sqlite3 v1.14.17 - github.com/nats-io/nats-server/v2 v2.9.19 - github.com/nats-io/nats.go v1.27.0 + github.com/nats-io/nats-server/v2 v2.9.23 + github.com/nats-io/nats.go v1.28.0 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 github.com/opentracing/opentracing-go v1.2.0 @@ -36,18 +36,18 @@ require ( github.com/prometheus/client_golang v1.16.0 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.8.2 - github.com/tidwall/gjson v1.16.0 + github.com/tidwall/gjson v1.17.0 github.com/tidwall/sjson v1.2.5 github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/uber/jaeger-lib v2.4.1+incompatible github.com/yggdrasil-network/yggdrasil-go v0.4.6 go.uber.org/atomic v1.10.0 - golang.org/x/crypto v0.13.0 + golang.org/x/crypto v0.14.0 golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819 - golang.org/x/image v0.5.0 + golang.org/x/image v0.10.0 golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e golang.org/x/sync v0.3.0 - golang.org/x/term v0.12.0 + golang.org/x/term v0.13.0 gopkg.in/h2non/bimg.v1 v1.1.9 gopkg.in/yaml.v2 v2.4.0 gotest.tools/v3 v3.4.0 @@ -94,7 +94,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/juju/errors v1.0.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/klauspost/compress v1.16.5 // indirect + github.com/klauspost/compress v1.16.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect @@ -104,8 +104,8 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/mschoch/smat v0.2.0 // indirect - github.com/nats-io/jwt/v2 v2.4.1 // indirect - github.com/nats-io/nkeys v0.4.4 // indirect + github.com/nats-io/jwt/v2 v2.5.0 // indirect + github.com/nats-io/nkeys v0.4.6 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/onsi/ginkgo/v2 v2.11.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect @@ -123,8 +123,8 @@ require ( github.com/tidwall/pretty v1.2.1 // indirect go.etcd.io/bbolt v1.3.6 // indirect golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.14.0 // indirect - golang.org/x/sys v0.12.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/sys v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.12.0 // indirect @@ -143,4 +143,4 @@ require ( modernc.org/token v1.0.1 // indirect ) -go 1.18 +go 1.20 diff --git a/go.sum b/go.sum index f7d21d96b9..0dbd3ac482 100644 --- a/go.sum +++ b/go.sum @@ -89,8 +89,8 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczC github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v24.0.5+incompatible h1:WmgcE4fxyI6EEXxBRxsHnZXrO1pQ3smi0k/jho4HLeY= -github.com/docker/docker v24.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM= +github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -190,8 +190,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= -github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -208,8 +208,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20230915142004-095d10f3a87a h1:+RC9Ddmt5v4y58qmdz5WuEEWCJ9gBWuYLyndnWkGfXU= -github.com/matrix-org/gomatrixserverlib v0.0.0-20230915142004-095d10f3a87a/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20231122130434-2beadf141c98 h1:+GsF24sG9WJccf5k54cZj9tLgwW3UON1ujz5u+8SHxc= +github.com/matrix-org/gomatrixserverlib v0.0.0-20231122130434-2beadf141c98/go.mod h1:M8m7seOroO5ePlgxA7AFZymnG90Cnh94rYQyngSrZkk= github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 h1:6t8kJr8i1/1I5nNttw6nn1ryQJgzVlBmSGgPiiaTdw4= github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7/go.mod h1:ReWMS/LoVnOiRAdq9sNUC2NZnd1mZkMNB52QhpTRWjg= github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y= @@ -242,14 +242,14 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= -github.com/nats-io/jwt/v2 v2.4.1 h1:Y35W1dgbbz2SQUYDPCaclXcuqleVmpbRa7646Jf2EX4= -github.com/nats-io/jwt/v2 v2.4.1/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI= -github.com/nats-io/nats-server/v2 v2.9.19 h1:OF9jSKZGo425C/FcVVIvNgpd36CUe7aVTTXEZRJk6kA= -github.com/nats-io/nats-server/v2 v2.9.19/go.mod h1:aTb/xtLCGKhfTFLxP591CMWfkdgBmcUUSkiSOe5A3gw= -github.com/nats-io/nats.go v1.27.0 h1:3o9fsPhmoKm+yK7rekH2GtWoE+D9jFbw8N3/ayI1C00= -github.com/nats-io/nats.go v1.27.0/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc= -github.com/nats-io/nkeys v0.4.4 h1:xvBJ8d69TznjcQl9t6//Q5xXuVhyYiSos6RPtvQNTwA= -github.com/nats-io/nkeys v0.4.4/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64= +github.com/nats-io/jwt/v2 v2.5.0 h1:WQQ40AAlqqfx+f6ku+i0pOVm+ASirD4fUh+oQsiE9Ak= +github.com/nats-io/jwt/v2 v2.5.0/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI= +github.com/nats-io/nats-server/v2 v2.9.23 h1:6Wj6H6QpP9FMlpCyWUaNu2yeZ/qGj+mdRkZ1wbikExU= +github.com/nats-io/nats-server/v2 v2.9.23/go.mod h1:wEjrEy9vnqIGE4Pqz4/c75v9Pmaq7My2IgFmnykc4C0= +github.com/nats-io/nats.go v1.28.0 h1:Th4G6zdsz2d0OqXdfzKLClo6bOfoI/b1kInhRtFIy5c= +github.com/nats-io/nats.go v1.28.0/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc= +github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY= +github.com/nats-io/nkeys v0.4.6/go.mod h1:4DxZNzenSVd1cYQoAa8948QY3QDjrHfcfVADymtkpts= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 h1:lrVQzBtkeQEGGYUHwSX1XPe1E5GL6U3KYCNe2G4bncQ= @@ -318,8 +318,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.16.0 h1:SyXa+dsSPpUlcwEDuKuEBJEz5vzTvOea+9rjyYodQFg= -github.com/tidwall/gjson v1.16.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= +github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= @@ -354,8 +354,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -366,8 +366,8 @@ golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819/go.mod h1:FXUEEKJgO7OQYeo8N0 golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI= -golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= +golang.org/x/image v0.10.0 h1:gXjUUtwtx5yOE0VKWq1CH4IJAClq4UGgUA3i+rpON9M= +golang.org/x/image v0.10.0/go.mod h1:jtrku+n79PfroUbvDdeUWMAI+heR786BofxrbiSF+J0= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e h1:zSgtO19fpg781xknwqiQPmOHaASr6E7ZVlTseLd9Fx4= golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e/go.mod h1:aAjjkJNdrh3PMckS4B10TGS2nag27cbKR1y2BpUxsiY= @@ -376,6 +376,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -386,14 +387,16 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -418,17 +421,20 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -445,6 +451,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/helm/dendrite/Chart.yaml b/helm/dendrite/Chart.yaml index 5590a39b18..32f4799604 100644 --- a/helm/dendrite/Chart.yaml +++ b/helm/dendrite/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: dendrite -version: "0.13.3" -appVersion: "0.13.2" +version: "0.13.5" +appVersion: "0.13.4" description: Dendrite Matrix Homeserver type: application keywords: diff --git a/helm/dendrite/README.md b/helm/dendrite/README.md index 7f7ea484a7..22daa18136 100644 --- a/helm/dendrite/README.md +++ b/helm/dendrite/README.md @@ -1,7 +1,7 @@ # dendrite -![Version: 0.13.2](https://img.shields.io/badge/Version-0.13.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.2](https://img.shields.io/badge/AppVersion-0.13.2-informational?style=flat-square) +![Version: 0.13.5](https://img.shields.io/badge/Version-0.13.5-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.4](https://img.shields.io/badge/AppVersion-0.13.4-informational?style=flat-square) Dendrite Matrix Homeserver Status: **NOT PRODUCTION READY** @@ -48,24 +48,18 @@ Create a folder `appservices` and place your configurations in there. The confi | signing_key.create | bool | `true` | Create a new signing key, if not exists | | signing_key.existingSecret | string | `""` | Use an existing secret | | resources | object | sets some sane default values | Default resource requests/limits. | -| persistence.storageClass | string | `""` | The storage class to use for volume claims. Used unless specified at the specific component. Defaults to the cluster default storage class. | +| persistence.jetstream | object | `{"capacity":"1Gi","existingClaim":""}` | The storage class to use for volume claims. Used unless specified at the specific component. Defaults to the cluster default storage class. # If defined, storageClassName: # If set to "-", storageClassName: "", which disables dynamic provisioning # If undefined (the default) or set to null, no storageClassName spec is # set, choosing the default provisioner. (gp2 on AWS, standard on # GKE, AWS & OpenStack) # storageClass: "" | | persistence.jetstream.existingClaim | string | `""` | Use an existing volume claim for jetstream | | persistence.jetstream.capacity | string | `"1Gi"` | PVC Storage Request for the jetstream volume | -| persistence.jetstream.storageClass | string | `""` | The storage class to use for volume claims. Defaults to persistence.storageClass | | persistence.media.existingClaim | string | `""` | Use an existing volume claim for media files | | persistence.media.capacity | string | `"1Gi"` | PVC Storage Request for the media volume | -| persistence.media.storageClass | string | `""` | The storage class to use for volume claims. Defaults to persistence.storageClass | | persistence.search.existingClaim | string | `""` | Use an existing volume claim for the fulltext search index | | persistence.search.capacity | string | `"1Gi"` | PVC Storage Request for the search volume | -| persistence.search.storageClass | string | `""` | The storage class to use for volume claims. Defaults to persistence.storageClass | | extraVolumes | list | `[]` | Add additional volumes to the Dendrite Pod | | extraVolumeMounts | list | `[]` | Configure additional mount points volumes in the Dendrite Pod | | strategy.type | string | `"RollingUpdate"` | Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) If you are using ReadWriteOnce volumes, you should probably use Recreate | | strategy.rollingUpdate.maxUnavailable | string | `"25%"` | Maximum number of pods that can be unavailable during the update process | | strategy.rollingUpdate.maxSurge | string | `"25%"` | Maximum number of pods that can be scheduled above the desired number of pods | -| strategy.type | string | `"RollingUpdate"` | Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) If you are using ReadWriteOnce volumes, you should probably use Recreate | -| strategy.rollingUpdate.maxUnavailable | string | `"25%"` | Maximum number of pods that can be unavailable during the update process | -| strategy.rollingUpdate.maxSurge | string | `"25%"` | Maximum number of pods that can be scheduled above the desired number of pods | | dendrite_config.version | int | `2` | | | dendrite_config.global.server_name | string | `""` | **REQUIRED** Servername for this Dendrite deployment. | | dendrite_config.global.private_key | string | `"/etc/dendrite/secrets/signing.key"` | The private key to use. (**NOTE**: This is overriden in Helm) | diff --git a/helm/dendrite/grafana_dashboards/dendrite-rev2.json b/helm/dendrite/grafana_dashboards/dendrite-rev2.json index 817f950b34..420d8bf1b8 100644 --- a/helm/dendrite/grafana_dashboards/dendrite-rev2.json +++ b/helm/dendrite/grafana_dashboards/dendrite-rev2.json @@ -119,7 +119,7 @@ "refId": "A" } ], - "title": "Registerd Users", + "title": "Registered Users", "type": "stat" }, { diff --git a/helm/dendrite/templates/pvc.yaml b/helm/dendrite/templates/pvc.yaml index 88eff3bede..70b1ce563a 100644 --- a/helm/dendrite/templates/pvc.yaml +++ b/helm/dendrite/templates/pvc.yaml @@ -12,7 +12,14 @@ spec: resources: requests: storage: {{ .Values.persistence.media.capacity }} - storageClassName: {{ default .Values.persistence.storageClass .Values.persistence.media.storageClass }} + {{ $storageClass := .Values.persistence.media.storageClass | default .Values.persistence.storageClass }} + {{- if $storageClass }} + {{- if (eq "-" $storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ $storageClass }}" + {{- end }} + {{- end }} {{ end }} {{ if not .Values.persistence.jetstream.existingClaim }} --- @@ -28,7 +35,14 @@ spec: resources: requests: storage: {{ .Values.persistence.jetstream.capacity }} - storageClassName: {{ default .Values.persistence.storageClass .Values.persistence.jetstream.storageClass }} + {{ $storageClass := .Values.persistence.jetstream.storageClass | default .Values.persistence.storageClass }} + {{- if $storageClass }} + {{- if (eq "-" $storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ $storageClass }}" + {{- end }} + {{- end }} {{ end }} {{ if not .Values.persistence.search.existingClaim }} --- @@ -44,5 +58,12 @@ spec: resources: requests: storage: {{ .Values.persistence.search.capacity }} - storageClassName: {{ default .Values.persistence.storageClass .Values.persistence.search.storageClass }} + {{ $storageClass := .Values.persistence.search.storageClass | default .Values.persistence.storageClass }} + {{- if $storageClass }} + {{- if (eq "-" $storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ $storageClass }}" + {{- end }} + {{- end }} {{ end }} diff --git a/helm/dendrite/values.yaml b/helm/dendrite/values.yaml index 8a72f66930..afce1d930f 100644 --- a/helm/dendrite/values.yaml +++ b/helm/dendrite/values.yaml @@ -26,7 +26,13 @@ persistence: # -- The storage class to use for volume claims. # Used unless specified at the specific component. # Defaults to the cluster default storage class. - storageClass: "" + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "" jetstream: # -- Use an existing volume claim for jetstream existingClaim: "" @@ -34,7 +40,13 @@ persistence: capacity: "1Gi" # -- The storage class to use for volume claims. # Defaults to persistence.storageClass - storageClass: "" + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "" media: # -- Use an existing volume claim for media files existingClaim: "" @@ -42,7 +54,13 @@ persistence: capacity: "1Gi" # -- The storage class to use for volume claims. # Defaults to persistence.storageClass - storageClass: "" + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "" search: # -- Use an existing volume claim for the fulltext search index existingClaim: "" @@ -50,7 +68,13 @@ persistence: capacity: "1Gi" # -- The storage class to use for volume claims. # Defaults to persistence.storageClass - storageClass: "" + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "" # -- Add additional volumes to the Dendrite Pod extraVolumes: [] diff --git a/internal/pushrules/action_test.go b/internal/pushrules/action_test.go index 72db9c998a..d1eb16ef18 100644 --- a/internal/pushrules/action_test.go +++ b/internal/pushrules/action_test.go @@ -13,8 +13,6 @@ func TestActionJSON(t *testing.T) { Want Action }{ {Action{Kind: NotifyAction}}, - {Action{Kind: DontNotifyAction}}, - {Action{Kind: CoalesceAction}}, {Action{Kind: SetTweakAction}}, {Action{Kind: SetTweakAction, Tweak: SoundTweak, Value: "default"}}, diff --git a/internal/pushrules/default_override.go b/internal/pushrules/default_override.go index f97427b719..cb825af5fd 100644 --- a/internal/pushrules/default_override.go +++ b/internal/pushrules/default_override.go @@ -10,6 +10,7 @@ func defaultOverrideRules(userID string) []*Rule { &mRuleRoomNotifDefinition, &mRuleTombstoneDefinition, &mRuleReactionDefinition, + &mRuleACLsDefinition, } } @@ -30,7 +31,7 @@ var ( RuleID: MRuleMaster, Default: true, Enabled: false, - Actions: []*Action{{Kind: DontNotifyAction}}, + Actions: []*Action{}, } mRuleSuppressNoticesDefinition = Rule{ RuleID: MRuleSuppressNotices, @@ -43,7 +44,7 @@ var ( Pattern: pointer("m.notice"), }, }, - Actions: []*Action{{Kind: DontNotifyAction}}, + Actions: []*Action{}, } mRuleMemberEventDefinition = Rule{ RuleID: MRuleMemberEvent, @@ -56,7 +57,7 @@ var ( Pattern: pointer("m.room.member"), }, }, - Actions: []*Action{{Kind: DontNotifyAction}}, + Actions: []*Action{}, } mRuleContainsDisplayNameDefinition = Rule{ RuleID: MRuleContainsDisplayName, @@ -152,9 +153,7 @@ var ( Pattern: pointer("m.reaction"), }, }, - Actions: []*Action{ - {Kind: DontNotifyAction}, - }, + Actions: []*Action{}, } ) diff --git a/internal/pushrules/default_pushrules_test.go b/internal/pushrules/default_pushrules_test.go index dea8298429..506fe3cc82 100644 --- a/internal/pushrules/default_pushrules_test.go +++ b/internal/pushrules/default_pushrules_test.go @@ -21,12 +21,12 @@ func TestDefaultRules(t *testing.T) { // Default override rules { name: ".m.rule.master", - inputBytes: []byte(`{"rule_id":".m.rule.master","default":true,"enabled":false,"actions":["dont_notify"]}`), + inputBytes: []byte(`{"rule_id":".m.rule.master","default":true,"enabled":false,"actions":[]}`), want: mRuleMasterDefinition, }, { name: ".m.rule.suppress_notices", - inputBytes: []byte(`{"rule_id":".m.rule.suppress_notices","default":true,"enabled":true,"conditions":[{"kind":"event_match","key":"content.msgtype","pattern":"m.notice"}],"actions":["dont_notify"]}`), + inputBytes: []byte(`{"rule_id":".m.rule.suppress_notices","default":true,"enabled":true,"conditions":[{"kind":"event_match","key":"content.msgtype","pattern":"m.notice"}],"actions":[]}`), want: mRuleSuppressNoticesDefinition, }, { @@ -36,7 +36,7 @@ func TestDefaultRules(t *testing.T) { }, { name: ".m.rule.member_event", - inputBytes: []byte(`{"rule_id":".m.rule.member_event","default":true,"enabled":true,"conditions":[{"kind":"event_match","key":"type","pattern":"m.room.member"}],"actions":["dont_notify"]}`), + inputBytes: []byte(`{"rule_id":".m.rule.member_event","default":true,"enabled":true,"conditions":[{"kind":"event_match","key":"type","pattern":"m.room.member"}],"actions":[]}`), want: mRuleMemberEventDefinition, }, { diff --git a/internal/pushrules/util.go b/internal/pushrules/util.go index de8fe5cd0f..e2821b57a5 100644 --- a/internal/pushrules/util.go +++ b/internal/pushrules/util.go @@ -16,10 +16,7 @@ func ActionsToTweaks(as []*Action) (ActionKind, map[string]interface{}, error) { for _, a := range as { switch a.Kind { - case DontNotifyAction: - // Don't bother processing any further - return DontNotifyAction, nil, nil - + case DontNotifyAction: // Ignored case SetTweakAction: if tweaks == nil { tweaks = map[string]interface{}{} diff --git a/internal/pushrules/util_test.go b/internal/pushrules/util_test.go index 89f8243d9b..83eee7ede6 100644 --- a/internal/pushrules/util_test.go +++ b/internal/pushrules/util_test.go @@ -17,17 +17,16 @@ func TestActionsToTweaks(t *testing.T) { {"empty", nil, UnknownAction, nil}, {"zero", []*Action{{}}, UnknownAction, nil}, {"onlyPrimary", []*Action{{Kind: NotifyAction}}, NotifyAction, nil}, - {"onlyPrimaryDontNotify", []*Action{{Kind: DontNotifyAction}}, DontNotifyAction, nil}, + {"onlyPrimaryDontNotify", []*Action{}, UnknownAction, nil}, {"onlyTweak", []*Action{{Kind: SetTweakAction, Tweak: HighlightTweak}}, UnknownAction, map[string]interface{}{"highlight": nil}}, {"onlyTweakWithValue", []*Action{{Kind: SetTweakAction, Tweak: SoundTweak, Value: "default"}}, UnknownAction, map[string]interface{}{"sound": "default"}}, { "all", []*Action{ - {Kind: CoalesceAction}, {Kind: SetTweakAction, Tweak: HighlightTweak}, {Kind: SetTweakAction, Tweak: SoundTweak, Value: "default"}, }, - CoalesceAction, + UnknownAction, map[string]interface{}{"highlight": nil, "sound": "default"}, }, } diff --git a/internal/pushrules/validate.go b/internal/pushrules/validate.go index b54ec3fb0c..4cc4793454 100644 --- a/internal/pushrules/validate.go +++ b/internal/pushrules/validate.go @@ -18,7 +18,7 @@ func ValidateRule(kind Kind, rule *Rule) []error { errs = append(errs, fmt.Errorf("invalid rule ID: %s", rule.RuleID)) } - if len(rule.Actions) == 0 { + if rule.Actions == nil { errs = append(errs, fmt.Errorf("missing actions")) } for _, action := range rule.Actions { diff --git a/internal/validate.go b/internal/validate.go index 99088f2403..da8b35cd3c 100644 --- a/internal/validate.go +++ b/internal/validate.go @@ -20,12 +20,15 @@ import ( "net/http" "regexp" + "github.com/matrix-org/dendrite/clientapi/userutil" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" ) const ( - maxUsernameLength = 254 // http://matrix.org/speculator/spec/HEAD/intro.html#user-identifiers TODO account for domain + maxUsernameLength = 254 // https://spec.matrix.org/v1.7/appendices/#user-identifiers TODO account for domain minPasswordLength = 8 // http://matrix.org/docs/spec/client_server/r0.2.0.html#password-based maxPasswordLength = 512 // https://github.com/matrix-org/synapse/blob/v0.20.0/synapse/rest/client/v2_alpha/register.py#L161 @@ -100,10 +103,139 @@ func UsernameResponse(err error) *util.JSONResponse { // ValidateApplicationServiceUsername returns an error if the username is invalid for an application service func ValidateApplicationServiceUsername(localpart string, domain spec.ServerName) error { - if id := fmt.Sprintf("@%s:%s", localpart, domain); len(id) > maxUsernameLength { + userID := userutil.MakeUserID(localpart, domain) + return ValidateApplicationServiceUserID(userID) +} + +func ValidateApplicationServiceUserID(userID string) error { + if len(userID) > maxUsernameLength { return ErrUsernameTooLong - } else if !validUsernameRegex.MatchString(localpart) { + } + + localpart, _, err := gomatrixserverlib.SplitID('@', userID) + if err != nil || !validUsernameRegex.MatchString(localpart) { return ErrUsernameInvalid } + return nil } + +// userIDIsWithinApplicationServiceNamespace checks to see if a given userID +// falls within any of the namespaces of a given Application Service. If no +// Application Service is given, it will check to see if it matches any +// Application Service's namespace. +func userIDIsWithinApplicationServiceNamespace( + cfg *config.ClientAPI, + userID string, + appservice *config.ApplicationService, +) bool { + var localpart, domain, err = gomatrixserverlib.SplitID('@', userID) + if err != nil { + // Not a valid userID + return false + } + + if !cfg.Matrix.IsLocalServerName(domain) { + // This is a federated userID + return false + } + + if localpart == appservice.SenderLocalpart { + // This is the application service bot userID + return true + } + + // Loop through given application service's namespaces and see if any match + for _, namespace := range appservice.NamespaceMap["users"] { + // Application service namespaces are checked for validity in config + if namespace.RegexpObject.MatchString(userID) { + return true + } + } + + return false +} + +// usernameMatchesMultipleExclusiveNamespaces will check if a given username matches +// more than one exclusive namespace. More than one is not allowed +func userIDMatchesMultipleExclusiveNamespaces( + cfg *config.ClientAPI, + userID string, +) bool { + // Check namespaces and see if more than one match + matchCount := 0 + for _, appservice := range cfg.Derived.ApplicationServices { + if appservice.OwnsNamespaceCoveringUserId(userID) { + if matchCount++; matchCount > 1 { + return true + } + } + } + return false +} + +// ValidateApplicationServiceRequest checks if a provided application service +// token corresponds to one that is registered, and, if so, checks if the +// supplied userIDOrLocalpart is within that application service's namespace. +// +// As long as these two requirements are met, the matched application service +// ID will be returned. Otherwise, it will return a JSON response with the +// appropriate error message. +func ValidateApplicationServiceRequest( + cfg *config.ClientAPI, + userIDOrLocalpart string, + accessToken string, +) (string, *util.JSONResponse) { + localpart, domain, err := userutil.ParseUsernameParam(userIDOrLocalpart, cfg.Matrix) + if err != nil { + return "", &util.JSONResponse{ + Code: http.StatusUnauthorized, + JSON: spec.InvalidUsername(err.Error()), + } + } + + userID := userutil.MakeUserID(localpart, domain) + + // Check if the token if the application service is valid with one we have + // registered in the config. + var matchedApplicationService *config.ApplicationService + for _, appservice := range cfg.Derived.ApplicationServices { + if appservice.ASToken == accessToken { + matchedApplicationService = &appservice + break + } + } + if matchedApplicationService == nil { + return "", &util.JSONResponse{ + Code: http.StatusUnauthorized, + JSON: spec.UnknownToken("Supplied access_token does not match any known application service"), + } + } + + // Ensure the desired username is within at least one of the application service's namespaces. + if !userIDIsWithinApplicationServiceNamespace(cfg, userID, matchedApplicationService) { + // If we didn't find any matches, return M_EXCLUSIVE + return "", &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.ASExclusive(fmt.Sprintf( + "Supplied username %s did not match any namespaces for application service ID: %s", userIDOrLocalpart, matchedApplicationService.ID)), + } + } + + // Check this user does not fit multiple application service namespaces + if userIDMatchesMultipleExclusiveNamespaces(cfg, userID) { + return "", &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.ASExclusive(fmt.Sprintf( + "Supplied username %s matches multiple exclusive application service namespaces. Only 1 match allowed", userIDOrLocalpart)), + } + } + + // Check username application service is trying to register is valid + if err := ValidateApplicationServiceUserID(userID); err != nil { + return "", UsernameResponse(err) + } + + // No errors, registration valid + return matchedApplicationService.ID, nil +} diff --git a/internal/validate_test.go b/internal/validate_test.go index e3a10178fa..cd26261337 100644 --- a/internal/validate_test.go +++ b/internal/validate_test.go @@ -3,9 +3,11 @@ package internal import ( "net/http" "reflect" + "regexp" "strings" "testing" + "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" ) @@ -38,7 +40,7 @@ func Test_validatePassword(t *testing.T) { t.Run(tt.name, func(t *testing.T) { gotErr := ValidatePassword(tt.password) if !reflect.DeepEqual(gotErr, tt.wantError) { - t.Errorf("validatePassword() = %v, wantJSON %v", gotErr, tt.wantError) + t.Errorf("validatePassword() = %v, wantError %v", gotErr, tt.wantError) } if got := PasswordResponse(gotErr); !reflect.DeepEqual(got, tt.wantJSON) { @@ -167,3 +169,133 @@ func Test_validateUsername(t *testing.T) { }) } } + +// This method tests validation of the provided Application Service token and +// username that they're registering +func TestValidateApplicationServiceRequest(t *testing.T) { + // Create a fake application service + regex := "@_appservice_.*" + fakeNamespace := config.ApplicationServiceNamespace{ + Exclusive: true, + Regex: regex, + RegexpObject: regexp.MustCompile(regex), + } + fakeSenderLocalpart := "_appservice_bot" + fakeApplicationService := config.ApplicationService{ + ID: "FakeAS", + URL: "null", + ASToken: "1234", + HSToken: "4321", + SenderLocalpart: fakeSenderLocalpart, + NamespaceMap: map[string][]config.ApplicationServiceNamespace{ + "users": {fakeNamespace}, + }, + } + + // Create a second fake application service where userIDs ending in + // "_overlap" overlap with the first. + regex = "@_.*_overlap" + fakeNamespace = config.ApplicationServiceNamespace{ + Exclusive: true, + Regex: regex, + RegexpObject: regexp.MustCompile(regex), + } + fakeApplicationServiceOverlap := config.ApplicationService{ + ID: "FakeASOverlap", + URL: fakeApplicationService.URL, + ASToken: fakeApplicationService.ASToken, + HSToken: fakeApplicationService.HSToken, + SenderLocalpart: "_appservice_bot_overlap", + NamespaceMap: map[string][]config.ApplicationServiceNamespace{ + "users": {fakeNamespace}, + }, + } + + // Set up a config + fakeConfig := &config.Dendrite{} + fakeConfig.Defaults(config.DefaultOpts{ + Generate: true, + }) + fakeConfig.Global.ServerName = "localhost" + fakeConfig.ClientAPI.Derived.ApplicationServices = []config.ApplicationService{fakeApplicationService, fakeApplicationServiceOverlap} + + tests := []struct { + name string + localpart string + asToken string + wantError bool + wantASID string + }{ + // Access token is correct, userID omitted so we are acting as SenderLocalpart + { + name: "correct access token but omitted userID", + localpart: fakeSenderLocalpart, + asToken: fakeApplicationService.ASToken, + wantError: false, + wantASID: fakeApplicationService.ID, + }, + // Access token is incorrect, userID omitted so we are acting as SenderLocalpart + { + name: "incorrect access token but omitted userID", + localpart: fakeSenderLocalpart, + asToken: "xxxx", + wantError: true, + wantASID: "", + }, + // Access token is correct, acting as valid userID + { + name: "correct access token and valid userID", + localpart: "_appservice_bob", + asToken: fakeApplicationService.ASToken, + wantError: false, + wantASID: fakeApplicationService.ID, + }, + // Access token is correct, acting as invalid userID + { + name: "correct access token but invalid userID", + localpart: "_something_else", + asToken: fakeApplicationService.ASToken, + wantError: true, + wantASID: "", + }, + // Access token is correct, acting as userID that matches two exclusive namespaces + { + name: "correct access token but non-exclusive userID", + localpart: "_appservice_overlap", + asToken: fakeApplicationService.ASToken, + wantError: true, + wantASID: "", + }, + // Access token is correct, acting as matching userID that is too long + { + name: "correct access token but too long userID", + localpart: "_appservice_" + strings.Repeat("a", maxUsernameLength), + asToken: fakeApplicationService.ASToken, + wantError: true, + wantASID: "", + }, + // Access token is correct, acting as userID that matches but is invalid + { + name: "correct access token and matching but invalid userID", + localpart: "@_appservice_bob::", + asToken: fakeApplicationService.ASToken, + wantError: true, + wantASID: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotASID, gotResp := ValidateApplicationServiceRequest(&fakeConfig.ClientAPI, tt.localpart, tt.asToken) + if tt.wantError && gotResp == nil { + t.Error("expected an error, but succeeded") + } + if !tt.wantError && gotResp != nil { + t.Errorf("expected success, but returned error: %v", *gotResp) + } + if gotASID != tt.wantASID { + t.Errorf("returned '%s', but expected '%s'", gotASID, tt.wantASID) + } + }) + } +} diff --git a/internal/version.go b/internal/version.go index 81e0fc5294..55726ddcbb 100644 --- a/internal/version.go +++ b/internal/version.go @@ -18,7 +18,7 @@ var build string const ( VersionMajor = 0 VersionMinor = 13 - VersionPatch = 2 + VersionPatch = 4 VersionTag = "" // example: "rc1" gitRevLen = 7 // 7 matches the displayed characters on github.com diff --git a/relayapi/storage/postgres/relay_queue_json_table.go b/relayapi/storage/postgres/relay_queue_json_table.go index 74410fc885..94ae414077 100644 --- a/relayapi/storage/postgres/relay_queue_json_table.go +++ b/relayapi/storage/postgres/relay_queue_json_table.go @@ -109,5 +109,5 @@ func (s *relayQueueJSONStatements) SelectQueueJSON( } blobs[nid] = blob } - return blobs, err + return blobs, rows.Err() } diff --git a/relayapi/storage/sqlite3/relay_queue_json_table.go b/relayapi/storage/sqlite3/relay_queue_json_table.go index 502da3b007..a1af82aa00 100644 --- a/relayapi/storage/sqlite3/relay_queue_json_table.go +++ b/relayapi/storage/sqlite3/relay_queue_json_table.go @@ -133,5 +133,5 @@ func (s *relayQueueJSONStatements) SelectQueueJSON( } blobs[nid] = blob } - return blobs, err + return blobs, rows.Err() } diff --git a/roomserver/acls/acls.go b/roomserver/acls/acls.go index 601ce90632..e247c7553a 100644 --- a/roomserver/acls/acls.go +++ b/roomserver/acls/acls.go @@ -29,6 +29,8 @@ import ( "github.com/sirupsen/logrus" ) +const MRoomServerACL = "m.room.server_acl" + type ServerACLDatabase interface { // GetKnownRooms returns a list of all rooms we know about. GetKnownRooms(ctx context.Context) ([]string, error) @@ -57,7 +59,7 @@ func NewServerACLs(db ServerACLDatabase) *ServerACLs { // do then we'll process it into memory so that we have the regexes to // hand. for _, room := range rooms { - state, err := db.GetStateEvent(ctx, room, "m.room.server_acl", "") + state, err := db.GetStateEvent(ctx, room, MRoomServerACL, "") if err != nil { logrus.WithError(err).Errorf("Failed to get server ACLs for room %q", room) continue diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index 77b50d0e21..520f82a802 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -33,6 +33,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" + "github.com/matrix-org/dendrite/roomserver/acls" "github.com/matrix-org/dendrite/roomserver/internal/helpers" userAPI "github.com/matrix-org/dendrite/userapi/api" @@ -491,6 +492,27 @@ func (r *Inputer) processRoomEvent( } } + // If this is a membership event, it is possible we newly joined a federated room and eventually + // missed to update our m.room.server_acl - the following ensures we set the ACLs + // TODO: This probably performs badly in benchmarks + if event.Type() == spec.MRoomMember { + membership, _ := event.Membership() + if membership == spec.Join { + _, serverName, _ := gomatrixserverlib.SplitID('@', *event.StateKey()) + // only handle local membership events + if r.Cfg.Matrix.IsLocalServerName(serverName) { + var aclEvent *types.HeaderedEvent + aclEvent, err = r.DB.GetStateEvent(ctx, event.RoomID().String(), acls.MRoomServerACL, "") + if err != nil { + logrus.WithError(err).Error("failed to get server ACLs") + } + if aclEvent != nil { + r.ACLs.OnServerACLUpdate(aclEvent) + } + } + } + } + // Handle remote room upgrades, e.g. remove published room if event.Type() == "m.room.tombstone" && event.StateKeyEquals("") && !r.Cfg.Matrix.IsLocalServerName(senderDomain) { if err = r.handleRemoteRoomUpgrade(ctx, event); err != nil { diff --git a/roomserver/internal/input/input_missing.go b/roomserver/internal/input/input_missing.go index d9ab291e94..21493287e7 100644 --- a/roomserver/internal/input/input_missing.go +++ b/roomserver/internal/input/input_missing.go @@ -498,6 +498,13 @@ func (t *missingStateReq) resolveStatesAndCheck(ctx context.Context, roomVersion roomVersion, gomatrixserverlib.ToPDUs(stateEventList), gomatrixserverlib.ToPDUs(authEventList), func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return t.inputer.Queryer.QueryUserIDForSender(ctx, roomID, senderID) }, + func(eventID string) bool { + isRejected, err := t.db.IsEventRejected(ctx, t.roomInfo.RoomNID, eventID) + if err != nil { + return true + } + return isRejected + }, ) if err != nil { return nil, err diff --git a/roomserver/internal/perform/perform_create_room.go b/roomserver/internal/perform/perform_create_room.go index cd6629d28c..eb8de78110 100644 --- a/roomserver/internal/perform/perform_create_room.go +++ b/roomserver/internal/perform/perform_create_room.go @@ -90,7 +90,16 @@ func (c *Creator) PerformCreateRoom(ctx context.Context, userID spec.UserID, roo } else { senderID = spec.SenderID(userID.String()) } - createContent["creator"] = senderID + + // TODO: Maybe, at some point, GMSL should return the events to create, so we can define the version + // entirely there. + switch createRequest.RoomVersion { + case gomatrixserverlib.RoomVersionV11: + // RoomVersionV11 removed the creator field from the create content: https://github.com/matrix-org/matrix-spec-proposals/pull/2175 + default: + createContent["creator"] = senderID + } + createContent["room_version"] = createRequest.RoomVersion powerLevelContent := eventutil.InitialPowerLevelsContent(string(senderID)) joinRuleContent := gomatrixserverlib.JoinRuleContent{ diff --git a/roomserver/internal/perform/perform_leave.go b/roomserver/internal/perform/perform_leave.go index 5c63a66845..5bea004458 100644 --- a/roomserver/internal/perform/perform_leave.go +++ b/roomserver/internal/perform/perform_leave.go @@ -93,11 +93,21 @@ func (r *Leaver) performLeaveRoomByID( isInvitePending, senderUser, eventID, _, err := helpers.IsInvitePending(ctx, r.DB, req.RoomID, *leaver) if err == nil && isInvitePending { sender, serr := r.RSAPI.QueryUserIDForSender(ctx, *roomID, senderUser) - if serr != nil || sender == nil { - return nil, fmt.Errorf("sender %q has no matching userID", senderUser) + if serr != nil { + return nil, fmt.Errorf("failed looking up userID for sender %q: %w", senderUser, serr) } - if !r.Cfg.Matrix.IsLocalServerName(sender.Domain()) { - return r.performFederatedRejectInvite(ctx, req, res, *sender, eventID, *leaver) + + var domain spec.ServerName + if sender == nil { + // TODO: Currently a federated invite has no way of knowing the mxid_mapping of the inviter. + // Should we add the inviter's m.room.member event (with mxid_mapping) to invite_room_state to allow + // the invited user to leave via the inviter's server? + domain = roomID.Domain() + } else { + domain = sender.Domain() + } + if !r.Cfg.Matrix.IsLocalServerName(domain) { + return r.performFederatedRejectInvite(ctx, req, res, domain, eventID, *leaver) } // check that this is not a "server notice room" accData := &userapi.QueryAccountDataResponse{} @@ -219,14 +229,14 @@ func (r *Leaver) performFederatedRejectInvite( ctx context.Context, req *api.PerformLeaveRequest, res *api.PerformLeaveResponse, // nolint:unparam - inviteSender spec.UserID, eventID string, + inviteDomain spec.ServerName, eventID string, leaver spec.SenderID, ) ([]api.OutputEvent, error) { // Ask the federation sender to perform a federated leave for us. leaveReq := fsAPI.PerformLeaveRequest{ RoomID: req.RoomID, UserID: req.Leaver.String(), - ServerNames: []spec.ServerName{inviteSender.Domain()}, + ServerNames: []spec.ServerName{inviteDomain}, } leaveRes := fsAPI.PerformLeaveResponse{} if err := r.FSAPI.PerformLeave(ctx, &leaveReq, &leaveRes); err != nil { diff --git a/roomserver/internal/perform/perform_upgrade.go b/roomserver/internal/perform/perform_upgrade.go index c32e10d535..9d7c7b5671 100644 --- a/roomserver/internal/perform/perform_upgrade.go +++ b/roomserver/internal/perform/perform_upgrade.go @@ -368,7 +368,16 @@ func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.Query // in the create event (such as for the room types MSC). newCreateContent := map[string]interface{}{} _ = json.Unmarshal(oldCreateEvent.Content(), &newCreateContent) - newCreateContent["creator"] = string(senderID) + + switch newVersion { + case gomatrixserverlib.RoomVersionV11: + // RoomVersionV11 removed the creator field from the create content: https://github.com/matrix-org/matrix-spec-proposals/pull/2175 + // So if we are upgrading from pre v11, we need to remove the field. + delete(newCreateContent, "creator") + default: + newCreateContent["creator"] = senderID + } + newCreateContent["room_version"] = newVersion newCreateContent["predecessor"] = gomatrixserverlib.PreviousRoom{ EventID: tombstoneEvent.EventID(), diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index f87a3f7ed7..74b0102811 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -165,6 +165,13 @@ func (r *Queryer) QueryStateAfterEvents( info.RoomVersion, gomatrixserverlib.ToPDUs(stateEvents), gomatrixserverlib.ToPDUs(authEvents), func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return r.QueryUserIDForSender(ctx, roomID, senderID) }, + func(eventID string) bool { + isRejected, rejectedErr := r.DB.IsEventRejected(ctx, info.RoomNID, eventID) + if rejectedErr != nil { + return true + } + return isRejected + }, ) if err != nil { return fmt.Errorf("state.ResolveConflictsAdhoc: %w", err) @@ -676,6 +683,13 @@ func (r *Queryer) QueryStateAndAuthChain( info.RoomVersion, gomatrixserverlib.ToPDUs(stateEvents), gomatrixserverlib.ToPDUs(authEvents), func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return r.QueryUserIDForSender(ctx, roomID, senderID) }, + func(eventID string) bool { + isRejected, rejectedErr := r.DB.IsEventRejected(ctx, info.RoomNID, eventID) + if rejectedErr != nil { + return true + } + return isRejected + }, ) if err != nil { return err diff --git a/roomserver/internal/query/query_room_hierarchy.go b/roomserver/internal/query/query_room_hierarchy.go index 76eba12bee..5f55980f04 100644 --- a/roomserver/internal/query/query_room_hierarchy.go +++ b/roomserver/internal/query/query_room_hierarchy.go @@ -17,6 +17,7 @@ package query import ( "context" "encoding/json" + "errors" "fmt" "sort" @@ -56,6 +57,12 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r break } + // If the context is canceled, we might still have discovered rooms + // return them to the client and let the client know there _may_ be more rooms. + if errors.Is(ctx.Err(), context.Canceled) { + break + } + // pop the stack queuedRoom := unvisited[len(unvisited)-1] unvisited = unvisited[:len(unvisited)-1] @@ -112,6 +119,11 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r pubRoom := publicRoomsChunk(ctx, querier, queuedRoom.RoomID) + if pubRoom == nil { + util.GetLogger(ctx).WithField("room_id", queuedRoom.RoomID).Debug("unable to get publicRoomsChunk") + continue + } + discoveredRooms = append(discoveredRooms, fclient.RoomHierarchyRoom{ PublicRoom: *pubRoom, RoomType: roomType, diff --git a/roomserver/producers/roomevent.go b/roomserver/producers/roomevent.go index 165304d49c..af7e105806 100644 --- a/roomserver/producers/roomevent.go +++ b/roomserver/producers/roomevent.go @@ -73,7 +73,7 @@ func (r *RoomEventProducer) ProduceRoomEvents(roomID string, updates []api.Outpu } } - if eventType == "m.room.server_acl" && update.NewRoomEvent.Event.StateKeyEquals("") { + if eventType == acls.MRoomServerACL && update.NewRoomEvent.Event.StateKeyEquals("") { ev := update.NewRoomEvent.Event.PDU defer r.ACLs.OnServerACLUpdate(ev) } diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go index 47626b30ac..e9cd926d72 100644 --- a/roomserver/roomserver_test.go +++ b/roomserver/roomserver_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/internal/httputil" @@ -15,6 +16,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/tidwall/gjson" + "github.com/matrix-org/dendrite/roomserver/acls" "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/userapi" @@ -34,6 +36,10 @@ import ( "github.com/matrix-org/dendrite/test/testrig" ) +var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) { + return &statistics.ServerStatistics{}, nil +} + type FakeQuerier struct { api.QuerySenderIDAPI } @@ -58,7 +64,7 @@ func TestUsers(t *testing.T) { }) t.Run("kick users", func(t *testing.T) { - usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) rsAPI.SetUserAPI(usrAPI) testKickUsers(t, rsAPI, usrAPI) }) @@ -258,7 +264,7 @@ func TestPurgeRoom(t *testing.T) { fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, fsAPI.IsBlacklistedOrBackingOff) syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics) // Create the room @@ -407,7 +413,9 @@ type fledglingEvent struct { RoomID string Redacts string Depth int64 - PrevEvents []interface{} + PrevEvents []any + AuthEvents []any + Content map[string]any } func mustCreateEvent(t *testing.T, ev fledglingEvent) (result *types.HeaderedEvent) { @@ -424,7 +432,13 @@ func mustCreateEvent(t *testing.T, ev fledglingEvent) (result *types.HeaderedEve Depth: ev.Depth, PrevEvents: ev.PrevEvents, }) - err := eb.SetContent(map[string]interface{}{}) + if ev.Content == nil { + ev.Content = map[string]any{} + } + if ev.AuthEvents != nil { + eb.AuthEvents = ev.AuthEvents + } + err := eb.SetContent(ev.Content) if err != nil { t.Fatalf("mustCreateEvent: failed to marshal event content %v", err) } @@ -1042,7 +1056,7 @@ func TestUpgrade(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) rsAPI.SetUserAPI(userAPI) for _, tc := range testCases { @@ -1077,3 +1091,143 @@ func TestUpgrade(t *testing.T) { } }) } + +func TestStateReset(t *testing.T) { + alice := test.NewUser(t) + bob := test.NewUser(t) + charlie := test.NewUser(t) + ctx := context.Background() + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + // Prepare APIs + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + defer close() + + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + natsInstance := jetstream.NATSInstance{} + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + + // create a new room + room := test.NewRoom(t, alice, test.RoomPreset(test.PresetPublicChat)) + + // join with Bob and Charlie + bobJoinEv := room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]any{"membership": "join"}, test.WithStateKey(bob.ID)) + charlieJoinEv := room.CreateAndInsert(t, charlie, spec.MRoomMember, map[string]any{"membership": "join"}, test.WithStateKey(charlie.ID)) + + // Send and create the room + if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil { + t.Errorf("failed to send events: %v", err) + } + + // send a message + bobMsg := room.CreateAndInsert(t, bob, "m.room.message", map[string]any{"body": "hello world"}) + charlieMsg := room.CreateAndInsert(t, charlie, "m.room.message", map[string]any{"body": "hello world"}) + + if err := api.SendEvents(ctx, rsAPI, api.KindNew, []*types.HeaderedEvent{bobMsg, charlieMsg}, "test", "test", "test", nil, false); err != nil { + t.Errorf("failed to send events: %v", err) + } + + // Bob changes his name + expectedDisplayname := "Bob!" + bobDisplayname := room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]any{"membership": "join", "displayname": expectedDisplayname}, test.WithStateKey(bob.ID)) + + if err := api.SendEvents(ctx, rsAPI, api.KindNew, []*types.HeaderedEvent{bobDisplayname}, "test", "test", "test", nil, false); err != nil { + t.Errorf("failed to send events: %v", err) + } + + // Change another state event + jrEv := room.CreateAndInsert(t, alice, spec.MRoomJoinRules, gomatrixserverlib.JoinRuleContent{JoinRule: "invite"}, test.WithStateKey("")) + if err := api.SendEvents(ctx, rsAPI, api.KindNew, []*types.HeaderedEvent{jrEv}, "test", "test", "test", nil, false); err != nil { + t.Errorf("failed to send events: %v", err) + } + + // send a message + bobMsg = room.CreateAndInsert(t, bob, "m.room.message", map[string]any{"body": "hello world"}) + charlieMsg = room.CreateAndInsert(t, charlie, "m.room.message", map[string]any{"body": "hello world"}) + + if err := api.SendEvents(ctx, rsAPI, api.KindNew, []*types.HeaderedEvent{bobMsg, charlieMsg}, "test", "test", "test", nil, false); err != nil { + t.Errorf("failed to send events: %v", err) + } + + // Craft the state reset message, which is using Bobs initial join event and the + // last message Charlie sent as the prev_events. This should trigger the recalculation + // of the "current" state, since the message event does not have state and no missing events in the DB. + stateResetMsg := mustCreateEvent(t, fledglingEvent{ + Type: "m.room.message", + SenderID: charlie.ID, + RoomID: room.ID, + Depth: charlieMsg.Depth() + 1, + PrevEvents: []any{ + bobJoinEv.EventID(), + charlieMsg.EventID(), + }, + AuthEvents: []any{ + room.Events()[0].EventID(), // create event + room.Events()[2].EventID(), // PL event + charlieJoinEv.EventID(), // Charlie join event + }, + }) + + // Send the state reset message + if err := api.SendEvents(ctx, rsAPI, api.KindNew, []*types.HeaderedEvent{stateResetMsg}, "test", "test", "test", nil, false); err != nil { + t.Errorf("failed to send events: %v", err) + } + + // Validate that there is a membership event for Bob + bobMembershipEv := api.GetStateEvent(ctx, rsAPI, room.ID, gomatrixserverlib.StateKeyTuple{ + EventType: spec.MRoomMember, + StateKey: bob.ID, + }) + + if bobMembershipEv == nil { + t.Fatalf("Membership event for Bob does not exist. State reset?") + } else { + // Validate it's the correct membership event + if dn := gjson.GetBytes(bobMembershipEv.Content(), "displayname").Str; dn != expectedDisplayname { + t.Fatalf("Expected displayname to be %q, got %q", expectedDisplayname, dn) + } + } + }) +} + +func TestNewServerACLs(t *testing.T) { + alice := test.NewUser(t) + roomWithACL := test.NewRoom(t, alice) + + roomWithACL.CreateAndInsert(t, alice, acls.MRoomServerACL, acls.ServerACL{ + Allowed: []string{"*"}, + Denied: []string{"localhost"}, + AllowIPLiterals: false, + }, test.WithStateKey("")) + + roomWithoutACL := test.NewRoom(t, alice) + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType) + defer closeDB() + + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + natsInstance := &jetstream.NATSInstance{} + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + // start JetStream listeners + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + + // let the RS create the events + err := api.SendEvents(context.Background(), rsAPI, api.KindNew, roomWithACL.Events(), "test", "test", "test", nil, false) + assert.NoError(t, err) + err = api.SendEvents(context.Background(), rsAPI, api.KindNew, roomWithoutACL.Events(), "test", "test", "test", nil, false) + assert.NoError(t, err) + + db, err := storage.Open(processCtx.Context(), cm, &cfg.RoomServer.Database, caches) + assert.NoError(t, err) + // create new server ACLs and verify server is banned/not banned + serverACLs := acls.NewServerACLs(db) + banned := serverACLs.IsServerBannedFromRoom("localhost", roomWithACL.ID) + assert.Equal(t, true, banned) + banned = serverACLs.IsServerBannedFromRoom("localhost", roomWithoutACL.ID) + assert.Equal(t, false, banned) + }) +} diff --git a/roomserver/state/state.go b/roomserver/state/state.go index 1e776ff6c8..dfd439a229 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -45,6 +45,7 @@ type StateResolutionStorage interface { AddState(ctx context.Context, roomNID types.RoomNID, stateBlockNIDs []types.StateBlockNID, state []types.StateEntry) (types.StateSnapshotNID, error) Events(ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, eventNIDs []types.EventNID) ([]types.Event, error) EventsFromIDs(ctx context.Context, roomInfo *types.RoomInfo, eventIDs []string) ([]types.Event, error) + IsEventRejected(ctx context.Context, roomNID types.RoomNID, eventID string) (bool, error) } type StateResolution struct { @@ -1066,6 +1067,13 @@ func (v *StateResolution) resolveConflictsV2( func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return v.Querier.QueryUserIDForSender(ctx, roomID, senderID) }, + func(eventID string) bool { + isRejected, err := v.db.IsEventRejected(ctx, v.roomInfo.RoomNID, eventID) + if err != nil { + return true + } + return isRejected + }, ) }() diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go index a00b4b1d76..1c9cd15994 100644 --- a/roomserver/storage/postgres/events_table.go +++ b/roomserver/storage/postgres/events_table.go @@ -249,6 +249,7 @@ func (s *eventStatements) BulkSelectSnapshotsFromEventIDs( if err != nil { return nil, err } + defer internal.CloseAndLogIfError(ctx, rows, "BulkSelectSnapshotsFromEventIDs: rows.close() failed") var eventID string var stateNID types.StateSnapshotNID @@ -563,7 +564,7 @@ func (s *eventStatements) SelectRoomNIDsForEventNIDs( } result[eventNID] = roomNID } - return result, nil + return result, rows.Err() } func eventNIDsAsArray(eventNIDs []types.EventNID) pq.Int64Array { diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go index 835a43b2d9..1a96e35270 100644 --- a/roomserver/storage/postgres/membership_table.go +++ b/roomserver/storage/postgres/membership_table.go @@ -363,7 +363,7 @@ func (s *membershipStatements) SelectRoomsWithMembership( } roomNIDs = append(roomNIDs, roomNID) } - return roomNIDs, nil + return roomNIDs, rows.Err() } func (s *membershipStatements) SelectJoinedUsersSetForRooms( diff --git a/roomserver/storage/postgres/rooms_table.go b/roomserver/storage/postgres/rooms_table.go index c8346733df..bc3820b2c8 100644 --- a/roomserver/storage/postgres/rooms_table.go +++ b/roomserver/storage/postgres/rooms_table.go @@ -137,7 +137,7 @@ func (s *roomStatements) SelectRoomIDsWithEvents(ctx context.Context, txn *sql.T } roomIDs = append(roomIDs, roomID) } - return roomIDs, nil + return roomIDs, rows.Err() } func (s *roomStatements) InsertRoomNID( ctx context.Context, txn *sql.Tx, @@ -255,7 +255,7 @@ func (s *roomStatements) SelectRoomVersionsForRoomNIDs( } result[roomNID] = roomVersion } - return result, nil + return result, rows.Err() } func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID) ([]string, error) { @@ -277,7 +277,7 @@ func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roo } roomIDs = append(roomIDs, roomID) } - return roomIDs, nil + return roomIDs, rows.Err() } func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, roomIDs []string) ([]types.RoomNID, error) { @@ -299,7 +299,7 @@ func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, ro } roomNIDs = append(roomNIDs, roomNID) } - return roomNIDs, nil + return roomNIDs, rows.Err() } func roomNIDsAsArray(roomNIDs []types.RoomNID) pq.Int64Array { diff --git a/roomserver/storage/postgres/user_room_keys_table.go b/roomserver/storage/postgres/user_room_keys_table.go index 217ee957f4..57e8f213bb 100644 --- a/roomserver/storage/postgres/user_room_keys_table.go +++ b/roomserver/storage/postgres/user_room_keys_table.go @@ -162,6 +162,7 @@ func (s *userRoomKeysStatements) SelectAllPublicKeysForUser(ctx context.Context, if errors.Is(err, sql.ErrNoRows) { return nil, nil } + defer internal.CloseAndLogIfError(ctx, rows, "SelectAllPublicKeysForUser: failed to close rows") resultMap := make(map[types.RoomNID]ed25519.PublicKey) @@ -173,5 +174,5 @@ func (s *userRoomKeysStatements) SelectAllPublicKeysForUser(ctx context.Context, } resultMap[roomNID] = pubkey } - return resultMap, err + return resultMap, rows.Err() } diff --git a/roomserver/storage/shared/room_updater.go b/roomserver/storage/shared/room_updater.go index 70672a33e1..06284d2e33 100644 --- a/roomserver/storage/shared/room_updater.go +++ b/roomserver/storage/shared/room_updater.go @@ -250,3 +250,7 @@ func (u *RoomUpdater) MarkEventAsSent(eventNID types.EventNID) error { func (u *RoomUpdater) MembershipUpdater(targetUserNID types.EventStateKeyNID, targetLocal bool) (*MembershipUpdater, error) { return u.d.membershipUpdaterTxn(u.ctx, u.txn, u.roomInfo.RoomNID, targetUserNID, targetLocal) } + +func (u *RoomUpdater) IsEventRejected(ctx context.Context, roomNID types.RoomNID, eventID string) (bool, error) { + return u.d.IsEventRejected(ctx, roomNID, eventID) +} diff --git a/roomserver/storage/sqlite3/event_json_table.go b/roomserver/storage/sqlite3/event_json_table.go index dc26885bb4..325951c7fb 100644 --- a/roomserver/storage/sqlite3/event_json_table.go +++ b/roomserver/storage/sqlite3/event_json_table.go @@ -109,5 +109,5 @@ func (s *eventJSONStatements) BulkSelectEventJSON( } result.EventNID = types.EventNID(eventNID) } - return results[:i], nil + return results[:i], rows.Err() } diff --git a/roomserver/storage/sqlite3/event_state_keys_table.go b/roomserver/storage/sqlite3/event_state_keys_table.go index 347524a812..a052d69ed2 100644 --- a/roomserver/storage/sqlite3/event_state_keys_table.go +++ b/roomserver/storage/sqlite3/event_state_keys_table.go @@ -136,7 +136,7 @@ func (s *eventStateKeyStatements) BulkSelectEventStateKeyNID( } result[stateKey] = types.EventStateKeyNID(stateKeyNID) } - return result, nil + return result, rows.Err() } func (s *eventStateKeyStatements) BulkSelectEventStateKey( @@ -167,5 +167,5 @@ func (s *eventStateKeyStatements) BulkSelectEventStateKey( } result[types.EventStateKeyNID(stateKeyNID)] = stateKey } - return result, nil + return result, rows.Err() } diff --git a/roomserver/storage/sqlite3/event_types_table.go b/roomserver/storage/sqlite3/event_types_table.go index 0581ec1944..c030fffea7 100644 --- a/roomserver/storage/sqlite3/event_types_table.go +++ b/roomserver/storage/sqlite3/event_types_table.go @@ -147,5 +147,5 @@ func (s *eventTypeStatements) BulkSelectEventTypeNID( } result[eventType] = types.EventTypeNID(eventTypeNID) } - return result, nil + return result, rows.Err() } diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index c49c6dc38a..2c269bced1 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -310,6 +310,9 @@ func (s *eventStatements) BulkSelectStateEventByID( } results = append(results, result) } + if err = rows.Err(); err != nil { + return nil, err + } if !excludeRejected && i != len(eventIDs) { // If there are fewer rows returned than IDs then we were asked to lookup event IDs we don't have. // We don't know which ones were missing because we don't return the string IDs in the query. @@ -377,7 +380,7 @@ func (s *eventStatements) BulkSelectStateEventByNID( return nil, err } } - return results[:i], err + return results[:i], rows.Err() } // bulkSelectStateAtEventByID lookups the state at a list of events by event ID. @@ -425,6 +428,9 @@ func (s *eventStatements) BulkSelectStateAtEventByID( ) } } + if err = rows.Err(); err != nil { + return nil, err + } if i != len(eventIDs) { return nil, types.MissingEventError( fmt.Sprintf("storage: event IDs missing from the database (%d != %d)", i, len(eventIDs)), @@ -507,6 +513,9 @@ func (s *eventStatements) BulkSelectStateAtEventAndReference( result.BeforeStateSnapshotNID = types.StateSnapshotNID(stateSnapshotNID) result.EventID = eventID } + if err = rows.Err(); err != nil { + return nil, err + } if i != len(eventNIDs) { return nil, fmt.Errorf("storage: event NIDs missing from the database (%d != %d)", i, len(eventNIDs)) } @@ -544,6 +553,9 @@ func (s *eventStatements) BulkSelectEventID(ctx context.Context, txn *sql.Tx, ev } results[types.EventNID(eventNID)] = eventID } + if err = rows.Err(); err != nil { + return nil, err + } if i != len(eventNIDs) { return nil, fmt.Errorf("storage: event NIDs missing from the database (%d != %d)", i, len(eventNIDs)) } @@ -602,7 +614,7 @@ func (s *eventStatements) bulkSelectEventNID(ctx context.Context, txn *sql.Tx, e RoomNID: types.RoomNID(roomNID), } } - return results, nil + return results, rows.Err() } func (s *eventStatements) SelectMaxEventDepth(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) (int64, error) { @@ -652,7 +664,7 @@ func (s *eventStatements) SelectRoomNIDsForEventNIDs( } result[eventNID] = roomNID } - return result, nil + return result, rows.Err() } func eventNIDsAsArray(eventNIDs []types.EventNID) string { diff --git a/roomserver/storage/sqlite3/invite_table.go b/roomserver/storage/sqlite3/invite_table.go index ca6e7c5110..b678d8add9 100644 --- a/roomserver/storage/sqlite3/invite_table.go +++ b/roomserver/storage/sqlite3/invite_table.go @@ -126,6 +126,9 @@ func (s *inviteStatements) UpdateInviteRetired( } eventIDs = append(eventIDs, inviteEventID) } + if err = rows.Err(); err != nil { + return + } // now retire the invites stmt = sqlutil.TxStmt(txn, s.updateInviteRetiredStmt) _, err = stmt.ExecContext(ctx, roomNID, targetUserNID) @@ -157,5 +160,5 @@ func (s *inviteStatements) SelectInviteActiveForUserInRoom( result = append(result, types.EventStateKeyNID(senderUserNID)) eventIDs = append(eventIDs, eventID) } - return result, eventIDs, eventJSON, nil + return result, eventIDs, eventJSON, rows.Err() } diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go index 977788d505..1012c074a7 100644 --- a/roomserver/storage/sqlite3/membership_table.go +++ b/roomserver/storage/sqlite3/membership_table.go @@ -250,6 +250,7 @@ func (s *membershipStatements) SelectMembershipsFromRoom( } eventNIDs = append(eventNIDs, eNID) } + err = rows.Err() return } @@ -277,6 +278,7 @@ func (s *membershipStatements) SelectMembershipsFromRoomAndMembership( } eventNIDs = append(eventNIDs, eNID) } + err = rows.Err() return } @@ -313,7 +315,7 @@ func (s *membershipStatements) SelectRoomsWithMembership( } roomNIDs = append(roomNIDs, roomNID) } - return roomNIDs, nil + return roomNIDs, rows.Err() } func (s *membershipStatements) SelectJoinedUsersSetForRooms(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID, localOnly bool) (map[types.EventStateKeyNID]int, error) { diff --git a/roomserver/storage/sqlite3/room_aliases_table.go b/roomserver/storage/sqlite3/room_aliases_table.go index 3bdbbaa352..815b42a279 100644 --- a/roomserver/storage/sqlite3/room_aliases_table.go +++ b/roomserver/storage/sqlite3/room_aliases_table.go @@ -121,7 +121,7 @@ func (s *roomAliasesStatements) SelectAliasesFromRoomID( aliases = append(aliases, alias) } - + err = rows.Err() return } diff --git a/roomserver/storage/sqlite3/rooms_table.go b/roomserver/storage/sqlite3/rooms_table.go index 7556b3461f..22700a7103 100644 --- a/roomserver/storage/sqlite3/rooms_table.go +++ b/roomserver/storage/sqlite3/rooms_table.go @@ -128,7 +128,7 @@ func (s *roomStatements) SelectRoomIDsWithEvents(ctx context.Context, txn *sql.T } roomIDs = append(roomIDs, roomID) } - return roomIDs, nil + return roomIDs, rows.Err() } func (s *roomStatements) SelectRoomInfo(ctx context.Context, txn *sql.Tx, roomID string) (*types.RoomInfo, error) { @@ -265,7 +265,7 @@ func (s *roomStatements) SelectRoomVersionsForRoomNIDs( } result[roomNID] = roomVersion } - return result, nil + return result, rows.Err() } func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID) ([]string, error) { @@ -293,7 +293,7 @@ func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roo } roomIDs = append(roomIDs, roomID) } - return roomIDs, nil + return roomIDs, rows.Err() } func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, roomIDs []string) ([]types.RoomNID, error) { @@ -321,5 +321,5 @@ func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, ro } roomNIDs = append(roomNIDs, roomNID) } - return roomNIDs, nil + return roomNIDs, rows.Err() } diff --git a/roomserver/storage/sqlite3/state_snapshot_table.go b/roomserver/storage/sqlite3/state_snapshot_table.go index 2edff0ba8a..dcac0b07c4 100644 --- a/roomserver/storage/sqlite3/state_snapshot_table.go +++ b/roomserver/storage/sqlite3/state_snapshot_table.go @@ -133,13 +133,16 @@ func (s *stateSnapshotStatements) BulkSelectStateBlockNIDs( var stateBlockNIDsJSON string for ; rows.Next(); i++ { result := &results[i] - if err := rows.Scan(&result.StateSnapshotNID, &stateBlockNIDsJSON); err != nil { + if err = rows.Scan(&result.StateSnapshotNID, &stateBlockNIDsJSON); err != nil { return nil, err } - if err := json.Unmarshal([]byte(stateBlockNIDsJSON), &result.StateBlockNIDs); err != nil { + if err = json.Unmarshal([]byte(stateBlockNIDsJSON), &result.StateBlockNIDs); err != nil { return nil, err } } + if err = rows.Err(); err != nil { + return nil, err + } if i != len(stateNIDs) { return nil, types.MissingStateError(fmt.Sprintf("storage: state NIDs missing from the database (%d != %d)", i, len(stateNIDs))) } diff --git a/roomserver/storage/sqlite3/user_room_keys_table.go b/roomserver/storage/sqlite3/user_room_keys_table.go index 434bad2959..13906f771b 100644 --- a/roomserver/storage/sqlite3/user_room_keys_table.go +++ b/roomserver/storage/sqlite3/user_room_keys_table.go @@ -177,6 +177,7 @@ func (s *userRoomKeysStatements) SelectAllPublicKeysForUser(ctx context.Context, if errors.Is(err, sql.ErrNoRows) { return nil, nil } + defer internal.CloseAndLogIfError(ctx, rows, "SelectAllPublicKeysForUser: failed to close rows") resultMap := make(map[types.RoomNID]ed25519.PublicKey) @@ -188,5 +189,5 @@ func (s *userRoomKeysStatements) SelectAllPublicKeysForUser(ctx context.Context, } resultMap[roomNID] = pubkey } - return resultMap, err + return resultMap, rows.Err() } diff --git a/roomserver/storage/tables/interface_test.go b/roomserver/storage/tables/interface_test.go new file mode 100644 index 0000000000..8727e24363 --- /dev/null +++ b/roomserver/storage/tables/interface_test.go @@ -0,0 +1,76 @@ +package tables + +import ( + "testing" + + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/stretchr/testify/assert" +) + +func TestExtractContentValue(t *testing.T) { + alice := test.NewUser(t) + room := test.NewRoom(t, alice) + + tests := []struct { + name string + event *types.HeaderedEvent + want string + }{ + { + name: "returns creator ID for create events", + event: room.Events()[0], + want: alice.ID, + }, + { + name: "returns the alias for canonical alias events", + event: room.CreateEvent(t, alice, spec.MRoomCanonicalAlias, map[string]string{"alias": "#test:test"}), + want: "#test:test", + }, + { + name: "returns the history_visibility for history visibility events", + event: room.CreateEvent(t, alice, spec.MRoomHistoryVisibility, map[string]string{"history_visibility": "shared"}), + want: "shared", + }, + { + name: "returns the join rules for join_rules events", + event: room.CreateEvent(t, alice, spec.MRoomJoinRules, map[string]string{"join_rule": "public"}), + want: "public", + }, + { + name: "returns the membership for room_member events", + event: room.CreateEvent(t, alice, spec.MRoomMember, map[string]string{"membership": "join"}, test.WithStateKey(alice.ID)), + want: "join", + }, + { + name: "returns the room name for room_name events", + event: room.CreateEvent(t, alice, spec.MRoomName, map[string]string{"name": "testing"}, test.WithStateKey(alice.ID)), + want: "testing", + }, + { + name: "returns the room avatar for avatar events", + event: room.CreateEvent(t, alice, spec.MRoomAvatar, map[string]string{"url": "mxc://testing"}, test.WithStateKey(alice.ID)), + want: "mxc://testing", + }, + { + name: "returns the room topic for topic events", + event: room.CreateEvent(t, alice, spec.MRoomTopic, map[string]string{"topic": "testing"}, test.WithStateKey(alice.ID)), + want: "testing", + }, + { + name: "returns guest_access for guest access events", + event: room.CreateEvent(t, alice, "m.room.guest_access", map[string]string{"guest_access": "forbidden"}, test.WithStateKey(alice.ID)), + want: "forbidden", + }, + { + name: "returns empty string if key can't be found or unknown event", + event: room.CreateEvent(t, alice, "idontexist", nil), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, ExtractContentValue(tt.event), "ExtractContentValue(%v)", tt.event) + }) + } +} diff --git a/setup/config/config_clientapi.go b/setup/config/config_clientapi.go index 44136e2a08..85dfe0beb7 100644 --- a/setup/config/config_clientapi.go +++ b/setup/config/config_clientapi.go @@ -93,17 +93,15 @@ func (c *ClientAPI) Verify(configErrs *ConfigErrors) { checkNotEmpty(configErrs, "client_api.recaptcha_sitekey_class", c.RecaptchaSitekeyClass) } // Ensure there is any spam counter measure when enabling registration - if !c.RegistrationDisabled && !c.OpenRegistrationWithoutVerificationEnabled { - if !c.RecaptchaEnabled { - configErrs.Add( - "You have tried to enable open registration without any secondary verification methods " + - "(such as reCAPTCHA). By enabling open registration, you are SIGNIFICANTLY " + - "increasing the risk that your server will be used to send spam or abuse, and may result in " + - "your server being banned from some rooms. If you are ABSOLUTELY CERTAIN you want to do this, " + - "start Dendrite with the -really-enable-open-registration command line flag. Otherwise, you " + - "should set the registration_disabled option in your Dendrite config.", - ) - } + if !c.RegistrationDisabled && !c.OpenRegistrationWithoutVerificationEnabled && !c.RecaptchaEnabled { + configErrs.Add( + "You have tried to enable open registration without any secondary verification methods " + + "(such as reCAPTCHA). By enabling open registration, you are SIGNIFICANTLY " + + "increasing the risk that your server will be used to send spam or abuse, and may result in " + + "your server being banned from some rooms. If you are ABSOLUTELY CERTAIN you want to do this, " + + "start Dendrite with the -really-enable-open-registration command line flag. Otherwise, you " + + "should set the registration_disabled option in your Dendrite config.", + ) } } diff --git a/setup/config/config_userapi.go b/setup/config/config_userapi.go index e64a3910c3..559de72ac2 100644 --- a/setup/config/config_userapi.go +++ b/setup/config/config_userapi.go @@ -21,6 +21,10 @@ type UserAPI struct { // Users who register on this homeserver will automatically // be joined to the rooms listed under this option. AutoJoinRooms []string `yaml:"auto_join_rooms"` + + // The number of workers to start for the DeviceListUpdater. Defaults to 8. + // This only needs updating if the "InputDeviceListUpdate" stream keeps growing indefinitely. + WorkerCount int `yaml:"worker_count"` } const DefaultOpenIDTokenLifetimeMS = 3600000 // 60 minutes @@ -28,6 +32,7 @@ const DefaultOpenIDTokenLifetimeMS = 3600000 // 60 minutes func (c *UserAPI) Defaults(opts DefaultOpts) { c.BCryptCost = bcrypt.DefaultCost c.OpenIDTokenLifetimeMS = DefaultOpenIDTokenLifetimeMS + c.WorkerCount = 8 if opts.Generate { if !opts.SingleDatabase { c.AccountDatabase.ConnectionString = "file:userapi_accounts.db" diff --git a/setup/mscs/msc2836/storage.go b/setup/mscs/msc2836/storage.go index ade2a1616c..696d0b0dad 100644 --- a/setup/mscs/msc2836/storage.go +++ b/setup/mscs/msc2836/storage.go @@ -301,7 +301,7 @@ func (p *DB) ChildrenForParent(ctx context.Context, eventID, relType string, rec } children = append(children, evInfo) } - return children, nil + return children, rows.Err() } func (p *DB) ParentForChild(ctx context.Context, eventID, relType string) (*eventInfo, error) { diff --git a/syncapi/internal/history_visibility_test.go b/syncapi/internal/history_visibility_test.go index 984f90edd7..24515bbb2d 100644 --- a/syncapi/internal/history_visibility_test.go +++ b/syncapi/internal/history_visibility_test.go @@ -58,7 +58,7 @@ type mockDB struct { roomID string } -func (s *mockDB) SelectMembershipForUser(ctx context.Context, roomID string, userID string, pos int64) (string, int, error) { +func (s *mockDB) SelectMembershipForUser(ctx context.Context, roomID string, userID string, pos int64) (string, int64, error) { if roomID == s.roomID { membership, ok := s.currentMembership[userID] if !ok { diff --git a/syncapi/routing/getevent.go b/syncapi/routing/getevent.go index c089539f05..d0227f4ea1 100644 --- a/syncapi/routing/getevent.go +++ b/syncapi/routing/getevent.go @@ -44,7 +44,7 @@ func GetEvent( rsAPI api.SyncRoomserverAPI, ) util.JSONResponse { ctx := req.Context() - db, err := syncDB.NewDatabaseTransaction(ctx) + db, err := syncDB.NewDatabaseSnapshot(ctx) logger := util.GetLogger(ctx).WithFields(logrus.Fields{ "event_id": eventID, "room_id": rawRoomID, @@ -56,6 +56,7 @@ func GetEvent( JSON: spec.InternalServerError{}, } } + defer db.Rollback() // nolint: errcheck roomID, err := spec.NewRoomID(rawRoomID) if err != nil { diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index dca5d1a143..97c781b9b1 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -107,7 +107,7 @@ type DatabaseTransaction interface { // SelectMembershipForUser returns the membership of the user before and including the given position. If no membership can be found // returns "leave", the topological position and no error. If an error occurs, other than sql.ErrNoRows, returns that and an empty // string as the membership. - SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int, err error) + SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int64, err error) // getUserUnreadNotificationCountsForRooms returns the unread notifications for the given rooms GetUserUnreadNotificationCountsForRooms(ctx context.Context, userID string, roomIDs map[string]string) (map[string]*eventutil.NotificationData, error) GetPresences(ctx context.Context, userID []string) ([]*types.PresenceInternal, error) diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index b0148bef54..ec0b27adce 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -392,7 +392,7 @@ func currentRoomStateRowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, er }) } - return events, nil + return events, rows.Err() } func rowsToEvents(rows *sql.Rows) ([]*rstypes.HeaderedEvent, error) { diff --git a/syncapi/storage/postgres/memberships_table.go b/syncapi/storage/postgres/memberships_table.go index fcbe14b16b..e5208b8919 100644 --- a/syncapi/storage/postgres/memberships_table.go +++ b/syncapi/storage/postgres/memberships_table.go @@ -19,6 +19,7 @@ import ( "database/sql" "fmt" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" rstypes "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/syncapi/storage/tables" @@ -131,7 +132,7 @@ func (s *membershipsStatements) SelectMembershipCount( // string as the membership. func (s *membershipsStatements) SelectMembershipForUser( ctx context.Context, txn *sql.Tx, roomID, userID string, pos int64, -) (membership string, topologyPos int, err error) { +) (membership string, topologyPos int64, err error) { stmt := sqlutil.TxStmt(txn, s.selectMembershipForUserStmt) err = stmt.QueryRowContext(ctx, roomID, userID, pos).Scan(&membership, &topologyPos) if err != nil { @@ -160,6 +161,7 @@ func (s *membershipsStatements) SelectMemberships( if err != nil { return } + defer internal.CloseAndLogIfError(ctx, rows, "SelectMemberships: failed to close rows") var ( eventID string ) diff --git a/syncapi/storage/postgres/peeks_table.go b/syncapi/storage/postgres/peeks_table.go index 64183073d4..1120dce0b3 100644 --- a/syncapi/storage/postgres/peeks_table.go +++ b/syncapi/storage/postgres/peeks_table.go @@ -164,7 +164,7 @@ func (s *peekStatements) SelectPeekingDevices( devices = append(devices, types.PeekingDevice{UserID: userID, DeviceID: deviceID}) result[roomID] = devices } - return result, nil + return result, rows.Err() } func (s *peekStatements) SelectMaxPeekID( diff --git a/syncapi/storage/postgres/presence_table.go b/syncapi/storage/postgres/presence_table.go index f37b5331ed..53acecce52 100644 --- a/syncapi/storage/postgres/presence_table.go +++ b/syncapi/storage/postgres/presence_table.go @@ -144,7 +144,7 @@ func (p *presenceStatements) GetPresenceForUsers( presence.ClientFields.Presence = presence.Presence.String() result = append(result, presence) } - return result, err + return result, rows.Err() } func (p *presenceStatements) GetMaxPresenceID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { diff --git a/syncapi/storage/shared/storage_consumer.go b/syncapi/storage/shared/storage_consumer.go index 0f4080d53f..923ead9bdd 100644 --- a/syncapi/storage/shared/storage_consumer.go +++ b/syncapi/storage/shared/storage_consumer.go @@ -583,7 +583,7 @@ func (d *Database) GetPresences(ctx context.Context, userIDs []string) ([]*types return d.Presence.GetPresenceForUsers(ctx, nil, userIDs) } -func (d *Database) SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int, err error) { +func (d *Database) SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int64, err error) { return d.Memberships.SelectMembershipForUser(ctx, nil, roomID, userID, pos) } diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index 78b2e397cf..f430fcc052 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -177,7 +177,7 @@ func (s *currentRoomStateStatements) SelectJoinedUsers( users = append(users, userID) result[roomID] = users } - return result, nil + return result, rows.Err() } // SelectJoinedUsersInRoom returns a map of room ID to a list of joined user IDs for a given room. @@ -236,7 +236,7 @@ func (s *currentRoomStateStatements) SelectRoomIDsWithMembership( } result = append(result, roomID) } - return result, nil + return result, rows.Err() } // SelectRoomIDsWithAnyMembership returns a map of all memberships for the given user. @@ -419,7 +419,7 @@ func currentRoomStateRowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, er }) } - return events, nil + return events, rows.Err() } func rowsToEvents(rows *sql.Rows) ([]*rstypes.HeaderedEvent, error) { diff --git a/syncapi/storage/sqlite3/invites_table.go b/syncapi/storage/sqlite3/invites_table.go index ebb469d24a..e50b5cbf81 100644 --- a/syncapi/storage/sqlite3/invites_table.go +++ b/syncapi/storage/sqlite3/invites_table.go @@ -176,7 +176,7 @@ func (s *inviteEventsStatements) SelectInviteEventsInRange( if lastPos == 0 { lastPos = r.To } - return result, retired, lastPos, nil + return result, retired, lastPos, rows.Err() } func (s *inviteEventsStatements) SelectMaxInviteID( diff --git a/syncapi/storage/sqlite3/memberships_table.go b/syncapi/storage/sqlite3/memberships_table.go index 05f756fdae..9e50422e5f 100644 --- a/syncapi/storage/sqlite3/memberships_table.go +++ b/syncapi/storage/sqlite3/memberships_table.go @@ -19,6 +19,7 @@ import ( "database/sql" "fmt" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" rstypes "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/syncapi/storage/tables" @@ -134,7 +135,7 @@ func (s *membershipsStatements) SelectMembershipCount( // string as the membership. func (s *membershipsStatements) SelectMembershipForUser( ctx context.Context, txn *sql.Tx, roomID, userID string, pos int64, -) (membership string, topologyPos int, err error) { +) (membership string, topologyPos int64, err error) { stmt := sqlutil.TxStmt(txn, s.selectMembershipForUserStmt) err = stmt.QueryRowContext(ctx, roomID, userID, pos).Scan(&membership, &topologyPos) if err != nil { @@ -163,6 +164,7 @@ func (s *membershipsStatements) SelectMemberships( if err != nil { return } + defer internal.CloseAndLogIfError(ctx, rows, "SelectMemberships: failed to close rows") var eventID string for rows.Next() { if err = rows.Scan(&eventID); err != nil { diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 93caee8063..c7b11d3eff 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -274,7 +274,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( } } - return stateNeeded, eventIDToEvent, nil + return stateNeeded, eventIDToEvent, rows.Err() } // MaxID returns the ID of the last inserted event in this table. 'txn' is optional. If it is not supplied, @@ -520,7 +520,7 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { ExcludeFromSync: excludeFromSync, }) } - return result, nil + return result, rows.Err() } func (s *outputRoomEventsStatements) SelectContextEvent( ctx context.Context, txn *sql.Tx, roomID, eventID string, diff --git a/syncapi/storage/sqlite3/output_room_events_topology_table.go b/syncapi/storage/sqlite3/output_room_events_topology_table.go index 36967d1e73..c00fb7a796 100644 --- a/syncapi/storage/sqlite3/output_room_events_topology_table.go +++ b/syncapi/storage/sqlite3/output_room_events_topology_table.go @@ -18,6 +18,7 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" rstypes "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/syncapi/storage/tables" @@ -137,6 +138,7 @@ func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange( } else if err != nil { return } + defer internal.CloseAndLogIfError(ctx, rows, "SelectEventIDsInRange: failed to close rows") // Return the IDs. var eventID string @@ -155,7 +157,7 @@ func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange( start = tokens[0] end = tokens[len(tokens)-1] } - + err = rows.Err() return } diff --git a/syncapi/storage/sqlite3/peeks_table.go b/syncapi/storage/sqlite3/peeks_table.go index 5d5200abcf..d8998e2b81 100644 --- a/syncapi/storage/sqlite3/peeks_table.go +++ b/syncapi/storage/sqlite3/peeks_table.go @@ -184,7 +184,7 @@ func (s *peekStatements) SelectPeekingDevices( devices = append(devices, types.PeekingDevice{UserID: userID, DeviceID: deviceID}) result[roomID] = devices } - return result, nil + return result, rows.Err() } func (s *peekStatements) SelectMaxPeekID( diff --git a/syncapi/storage/sqlite3/presence_table.go b/syncapi/storage/sqlite3/presence_table.go index 573fbad6c8..40b57e75de 100644 --- a/syncapi/storage/sqlite3/presence_table.go +++ b/syncapi/storage/sqlite3/presence_table.go @@ -169,7 +169,7 @@ func (p *presenceStatements) GetPresenceForUsers( presence.ClientFields.Presence = presence.Presence.String() result = append(result, presence) } - return result, err + return result, rows.Err() } func (p *presenceStatements) GetMaxPresenceID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index f5c66c42d0..45117d6d30 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -194,7 +194,7 @@ type Receipts interface { type Memberships interface { UpsertMembership(ctx context.Context, txn *sql.Tx, event *rstypes.HeaderedEvent, streamPos, topologicalPos types.StreamPosition) error SelectMembershipCount(ctx context.Context, txn *sql.Tx, roomID, membership string, pos types.StreamPosition) (count int, err error) - SelectMembershipForUser(ctx context.Context, txn *sql.Tx, roomID, userID string, pos int64) (membership string, topologicalPos int, err error) + SelectMembershipForUser(ctx context.Context, txn *sql.Tx, roomID, userID string, pos int64) (membership string, topologicalPos int64, err error) PurgeMemberships(ctx context.Context, txn *sql.Tx, roomID string) error SelectMemberships( ctx context.Context, txn *sql.Tx, diff --git a/syncapi/storage/tables/memberships_test.go b/syncapi/storage/tables/memberships_test.go index a421a97727..0a36f5887b 100644 --- a/syncapi/storage/tables/memberships_test.go +++ b/syncapi/storage/tables/memberships_test.go @@ -124,7 +124,7 @@ func testUpsert(t *testing.T, ctx context.Context, table tables.Memberships, mem if err != nil { t.Fatalf("failed to select membership: %s", err) } - expectedPos := 1 + var expectedPos int64 = 1 if pos != expectedPos { t.Fatalf("expected pos to be %d, got %d", expectedPos, pos) } diff --git a/syncapi/synctypes/clientevent.go b/syncapi/synctypes/clientevent.go index 6812f83322..fe4f6c07f0 100644 --- a/syncapi/synctypes/clientevent.go +++ b/syncapi/synctypes/clientevent.go @@ -347,71 +347,81 @@ func updatePowerLevelEvent(userIDForSender spec.UserIDForSender, se gomatrixserv return se, nil } - pls, err := gomatrixserverlib.NewPowerLevelContentFromEvent(se) - if err != nil { - return nil, err - } - newPls := make(map[string]int64) - var userID *spec.UserID - for user, level := range pls.Users { - if eventFormat != FormatSyncFederation { - userID, err = userIDForSender(se.RoomID(), spec.SenderID(user)) - if err != nil { - return nil, err + newEv := se.JSON() + + usersField := gjson.GetBytes(se.JSON(), "content.users") + if usersField.Exists() { + pls, err := gomatrixserverlib.NewPowerLevelContentFromEvent(se) + if err != nil { + return nil, err + } + + newPls := make(map[string]int64) + var userID *spec.UserID + for user, level := range pls.Users { + if eventFormat != FormatSyncFederation { + userID, err = userIDForSender(se.RoomID(), spec.SenderID(user)) + if err != nil { + return nil, err + } + user = userID.String() } - user = userID.String() + newPls[user] = level } - newPls[user] = level - } - var newPlBytes, newEv []byte - newPlBytes, err = json.Marshal(newPls) - if err != nil { - return nil, err - } - newEv, err = sjson.SetRawBytes(se.JSON(), "content.users", newPlBytes) - if err != nil { - return nil, err - } - // do the same for prev content - prevContent := gjson.GetBytes(se.JSON(), "unsigned.prev_content") - if !prevContent.Exists() { - var evNew gomatrixserverlib.PDU - evNew, err = gomatrixserverlib.MustGetRoomVersion(se.Version()).NewEventFromTrustedJSON(newEv, false) + var newPlBytes []byte + newPlBytes, err = json.Marshal(newPls) + if err != nil { + return nil, err + } + newEv, err = sjson.SetRawBytes(se.JSON(), "content.users", newPlBytes) if err != nil { return nil, err } - - return evNew, err - } - pls = gomatrixserverlib.PowerLevelContent{} - err = json.Unmarshal([]byte(prevContent.Raw), &pls) - if err != nil { - return nil, err } - newPls = make(map[string]int64) - for user, level := range pls.Users { - if eventFormat != FormatSyncFederation { - userID, err = userIDForSender(se.RoomID(), spec.SenderID(user)) + // do the same for prev content + prevUsersField := gjson.GetBytes(se.JSON(), "unsigned.prev_content.users") + if prevUsersField.Exists() { + prevContent := gjson.GetBytes(se.JSON(), "unsigned.prev_content") + if !prevContent.Exists() { + evNew, err := gomatrixserverlib.MustGetRoomVersion(se.Version()).NewEventFromTrustedJSON(newEv, false) if err != nil { return nil, err } - user = userID.String() + + return evNew, err + } + pls := gomatrixserverlib.PowerLevelContent{} + err := json.Unmarshal([]byte(prevContent.Raw), &pls) + if err != nil { + return nil, err + } + + newPls := make(map[string]int64) + for user, level := range pls.Users { + if eventFormat != FormatSyncFederation { + userID, userErr := userIDForSender(se.RoomID(), spec.SenderID(user)) + if userErr != nil { + return nil, userErr + } + user = userID.String() + } + newPls[user] = level + } + + var newPlBytes []byte + newPlBytes, err = json.Marshal(newPls) + if err != nil { + return nil, err + } + newEv, err = sjson.SetRawBytes(newEv, "unsigned.prev_content.users", newPlBytes) + if err != nil { + return nil, err } - newPls[user] = level - } - newPlBytes, err = json.Marshal(newPls) - if err != nil { - return nil, err - } - newEv, err = sjson.SetRawBytes(newEv, "unsigned.prev_content.users", newPlBytes) - if err != nil { - return nil, err } - var evNew gomatrixserverlib.PDU - evNew, err = gomatrixserverlib.MustGetRoomVersion(se.Version()).NewEventFromTrustedJSONWithEventID(se.EventID(), newEv, false) + evNew, err := gomatrixserverlib.MustGetRoomVersion(se.Version()).NewEventFromTrustedJSONWithEventID(se.EventID(), newEv, false) if err != nil { return nil, err } diff --git a/sytest-whitelist b/sytest-whitelist index c61e0bc3c6..492c756baf 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -783,4 +783,16 @@ Invited user can reject invite for empty room Invited user can reject local invite after originator leaves Guest users can join guest_access rooms Forgotten room messages cannot be paginated -Local device key changes get to remote servers with correct prev_id \ No newline at end of file +Local device key changes get to remote servers with correct prev_id +HS provides query metadata +HS can provide query metadata on a single protocol +Invites over federation are correctly pushed +Invites over federation are correctly pushed with name +User can create and send/receive messages in a room with version 11 +local user can join room with version 11 +User can invite local user to room with version 11 +remote user can join room with version 11 +User can invite remote user to room with version 11 +Remote user can backfill in a room with version 11 +Can reject invites over federation for rooms with version 11 +Can receive redactions from regular users over federation in room version 11 \ No newline at end of file diff --git a/userapi/consumers/devicelistupdate.go b/userapi/consumers/devicelistupdate.go index 3389bb808d..b3ccb573b3 100644 --- a/userapi/consumers/devicelistupdate.go +++ b/userapi/consumers/devicelistupdate.go @@ -17,6 +17,7 @@ package consumers import ( "context" "encoding/json" + "time" "github.com/matrix-org/dendrite/userapi/internal" "github.com/matrix-org/gomatrixserverlib" @@ -82,7 +83,10 @@ func (t *DeviceListUpdateConsumer) onMessage(ctx context.Context, msgs []*nats.M return true } - err := t.updater.Update(ctx, m) + timeoutCtx, cancel := context.WithTimeout(ctx, time.Second*30) + defer cancel() + + err := t.updater.Update(timeoutCtx, m) if err != nil { logrus.WithFields(logrus.Fields{ "user_id": m.UserID, diff --git a/userapi/consumers/roomserver.go b/userapi/consumers/roomserver.go index 047fe9216b..fca7412983 100644 --- a/userapi/consumers/roomserver.go +++ b/userapi/consumers/roomserver.go @@ -92,18 +92,36 @@ func (s *OutputRoomEventConsumer) Start() error { func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool { msg := msgs[0] // Guaranteed to exist if onMessage is called // Only handle events we care about - if rsapi.OutputType(msg.Header.Get(jetstream.RoomEventType)) != rsapi.OutputTypeNewRoomEvent { - return true - } - var output rsapi.OutputEvent - if err := json.Unmarshal(msg.Data, &output); err != nil { - // If the message was invalid, log it and move on to the next message in the stream - log.WithError(err).Errorf("roomserver output log: message parse failure") - return true - } - event := output.NewRoomEvent.Event - if event == nil { - log.Errorf("userapi consumer: expected event") + + var event *rstypes.HeaderedEvent + var isNewRoomEvent bool + switch rsapi.OutputType(msg.Header.Get(jetstream.RoomEventType)) { + case rsapi.OutputTypeNewRoomEvent: + isNewRoomEvent = true + fallthrough + case rsapi.OutputTypeNewInviteEvent: + var output rsapi.OutputEvent + if err := json.Unmarshal(msg.Data, &output); err != nil { + // If the message was invalid, log it and move on to the next message in the stream + log.WithError(err).Errorf("roomserver output log: message parse failure") + return true + } + if isNewRoomEvent { + event = output.NewRoomEvent.Event + } else { + event = output.NewInviteEvent.Event + } + + if event == nil { + log.Errorf("userapi consumer: expected event") + return true + } + + log.WithFields(log.Fields{ + "event_id": event.EventID(), + "event_type": event.Type(), + }).Tracef("Received message from roomserver: %#v", output) + default: return true } @@ -111,11 +129,6 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Ms go s.storeMessageStats(ctx, event.Type(), string(event.SenderID()), event.RoomID().String()) } - log.WithFields(log.Fields{ - "event_id": event.EventID(), - "event_type": event.Type(), - }).Tracef("Received message from roomserver: %#v", output) - metadata, err := msg.Metadata() if err != nil { return true @@ -253,8 +266,8 @@ func (s *OutputRoomEventConsumer) updateMDirect(ctx context.Context, oldRoomID, directChats := gjson.ParseBytes(directChatsRaw) newDirectChats := make(map[string][]string) // iterate over all userID -> roomIDs + var found bool directChats.ForEach(func(userID, roomIDs gjson.Result) bool { - var found bool for _, roomID := range roomIDs.Array() { newDirectChats[userID.Str] = append(newDirectChats[userID.Str], roomID.Str) // add the new roomID to m.direct @@ -263,22 +276,21 @@ func (s *OutputRoomEventConsumer) updateMDirect(ctx context.Context, oldRoomID, newDirectChats[userID.Str] = append(newDirectChats[userID.Str], newRoomID) } } - // Only hit the database if we found the old room as a DM for this user - if found { - var data []byte - data, err = json.Marshal(newDirectChats) - if err != nil { - return true - } - if err = s.db.SaveAccountData(ctx, localpart, serverName, "", "m.direct", data); err != nil { - return true - } - } return true }) - if err != nil { - return fmt.Errorf("failed to update m.direct state") + + // Only hit the database if we found the old room as a DM for this user + if found { + var data []byte + data, err = json.Marshal(newDirectChats) + if err != nil { + return err + } + if err = s.db.SaveAccountData(ctx, localpart, serverName, "", "m.direct", data); err != nil { + return fmt.Errorf("failed to update m.direct state: %w", err) + } } + return nil } @@ -448,6 +460,19 @@ func (s *OutputRoomEventConsumer) roomName(ctx context.Context, event *rstypes.H } } + // Special case for invites, as we don't store any "current state" for these events, + // we need to make sure that, if present, the m.room.name is sent as well. + if event.Type() == spec.MRoomMember && + gjson.GetBytes(event.Content(), "membership").Str == "invite" { + invState := gjson.GetBytes(event.JSON(), "unsigned.invite_room_state") + for _, ev := range invState.Array() { + if ev.Get("type").Str == spec.MRoomName { + name := ev.Get("content.name").Str + return name, nil + } + } + } + req := &rsapi.QueryCurrentStateRequest{ RoomID: event.RoomID().String(), StateTuples: []gomatrixserverlib.StateKeyTuple{roomNameTuple, canonicalAliasTuple}, @@ -513,8 +538,8 @@ func (s *OutputRoomEventConsumer) notifyLocal(ctx context.Context, event *rstype if err != nil { return fmt.Errorf("pushrules.ActionsToTweaks: %w", err) } - // TODO: support coalescing. - if a != pushrules.NotifyAction && a != pushrules.CoalesceAction { + + if a != pushrules.NotifyAction { log.WithFields(log.Fields{ "event_id": event.EventID(), "room_id": event.RoomID().String(), diff --git a/userapi/consumers/roomserver_test.go b/userapi/consumers/roomserver_test.go index 49dd5b2381..7b7c086183 100644 --- a/userapi/consumers/roomserver_test.go +++ b/userapi/consumers/roomserver_test.go @@ -81,12 +81,8 @@ func Test_evaluatePushRules(t *testing.T) { { name: "m.reaction doesn't notify", eventContent: `{"type":"m.reaction","room_id":"!room:example.com"}`, - wantAction: pushrules.DontNotifyAction, - wantActions: []*pushrules.Action{ - { - Kind: pushrules.DontNotifyAction, - }, - }, + wantAction: pushrules.UnknownAction, + wantActions: []*pushrules.Action{}, }, { name: "m.room.message notifies", @@ -136,7 +132,7 @@ func Test_evaluatePushRules(t *testing.T) { t.Fatalf("expected action to be '%s', got '%s'", tc.wantAction, gotAction) } // this is taken from `notifyLocal` - if tc.wantNotify && gotAction != pushrules.NotifyAction && gotAction != pushrules.CoalesceAction { + if tc.wantNotify && gotAction != pushrules.NotifyAction { t.Fatalf("expected to notify but didn't") } }) diff --git a/userapi/internal/device_list_update.go b/userapi/internal/device_list_update.go index 3fccf56bb5..b406351605 100644 --- a/userapi/internal/device_list_update.go +++ b/userapi/internal/device_list_update.go @@ -21,9 +21,11 @@ import ( "fmt" "hash/fnv" "net" + "strconv" "sync" "time" + "github.com/matrix-org/dendrite/federationapi/statistics" rsapi "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" @@ -107,6 +109,8 @@ type DeviceListUpdater struct { userIDToChan map[string]chan bool userIDToChanMu *sync.Mutex rsAPI rsapi.KeyserverRoomserverAPI + + isBlacklistedOrBackingOffFn func(s spec.ServerName) (*statistics.ServerStatistics, error) } // DeviceListUpdaterDatabase is the subset of functionality from storage.Database required for the updater. @@ -142,26 +146,52 @@ type KeyChangeProducer interface { ProduceKeyChanges(keys []api.DeviceMessage) error } +var deviceListUpdaterBackpressure = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "dendrite", + Subsystem: "keyserver", + Name: "worker_backpressure", + Help: "How many device list updater requests are queued", + }, + []string{"worker_id"}, +) +var deviceListUpdaterServersRetrying = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "dendrite", + Subsystem: "keyserver", + Name: "worker_servers_retrying", + Help: "How many servers are queued for retry", + }, + []string{"worker_id"}, +) + // NewDeviceListUpdater creates a new updater which fetches fresh device lists when they go stale. func NewDeviceListUpdater( process *process.ProcessContext, db DeviceListUpdaterDatabase, api DeviceListUpdaterAPI, producer KeyChangeProducer, fedClient fedsenderapi.KeyserverFederationAPI, numWorkers int, - rsAPI rsapi.KeyserverRoomserverAPI, thisServer spec.ServerName, + rsAPI rsapi.KeyserverRoomserverAPI, + thisServer spec.ServerName, + enableMetrics bool, + isBlacklistedOrBackingOffFn func(s spec.ServerName) (*statistics.ServerStatistics, error), ) *DeviceListUpdater { + if enableMetrics { + prometheus.MustRegister(deviceListUpdaterBackpressure, deviceListUpdaterServersRetrying) + } return &DeviceListUpdater{ - process: process, - userIDToMutex: make(map[string]*sync.Mutex), - mu: &sync.Mutex{}, - db: db, - api: api, - producer: producer, - fedClient: fedClient, - thisServer: thisServer, - workerChans: make([]chan spec.ServerName, numWorkers), - userIDToChan: make(map[string]chan bool), - userIDToChanMu: &sync.Mutex{}, - rsAPI: rsAPI, + process: process, + userIDToMutex: make(map[string]*sync.Mutex), + mu: &sync.Mutex{}, + db: db, + api: api, + producer: producer, + fedClient: fedClient, + thisServer: thisServer, + workerChans: make([]chan spec.ServerName, numWorkers), + userIDToChan: make(map[string]chan bool), + userIDToChanMu: &sync.Mutex{}, + rsAPI: rsAPI, + isBlacklistedOrBackingOffFn: isBlacklistedOrBackingOffFn, } } @@ -173,18 +203,20 @@ func (u *DeviceListUpdater) Start() error { // to stop (in this transaction) until key requests can be made. ch := make(chan spec.ServerName, 10) u.workerChans[i] = ch - go u.worker(ch) + go u.worker(ch, i) } staleLists, err := u.db.StaleDeviceLists(u.process.Context(), []spec.ServerName{}) if err != nil { return err } + + newStaleLists := dedupeStaleLists(staleLists) offset, step := time.Second*10, time.Second - if max := len(staleLists); max > 120 { + if max := len(newStaleLists); max > 120 { step = (time.Second * 120) / time.Duration(max) } - for _, userID := range staleLists { + for _, userID := range newStaleLists { userID := userID // otherwise we are only sending the last entry time.AfterFunc(offset, func() { u.notifyWorkers(userID) @@ -336,11 +368,22 @@ func (u *DeviceListUpdater) notifyWorkers(userID string) { if err != nil { return } + _, err = u.isBlacklistedOrBackingOffFn(remoteServer) + var federationClientError *fedsenderapi.FederationClientError + if errors.As(err, &federationClientError) { + if federationClientError.Blacklisted { + return + } + } + hash := fnv.New32a() _, _ = hash.Write([]byte(remoteServer)) index := int(int64(hash.Sum32()) % int64(len(u.workerChans))) ch := u.assignChannel(userID) + // Since workerChans are buffered, we only increment here and let the worker + // decrement it once it is done processing. + deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(index)}).Inc() u.workerChans[index] <- remoteServer select { case <-ch: @@ -370,27 +413,44 @@ func (u *DeviceListUpdater) clearChannel(userID string) { } } -func (u *DeviceListUpdater) worker(ch chan spec.ServerName) { +func (u *DeviceListUpdater) worker(ch chan spec.ServerName, workerID int) { retries := make(map[spec.ServerName]time.Time) retriesMu := &sync.Mutex{} // restarter goroutine which will inject failed servers into ch when it is time go func() { var serversToRetry []spec.ServerName for { - serversToRetry = serversToRetry[:0] // reuse memory - time.Sleep(time.Second) + // nuke serversToRetry by re-slicing it to be "empty". + // The capacity of the slice is unchanged, which ensures we can reuse the memory. + serversToRetry = serversToRetry[:0] + + deviceListUpdaterServersRetrying.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Set(float64(len(retries))) + time.Sleep(time.Second * 2) + + // -2, so we have space for incoming device list updates over federation + maxServers := (cap(ch) - len(ch)) - 2 + if maxServers <= 0 { + continue + } + retriesMu.Lock() now := time.Now() for srv, retryAt := range retries { if now.After(retryAt) { serversToRetry = append(serversToRetry, srv) + if maxServers == len(serversToRetry) { + break + } } } + for _, srv := range serversToRetry { delete(retries, srv) } retriesMu.Unlock() + for _, srv := range serversToRetry { + deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Inc() ch <- srv } } @@ -399,8 +459,18 @@ func (u *DeviceListUpdater) worker(ch chan spec.ServerName) { retriesMu.Lock() _, exists := retries[serverName] retriesMu.Unlock() - if exists { - // Don't retry a server that we're already waiting for. + + // If the serverName is coming from retries, maybe it was + // blacklisted in the meantime. + _, err := u.isBlacklistedOrBackingOffFn(serverName) + var federationClientError *fedsenderapi.FederationClientError + // unwrap errors and check for FederationClientError, if found, federationClientError will be not nil + errors.As(err, &federationClientError) + isBlacklisted := federationClientError != nil && federationClientError.Blacklisted + + // Don't retry a server that we're already waiting for or is blacklisted by now. + if exists || isBlacklisted { + deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Dec() continue } waitTime, shouldRetry := u.processServer(serverName) @@ -411,11 +481,18 @@ func (u *DeviceListUpdater) worker(ch chan spec.ServerName) { } retriesMu.Unlock() } + deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Dec() } } func (u *DeviceListUpdater) processServer(serverName spec.ServerName) (time.Duration, bool) { ctx := u.process.Context() + // If the process.Context is canceled, there is no need to go further. + // This avoids spamming the logs when shutting down + if errors.Is(ctx.Err(), context.Canceled) { + return defaultWaitTime, false + } + logger := util.GetLogger(ctx).WithField("server_name", serverName) deviceListUpdateCount.WithLabelValues(string(serverName)).Inc() @@ -428,13 +505,6 @@ func (u *DeviceListUpdater) processServer(serverName spec.ServerName) (time.Dura return waitTime, true } - defer func() { - for _, userID := range userIDs { - // always clear the channel to unblock Update calls regardless of success/failure - u.clearChannel(userID) - } - }() - for _, userID := range userIDs { userWait, err := u.processServerUser(ctx, serverName, userID) if err != nil { @@ -461,6 +531,11 @@ func (u *DeviceListUpdater) processServer(serverName spec.ServerName) (time.Dura func (u *DeviceListUpdater) processServerUser(ctx context.Context, serverName spec.ServerName, userID string) (time.Duration, error) { ctx, cancel := context.WithTimeout(ctx, requestTimeout) defer cancel() + + // If we are processing more than one user per server, this unblocks further calls to Update + // immediately instead of just after **all** users have been processed. + defer u.clearChannel(userID) + logger := util.GetLogger(ctx).WithFields(logrus.Fields{ "server_name": serverName, "user_id": userID, @@ -579,3 +654,24 @@ func (u *DeviceListUpdater) updateDeviceList(res *fclient.RespUserDevices) error } return nil } + +// dedupeStaleLists de-duplicates the stateList entries using the domain. +// This is used on startup, processServer is getting all users anyway, so +// there is no need to send every user to the workers. +func dedupeStaleLists(staleLists []string) []string { + seenDomains := make(map[spec.ServerName]struct{}) + newStaleLists := make([]string, 0, len(staleLists)) + for _, userID := range staleLists { + _, domain, err := gomatrixserverlib.SplitID('@', userID) + if err != nil { + // non-fatal and should not block starting up + continue + } + if _, ok := seenDomains[domain]; ok { + continue + } + newStaleLists = append(newStaleLists, userID) + seenDomains[domain] = struct{}{} + } + return newStaleLists +} diff --git a/userapi/internal/device_list_update_test.go b/userapi/internal/device_list_update_test.go index 10b9c6521f..a2f1869d1c 100644 --- a/userapi/internal/device_list_update_test.go +++ b/userapi/internal/device_list_update_test.go @@ -27,6 +27,9 @@ import ( "testing" "time" + api2 "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/dendrite/federationapi/statistics" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/fclient" @@ -128,6 +131,10 @@ type mockDeviceListUpdaterAPI struct { func (d *mockDeviceListUpdaterAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.PerformUploadDeviceKeysRequest, res *api.PerformUploadDeviceKeysResponse) { } +var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) { + return &statistics.ServerStatistics{}, nil +} + type roundTripper struct { fn func(*http.Request) (*http.Response, error) } @@ -161,7 +168,7 @@ func TestUpdateHavePrevID(t *testing.T) { } ap := &mockDeviceListUpdaterAPI{} producer := &mockKeyChangeProducer{} - updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, nil, 1, nil, "localhost") + updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, nil, 1, nil, "localhost", caching.DisableMetrics, testIsBlacklistedOrBackingOff) event := gomatrixserverlib.DeviceListUpdateEvent{ DeviceDisplayName: "Foo Bar", Deleted: false, @@ -233,7 +240,7 @@ func TestUpdateNoPrevID(t *testing.T) { `)), }, nil }) - updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 2, nil, "example.test") + updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 2, nil, "example.test", caching.DisableMetrics, testIsBlacklistedOrBackingOff) if err := updater.Start(); err != nil { t.Fatalf("failed to start updater: %s", err) } @@ -303,7 +310,7 @@ func TestDebounce(t *testing.T) { close(incomingFedReq) return <-fedCh, nil }) - updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 1, nil, "localhost") + updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 1, nil, "localhost", caching.DisableMetrics, testIsBlacklistedOrBackingOff) if err := updater.Start(); err != nil { t.Fatalf("failed to start updater: %s", err) } @@ -406,7 +413,7 @@ func TestDeviceListUpdater_CleanUp(t *testing.T) { updater := NewDeviceListUpdater(processCtx, db, nil, nil, nil, - 0, rsAPI, "test") + 0, rsAPI, "test", caching.DisableMetrics, testIsBlacklistedOrBackingOff) if err := updater.CleanUp(); err != nil { t.Error(err) } @@ -428,3 +435,114 @@ func TestDeviceListUpdater_CleanUp(t *testing.T) { } }) } + +func Test_dedupeStateList(t *testing.T) { + alice := "@alice:localhost" + bob := "@bob:localhost" + charlie := "@charlie:notlocalhost" + invalidUserID := "iaminvalid:localhost" + + tests := []struct { + name string + staleLists []string + want []string + }{ + { + name: "empty stateLists", + staleLists: []string{}, + want: []string{}, + }, + { + name: "single entry", + staleLists: []string{alice}, + want: []string{alice}, + }, + { + name: "multiple entries without dupe servers", + staleLists: []string{alice, charlie}, + want: []string{alice, charlie}, + }, + { + name: "multiple entries with dupe servers", + staleLists: []string{alice, bob, charlie}, + want: []string{alice, charlie}, + }, + { + name: "list with invalid userID", + staleLists: []string{alice, bob, invalidUserID}, + want: []string{alice}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := dedupeStaleLists(tt.staleLists); !reflect.DeepEqual(got, tt.want) { + t.Errorf("dedupeStaleLists() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestDeviceListUpdaterIgnoreBlacklisted(t *testing.T) { + unreachableServer := spec.ServerName("notlocalhost") + + updater := DeviceListUpdater{ + workerChans: make([]chan spec.ServerName, 1), + isBlacklistedOrBackingOffFn: func(s spec.ServerName) (*statistics.ServerStatistics, error) { + switch s { + case unreachableServer: + return nil, &api2.FederationClientError{Blacklisted: true} + } + return nil, nil + }, + mu: &sync.Mutex{}, + userIDToChanMu: &sync.Mutex{}, + userIDToChan: make(map[string]chan bool), + userIDToMutex: make(map[string]*sync.Mutex), + } + workerCh := make(chan spec.ServerName) + defer close(workerCh) + updater.workerChans[0] = workerCh + + // happy case + alice := "@alice:localhost" + aliceCh := updater.assignChannel(alice) + defer updater.clearChannel(alice) + + // failing case + bob := "@bob:" + unreachableServer + bobCh := updater.assignChannel(string(bob)) + defer updater.clearChannel(string(bob)) + + expectedServers := map[spec.ServerName]struct{}{ + "localhost": {}, + } + unexpectedServers := make(map[spec.ServerName]struct{}) + + go func() { + for serverName := range workerCh { + switch serverName { + case "localhost": + delete(expectedServers, serverName) + aliceCh <- true // unblock notifyWorkers + case unreachableServer: // this should not happen as it is "filtered" away by the blacklist + unexpectedServers[serverName] = struct{}{} + bobCh <- true + default: + unexpectedServers[serverName] = struct{}{} + } + } + }() + + // alice is not blacklisted + updater.notifyWorkers(alice) + // bob is blacklisted + updater.notifyWorkers(string(bob)) + + for server := range expectedServers { + t.Errorf("Server still in expectedServers map: %s", server) + } + + for server := range unexpectedServers { + t.Errorf("unexpected server in result: %s", server) + } +} diff --git a/userapi/storage/postgres/cross_signing_keys_table.go b/userapi/storage/postgres/cross_signing_keys_table.go index 138b629d71..deb355718c 100644 --- a/userapi/storage/postgres/cross_signing_keys_table.go +++ b/userapi/storage/postgres/cross_signing_keys_table.go @@ -77,7 +77,7 @@ func (s *crossSigningKeysStatements) SelectCrossSigningKeysForUser( for rows.Next() { var keyTypeInt int16 var keyData spec.Base64Bytes - if err := rows.Scan(&keyTypeInt, &keyData); err != nil { + if err = rows.Scan(&keyTypeInt, &keyData); err != nil { return nil, err } keyType, ok := types.KeyTypeIntToPurpose[keyTypeInt] @@ -86,6 +86,7 @@ func (s *crossSigningKeysStatements) SelectCrossSigningKeysForUser( } r[keyType] = keyData } + err = rows.Err() return } diff --git a/userapi/storage/postgres/cross_signing_sigs_table.go b/userapi/storage/postgres/cross_signing_sigs_table.go index 61a3811841..cba015e136 100644 --- a/userapi/storage/postgres/cross_signing_sigs_table.go +++ b/userapi/storage/postgres/cross_signing_sigs_table.go @@ -98,7 +98,7 @@ func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget( var userID string var keyID gomatrixserverlib.KeyID var signature spec.Base64Bytes - if err := rows.Scan(&userID, &keyID, &signature); err != nil { + if err = rows.Scan(&userID, &keyID, &signature); err != nil { return nil, err } if _, ok := r[userID]; !ok { @@ -106,6 +106,7 @@ func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget( } r[userID][keyID] = signature } + err = rows.Err() return } diff --git a/userapi/storage/postgres/key_backup_table.go b/userapi/storage/postgres/key_backup_table.go index 91a34c357f..59944a125b 100644 --- a/userapi/storage/postgres/key_backup_table.go +++ b/userapi/storage/postgres/key_backup_table.go @@ -162,5 +162,5 @@ func unpackKeys(ctx context.Context, rows *sql.Rows) (map[string]map[string]api. roomData[key.SessionID] = key.KeyBackupSession result[key.RoomID] = roomData } - return result, nil + return result, rows.Err() } diff --git a/userapi/storage/postgres/key_changes_table.go b/userapi/storage/postgres/key_changes_table.go index a00494140e..de3a9e9c8b 100644 --- a/userapi/storage/postgres/key_changes_table.go +++ b/userapi/storage/postgres/key_changes_table.go @@ -115,7 +115,7 @@ func (s *keyChangesStatements) SelectKeyChanges( for rows.Next() { var userID string var offset int64 - if err := rows.Scan(&userID, &offset); err != nil { + if err = rows.Scan(&userID, &offset); err != nil { return nil, 0, err } if offset > latestOffset { @@ -123,5 +123,6 @@ func (s *keyChangesStatements) SelectKeyChanges( } userIDs = append(userIDs, userID) } + err = rows.Err() return } diff --git a/userapi/storage/postgres/one_time_keys_table.go b/userapi/storage/postgres/one_time_keys_table.go index 972a591473..a00f4d6f66 100644 --- a/userapi/storage/postgres/one_time_keys_table.go +++ b/userapi/storage/postgres/one_time_keys_table.go @@ -134,7 +134,7 @@ func (s *oneTimeKeysStatements) CountOneTimeKeys(ctx context.Context, userID, de } counts.KeyCount[algorithm] = count } - return counts, nil + return counts, rows.Err() } func (s *oneTimeKeysStatements) InsertOneTimeKeys(ctx context.Context, txn *sql.Tx, keys api.OneTimeKeys) (*api.OneTimeKeysCount, error) { diff --git a/userapi/storage/postgres/profile_table.go b/userapi/storage/postgres/profile_table.go index e404c32f29..e4f55ed94e 100644 --- a/userapi/storage/postgres/profile_table.go +++ b/userapi/storage/postgres/profile_table.go @@ -165,5 +165,5 @@ func (s *profilesStatements) SelectProfilesBySearch( profiles = append(profiles, profile) } } - return profiles, nil + return profiles, rows.Err() } diff --git a/userapi/storage/postgres/threepid_table.go b/userapi/storage/postgres/threepid_table.go index 15b42a0a6a..fc46061dc0 100644 --- a/userapi/storage/postgres/threepid_table.go +++ b/userapi/storage/postgres/threepid_table.go @@ -18,6 +18,7 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/storage/tables" "github.com/matrix-org/gomatrixserverlib/spec" @@ -94,6 +95,7 @@ func (s *threepidStatements) SelectThreePIDsForLocalpart( if err != nil { return } + defer internal.CloseAndLogIfError(ctx, rows, "SelectThreePIDsForLocalpart: failed to close rows") threepids = []authtypes.ThreePID{} for rows.Next() { @@ -107,7 +109,7 @@ func (s *threepidStatements) SelectThreePIDsForLocalpart( Medium: medium, }) } - + err = rows.Err() return } diff --git a/userapi/storage/sqlite3/account_data_table.go b/userapi/storage/sqlite3/account_data_table.go index 3a6367c450..240647b32e 100644 --- a/userapi/storage/sqlite3/account_data_table.go +++ b/userapi/storage/sqlite3/account_data_table.go @@ -19,6 +19,7 @@ import ( "database/sql" "encoding/json" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/storage/tables" "github.com/matrix-org/gomatrixserverlib/spec" @@ -95,6 +96,7 @@ func (s *accountDataStatements) SelectAccountData( if err != nil { return nil, nil, err } + defer internal.CloseAndLogIfError(ctx, rows, "SelectAccountData: failed to close rows") global := map[string]json.RawMessage{} rooms := map[string]map[string]json.RawMessage{} @@ -118,7 +120,7 @@ func (s *accountDataStatements) SelectAccountData( } } - return global, rooms, nil + return global, rooms, rows.Err() } func (s *accountDataStatements) SelectAccountDataByType( diff --git a/userapi/storage/sqlite3/cross_signing_keys_table.go b/userapi/storage/sqlite3/cross_signing_keys_table.go index 5c2ce70397..65b9ff1af3 100644 --- a/userapi/storage/sqlite3/cross_signing_keys_table.go +++ b/userapi/storage/sqlite3/cross_signing_keys_table.go @@ -76,7 +76,7 @@ func (s *crossSigningKeysStatements) SelectCrossSigningKeysForUser( for rows.Next() { var keyTypeInt int16 var keyData spec.Base64Bytes - if err := rows.Scan(&keyTypeInt, &keyData); err != nil { + if err = rows.Scan(&keyTypeInt, &keyData); err != nil { return nil, err } keyType, ok := types.KeyTypeIntToPurpose[keyTypeInt] @@ -85,6 +85,7 @@ func (s *crossSigningKeysStatements) SelectCrossSigningKeysForUser( } r[keyType] = keyData } + err = rows.Err() return } diff --git a/userapi/storage/sqlite3/cross_signing_sigs_table.go b/userapi/storage/sqlite3/cross_signing_sigs_table.go index 6572641158..bf400a00e5 100644 --- a/userapi/storage/sqlite3/cross_signing_sigs_table.go +++ b/userapi/storage/sqlite3/cross_signing_sigs_table.go @@ -96,7 +96,7 @@ func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget( var userID string var keyID gomatrixserverlib.KeyID var signature spec.Base64Bytes - if err := rows.Scan(&userID, &keyID, &signature); err != nil { + if err = rows.Scan(&userID, &keyID, &signature); err != nil { return nil, err } if _, ok := r[userID]; !ok { @@ -104,6 +104,7 @@ func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget( } r[userID][keyID] = signature } + err = rows.Err() return } diff --git a/userapi/storage/sqlite3/devices_table.go b/userapi/storage/sqlite3/devices_table.go index 23e8231168..5ce285c870 100644 --- a/userapi/storage/sqlite3/devices_table.go +++ b/userapi/storage/sqlite3/devices_table.go @@ -296,6 +296,7 @@ func (s *devicesStatements) SelectDevicesByLocalpart( if err != nil { return devices, err } + defer internal.CloseAndLogIfError(ctx, rows, "SelectDevicesByLocalpart: failed to close rows") var dev api.Device var lastseents sql.NullInt64 @@ -325,7 +326,7 @@ func (s *devicesStatements) SelectDevicesByLocalpart( devices = append(devices, dev) } - return devices, nil + return devices, rows.Err() } func (s *devicesStatements) SelectDevicesByID(ctx context.Context, deviceIDs []string) ([]api.Device, error) { diff --git a/userapi/storage/sqlite3/key_backup_table.go b/userapi/storage/sqlite3/key_backup_table.go index ed27463109..1cdaca1802 100644 --- a/userapi/storage/sqlite3/key_backup_table.go +++ b/userapi/storage/sqlite3/key_backup_table.go @@ -162,5 +162,5 @@ func unpackKeys(ctx context.Context, rows *sql.Rows) (map[string]map[string]api. roomData[key.SessionID] = key.KeyBackupSession result[key.RoomID] = roomData } - return result, nil + return result, rows.Err() } diff --git a/userapi/storage/sqlite3/key_changes_table.go b/userapi/storage/sqlite3/key_changes_table.go index 923bb57eb3..7a4898cfb4 100644 --- a/userapi/storage/sqlite3/key_changes_table.go +++ b/userapi/storage/sqlite3/key_changes_table.go @@ -113,7 +113,7 @@ func (s *keyChangesStatements) SelectKeyChanges( for rows.Next() { var userID string var offset int64 - if err := rows.Scan(&userID, &offset); err != nil { + if err = rows.Scan(&userID, &offset); err != nil { return nil, 0, err } if offset > latestOffset { @@ -121,5 +121,6 @@ func (s *keyChangesStatements) SelectKeyChanges( } userIDs = append(userIDs, userID) } + err = rows.Err() return } diff --git a/userapi/storage/sqlite3/one_time_keys_table.go b/userapi/storage/sqlite3/one_time_keys_table.go index a992d399c3..2a5b1280bf 100644 --- a/userapi/storage/sqlite3/one_time_keys_table.go +++ b/userapi/storage/sqlite3/one_time_keys_table.go @@ -140,7 +140,7 @@ func (s *oneTimeKeysStatements) CountOneTimeKeys(ctx context.Context, userID, de } counts.KeyCount[algorithm] = count } - return counts, nil + return counts, rows.Err() } func (s *oneTimeKeysStatements) InsertOneTimeKeys( diff --git a/userapi/storage/sqlite3/profile_table.go b/userapi/storage/sqlite3/profile_table.go index a20d7e848f..7285110bc4 100644 --- a/userapi/storage/sqlite3/profile_table.go +++ b/userapi/storage/sqlite3/profile_table.go @@ -173,5 +173,5 @@ func (s *profilesStatements) SelectProfilesBySearch( profiles = append(profiles, profile) } } - return profiles, nil + return profiles, rows.Err() } diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go index a46ee9ebb1..5a789dfd79 100644 --- a/userapi/storage/storage_test.go +++ b/userapi/storage/storage_test.go @@ -1,6 +1,7 @@ package storage_test import ( + "bytes" "context" "encoding/json" "fmt" @@ -758,3 +759,53 @@ func TestDeviceKeysStreamIDGeneration(t *testing.T) { } }) } + +func TestOneTimeKeys(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, clean := mustCreateKeyDatabase(t, dbType) + defer clean() + userID := "@alice:localhost" + deviceID := "alice_device" + otk := api.OneTimeKeys{ + UserID: userID, + DeviceID: deviceID, + KeyJSON: map[string]json.RawMessage{"curve25519:KEY1": []byte(`{"key":"v1"}`)}, + } + + // Add a one time key to the DB + _, err := db.StoreOneTimeKeys(ctx, otk) + MustNotError(t, err) + + // Check the count of one time keys is correct + count, err := db.OneTimeKeysCount(ctx, userID, deviceID) + MustNotError(t, err) + if count.KeyCount["curve25519"] != 1 { + t.Fatalf("Expected 1 key, got %d", count.KeyCount["curve25519"]) + } + + // Check the actual key contents are correct + keysJSON, err := db.ExistingOneTimeKeys(ctx, userID, deviceID, []string{"curve25519:KEY1"}) + MustNotError(t, err) + keyJSON, err := keysJSON["curve25519:KEY1"].MarshalJSON() + MustNotError(t, err) + if !bytes.Equal(keyJSON, []byte(`{"key":"v1"}`)) { + t.Fatalf("Existing keys do not match expected. Got %v", keysJSON["curve25519:KEY1"]) + } + + // Claim a one time key from the database. This should remove it from the database. + claimedKeys, err := db.ClaimKeys(ctx, map[string]map[string]string{userID: {deviceID: "curve25519"}}) + MustNotError(t, err) + + // Check the claimed key contents are correct + if !reflect.DeepEqual(claimedKeys[0], otk) { + t.Fatalf("Expected to claim stored key %v. Got %v", otk, claimedKeys[0]) + } + + // Check the count of one time keys is now zero + count, err = db.OneTimeKeysCount(ctx, userID, deviceID) + MustNotError(t, err) + if count.KeyCount["curve25519"] != 0 { + t.Fatalf("Expected 0 keys, got %d", count.KeyCount["curve25519"]) + } + }) +} diff --git a/userapi/userapi.go b/userapi/userapi.go index 6b6dac884d..a1c9b94a90 100644 --- a/userapi/userapi.go +++ b/userapi/userapi.go @@ -18,10 +18,12 @@ import ( "time" fedsenderapi "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/internal/pushgateway" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/process" + "github.com/matrix-org/gomatrixserverlib/spec" "github.com/sirupsen/logrus" rsapi "github.com/matrix-org/dendrite/roomserver/api" @@ -46,6 +48,8 @@ func NewInternalAPI( natsInstance *jetstream.NATSInstance, rsAPI rsapi.UserRoomserverAPI, fedClient fedsenderapi.KeyserverFederationAPI, + enableMetrics bool, + blacklistedOrBackingOffFn func(s spec.ServerName) (*statistics.ServerStatistics, error), ) *internal.UserInternalAPI { js, _ := natsInstance.Prepare(processContext, &dendriteCfg.Global.JetStream) appServices := dendriteCfg.Derived.ApplicationServices @@ -99,7 +103,7 @@ func NewInternalAPI( FedClient: fedClient, } - updater := internal.NewDeviceListUpdater(processContext, keyDB, userAPI, keyChangeProducer, fedClient, 8, rsAPI, dendriteCfg.Global.ServerName) // 8 workers TODO: configurable + updater := internal.NewDeviceListUpdater(processContext, keyDB, userAPI, keyChangeProducer, fedClient, dendriteCfg.UserAPI.WorkerCount, rsAPI, dendriteCfg.Global.ServerName, enableMetrics, blacklistedOrBackingOffFn) userAPI.Updater = updater // Remove users which we don't share a room with anymore if err := updater.CleanUp(); err != nil {