Merge branch 'wireapp:develop' into develop

cabal.project

@@ -29,6 +29,7 @@ packages:
   , libs/wai-utilities/
   , libs/wire-api/
   , libs/wire-api-federation/
+  , libs/wire-otel/
   , libs/wire-message-proto-lens/
   , libs/wire-subsystems/
   , libs/zauth/

changelog.d/1-api-changes/wpb-10708

@@ -0,0 +1 @@
+Add endpoint to upgrade a personal user to a team owner

changelog.d/2-features/WPB-10772

@@ -0,0 +1,5 @@
+Makes it impossible for a user to join an MLS conversation while already under legalhold (at least pending)
+
+This implies two things: 
+1. If a user is under legalhold they cannot ever join an MLS conversation, not even an MLS self conversation.
+2. A user has to reject to be put under legalhold when they want to join an MLS conversation (ignoring the request to be put under legalhold is not enough).

changelog.d/2-features/block-lh-for-mls-users

@@ -0,0 +1 @@
+Deny requests for a legalhold device for users who are part of any MLS conversations

changelog.d/2-features/open-telemetry-instrumentation-brig-galley-gundeck-cannon

@@ -0,0 +1 @@
+added open telemetry instrumentation for brig, galley, gundeck and cannon

changelog.d/2-features/personal-account-to-team-email

@@ -0,0 +1 @@
+Send confirmation email after adding a personal user to a new team

changelog.d/3-bug-fixes/remove-spam-from-nginx

@@ -0,0 +1 @@
+removed spam from nginx (nginz) by using the new style http/2 directive

changelog.d/3-bug-fixes/services-tags

@@ -0,0 +1 @@
+brig: Make `GET /services/tags` work again

changelog.d/5-internal/WPB-11000

@@ -0,0 +1 @@
+Additional test for password reset, port tests to new integration test suite

changelog.d/5-internal/WPB-1228-servantify-gundeck-internal-api

@@ -0,0 +1 @@
+Servantify gundeck internal api

changelog.d/5-internal/WPB-888-2

@@ -0,0 +1 @@
+Removed `indexReindex` and `indexReindexIfSameOrNewer` from internal Brig/SearchIndex.

changelog.d/5-internal/WPB-8888

@@ -0,0 +1 @@
+Introduced ElasticSearch effects related to user search.

changelog.d/5-internal/background-worker

@@ -1 +1 @@
-charts/wire-server: Deploy background-worker even when tags.federation is `false`
+charts/wire-server: Deploy background-worker even when tags.federation is `false` (#4342, #4248)

changelog.d/5-internal/new-team-types-refactoring

@@ -0,0 +1 @@
+Simplify NewTeam and related types and remove lenses

deploy/dockerephemeral/federation-v0/nginz/conf/integration.conf

@@ -7,13 +7,15 @@ listen 8081;
 # port.
 # This port is only used for trying out nginx http2 forwarding without TLS locally and should not
 # be ported to any production nginz config.
-listen 8090 http2;
+listen 8090;
 
 ######## TLS/SSL block start ##############
 #
 # Most integration tests simply use the http ports 8080 and 8081
 # But to also test tls forwarding, this port can be used.
 # This applies only locally, as for kubernetes (helm chart) based deployments,
 # TLS is terminated at the ingress level, not at nginz level
-listen 8443 ssl http2;
-listen [::]:8443 ssl http2;
+listen 8443 ssl;
+listen [::]:8443 ssl;
+
+http2 on;

deploy/dockerephemeral/federation-v1/nginz/conf/integration.conf

@@ -7,13 +7,15 @@ listen 8081;
 # port.
 # This port is only used for trying out nginx http2 forwarding without TLS locally and should not
 # be ported to any production nginz config.
-listen 8090 http2;
+listen 8090;
 
 ######## TLS/SSL block start ##############
 #
 # Most integration tests simply use the http ports 8080 and 8081
 # But to also test tls forwarding, this port can be used.
 # This applies only locally, as for kubernetes (helm chart) based deployments,
 # TLS is terminated at the ingress level, not at nginz level
-listen 8443 ssl http2;
-listen [::]:8443 ssl http2;
+listen 8443 ssl;
+listen [::]:8443 ssl;
+
+http2 on;

docs/src/developer/developer/open-telemetry.md

@@ -0,0 +1,47 @@
+# OpenTelemetry Instrumentation
+
+## Current Status
+
+The following components have been instrumented:
+- brig
+- galley
+- gundeck
+- cannon
+
+## Known Issues and future work
+
+- Proper HTTP/2 instrumentation is missing for federator & co - this is related to http/2 outobj in the http2 libraray throwing away all structured information
+- Some parts of the service, such as background jobs, may need additional instrumentation. It's currently unclear if these are appearing in the tracing data.
+- we need to ingest the data into grafana tempo
+
+
+## Setup instructions for local use
+
+To view the tracing data:
+
+1. Start Jaeger using Docker:
+   ```bash
+   docker run --rm --name jaeger \
+     -e COLLECTOR_ZIPKIN_HOST_PORT=:9411 \
+     -p 6831:6831/udp \
+     -p 6832:6832/udp \
+     -p 5778:5778 \
+     -p 16686:16686 \
+     -p 4317:4317 \
+     -p 4318:4318 \
+     -p 14250:14250 \
+     -p 14268:14268 \
+     -p 14269:14269 \
+     -p 9411:9411 \
+     jaegertracing/all-in-one:latest
+   ```
+
+2. Start your services or run integration tests.
+3. Open the Jaeger UI at [http://localhost:16686/](http://localhost:16686/)
+
+## Relevant Resources
+
+We're using the `hs-opentelemetry-*` family of haskell packages available [here](https://github.com/iand675/hs-opentelemetry).
+
+- [hs-opentelemetry-instrumentation-wai](https://hackage.haskell.org/package/hs-opentelemetry-instrumentation-wai-0.1.0.0/docs/src/OpenTelemetry.Instrumentation.Wai.html#local-6989586621679045744)
+- [hs-opentelemetry-sdk](https://hackage.haskell.org/package/hs-opentelemetry-sdk-0.0.3.6)

integration/default.nix

@@ -19,6 +19,7 @@
 , Cabal
 , case-insensitive
 , containers
+, cookie
 , cql
 , cql-io
 , crypton
@@ -115,6 +116,7 @@ mkDerivation {
     bytestring-conversion
     case-insensitive
     containers
+    cookie
     cql
     cql-io
     crypton

integration/integration.cabal

@@ -140,6 +140,7 @@ library
     Test.MLS.Unreachable
     Test.Notifications
     Test.OAuth
+    Test.PasswordReset
     Test.Presence
     Test.Property
     Test.Provider
@@ -193,6 +194,7 @@ library
     , bytestring-conversion
     , case-insensitive
     , containers
+    , cookie
     , cql
     , cql-io
     , crypton

integration/test/API/Brig.hs

@@ -795,3 +795,28 @@ listInvitations :: (HasCallStack, MakesValue user) => user -> String -> App Resp
 listInvitations user tid = do
   req <- baseRequest user Brig Versioned $ joinHttpPath ["teams", tid, "invitations"]
   submit "GET" req
+
+passwordReset :: (HasCallStack, MakesValue domain) => domain -> String -> App Response
+passwordReset domain email = do
+  req <- baseRequest domain Brig Versioned "password-reset"
+  submit "POST" $ req & addJSONObject ["email" .= email]
+
+completePasswordReset :: (HasCallStack, MakesValue domain) => domain -> String -> String -> String -> App Response
+completePasswordReset domain key code pw = do
+  req <- baseRequest domain Brig Versioned $ joinHttpPath ["password-reset", "complete"]
+  submit "POST" $ req & addJSONObject ["key" .= key, "code" .= code, "password" .= pw]
+
+login :: (HasCallStack, MakesValue domain) => domain -> String -> String -> App Response
+login domain email password = do
+  req <- baseRequest domain Brig Versioned "login"
+  submit "POST" $ req & addJSONObject ["email" .= email, "password" .= password] & addQueryParams [("persist", "true")]
+
+updateEmail :: (HasCallStack, MakesValue user) => user -> String -> String -> String -> App Response
+updateEmail user email cookie token = do
+  req <- baseRequest user Brig Versioned $ joinHttpPath ["access", "self", "email"]
+  submit "PUT" $ req & addJSONObject ["email" .= email] & setCookie cookie & addHeader "Authorization" ("Bearer " <> token)
+
+upgradePersonalToTeam :: (HasCallStack, MakesValue user) => user -> String -> App Response
+upgradePersonalToTeam user name = do
+  req <- baseRequest user Brig Versioned $ joinHttpPath ["upgrade-personal-to-team"]
+  submit "POST" $ req & addJSONObject ["name" .= name, "icon" .= "default"]

integration/test/API/BrigInternal.hs

@@ -298,3 +298,8 @@ getActivationCode :: (HasCallStack, MakesValue domain) => domain -> String -> Ap
 getActivationCode domain email = do
   req <- baseRequest domain Brig Unversioned "i/users/activation-code"
   submit "GET" $ req & addQueryParams [("email", email)]
+
+getPasswordResetCode :: (HasCallStack, MakesValue domain) => domain -> String -> App Response
+getPasswordResetCode domain email = do
+  req <- baseRequest domain Brig Unversioned "i/users/password-reset-code"
+  submit "GET" $ req & addQueryParams [("email", email)]

integration/test/API/Galley.hs

@@ -519,6 +519,12 @@ updateMessageTimer user qcnv update = do
   req <- baseRequest user Galley Versioned path
   submit "PUT" (addJSONObject ["message_timer" .= updateReq] req)
 
+getTeam :: (HasCallStack, MakesValue user, MakesValue tid) => user -> tid -> App Response
+getTeam user tid = do
+  tidStr <- asString tid
+  req <- baseRequest user Galley Versioned (joinHttpPath ["teams", tidStr])
+  submit "GET" req
+
 getTeamMembers :: (HasCallStack, MakesValue user, MakesValue tid) => user -> tid -> App Response
 getTeamMembers user tid = do
   tidStr <- asString tid

integration/test/Test/Conversation.hs

@@ -874,3 +874,37 @@ testConversationWithoutFederation = withModifiedBackend
   $ \domain -> do
     [alice, bob] <- createAndConnectUsers [domain, domain]
     void $ postConversation alice (defProteus {qualifiedUsers = [bob]}) >>= getJSON 201
+
+testPostConvWithUnreachableRemoteUsers :: App ()
+testPostConvWithUnreachableRemoteUsers = do
+  [alice, alex] <- createAndConnectUsers [OwnDomain, OtherDomain]
+  resourcePool <- asks resourcePool
+  runCodensity (acquireResources 2 resourcePool) $ \[unreachableBackend, reachableBackend] -> do
+    runCodensity (startDynamicBackend reachableBackend mempty) $ \_ -> do
+      unreachableUsers <- runCodensity (startDynamicBackend unreachableBackend mempty) $ \_ -> do
+        let downDomain = unreachableBackend.berDomain
+        ownDomain <- asString OwnDomain
+        otherDomain <- asString OtherDomain
+        void $ BrigI.createFedConn downDomain (BrigI.FedConn ownDomain "full_search" Nothing)
+        void $ BrigI.createFedConn downDomain (BrigI.FedConn otherDomain "full_search" Nothing)
+        users <- replicateM 3 (randomUser downDomain def)
+        for_ users $ \user -> do
+          connectUsers [alice, user]
+          connectUsers [alex, user]
+        -- creating the conv here would work.
+        pure users
+
+      reachableUsers <- replicateM 2 (randomUser reachableBackend.berDomain def)
+      for_ reachableUsers $ \user -> do
+        connectUsers [alice, user]
+        connectUsers [alex, user]
+
+      withWebSockets [alice, alex] $ \[wssAlice, wssAlex] -> do
+        -- unreachableBackend is still allocated, but the backend is down.  creating the conv here doesn't work.
+        let payload = defProteus {name = Just "some chat", qualifiedUsers = [alex] <> reachableUsers <> unreachableUsers}
+        postConversation alice payload >>= assertStatus 533
+
+        convs <- getAllConvs alice
+        for_ convs $ \conv -> conv %. "type" `shouldNotMatchInt` 0
+        assertNoEvent 2 wssAlice
+        assertNoEvent 2 wssAlex

integration/test/Test/LegalHold.hs

@@ -35,6 +35,7 @@ import Data.ProtoLens.Labels ()
 import qualified Data.Set as Set
 import qualified Data.Text as T
 import GHC.Stack
+import MLS.Util
 import Network.Wai (Request (pathInfo, requestMethod))
 import Notifications
 import Numeric.Lens (hex)
@@ -904,3 +905,90 @@ testLHDisableBeforeApproval = do
     disableLegalHold tid alice bob defPassword
       >>= assertStatus 200
     getBob'sStatus `shouldMatch` "disabled"
+
+-- ---------
+-- WPB-10783
+-- ---------
+testBlockLHForMLSUsers :: (HasCallStack) => App ()
+testBlockLHForMLSUsers = do
+  -- scenario 1:
+  -- if charlie is in any MLS conversation, he cannot approve to be put under legalhold
+  (charlie, tid, []) <- createTeam OwnDomain 1
+  [charlie1] <- traverse (createMLSClient def) [charlie]
+  void $ createNewGroup charlie1
+  void $ createAddCommit charlie1 [charlie] >>= sendAndConsumeCommitBundle
+
+  legalholdWhitelistTeam tid charlie >>= assertStatus 200
+  withMockServer def lhMockApp \lhDomAndPort _chan -> do
+    postLegalHoldSettings tid charlie (mkLegalHoldSettings lhDomAndPort) >>= assertStatus 201
+    requestLegalHoldDevice tid charlie charlie `bindResponse` do
+      assertLabel 409 "mls-legal-hold-not-allowed"
+
+-- ---------
+-- WPB-10772
+-- ---------
+
+--  | scenario 2.1:
+-- charlie first is put under legalhold and after that wants to join an MLS conversation
+-- claiming a keypackage of charlie to add them to a conversation should not be possible
+testBlockClaimingKeyPackageForLHUsers :: (HasCallStack) => App ()
+testBlockClaimingKeyPackageForLHUsers = do
+  (alice, tid, [charlie]) <- createTeam OwnDomain 2
+  [alice1, charlie1] <- traverse (createMLSClient def) [alice, charlie]
+  _ <- uploadNewKeyPackage charlie1
+  _ <- createNewGroup alice1
+  legalholdWhitelistTeam tid alice >>= assertStatus 200
+  withMockServer def lhMockApp \lhDomAndPort _chan -> do
+    postLegalHoldSettings tid alice (mkLegalHoldSettings lhDomAndPort) >>= assertStatus 201
+    requestLegalHoldDevice tid alice charlie >>= assertSuccess
+    approveLegalHoldDevice tid (charlie %. "qualified_id") defPassword >>= assertSuccess
+    profile <- getUser alice charlie >>= getJSON 200
+    pStatus <- profile %. "legalhold_status" & asString
+    pStatus `shouldMatch` "enabled"
+
+    mls <- getMLSState
+    claimKeyPackages mls.ciphersuite alice1 charlie
+      `bindResponse` assertLabel 409 "mls-legal-hold-not-allowed"
+
+-- | scenario 2.2:
+-- charlie is put under legalhold but creates an MLS Group himself
+-- since he doesn't need to claim his own keypackage to do so, this would succeed
+-- we need to check upon group creation if the user is under legalhold and reject
+-- the operation if they are
+testBlockCreateMLSConvForLHUsers :: (HasCallStack) => App ()
+testBlockCreateMLSConvForLHUsers = do
+  (alice, tid, [charlie]) <- createTeam OwnDomain 2
+  [alice1, charlie1] <- traverse (createMLSClient def) [alice, charlie]
+  _ <- uploadNewKeyPackage alice1
+  legalholdWhitelistTeam tid alice >>= assertStatus 200
+  withMockServer def lhMockApp \lhDomAndPort _chan -> do
+    postLegalHoldSettings tid alice (mkLegalHoldSettings lhDomAndPort) >>= assertStatus 201
+    requestLegalHoldDevice tid alice charlie >>= assertSuccess
+    approveLegalHoldDevice tid (charlie %. "qualified_id") defPassword >>= assertSuccess
+    profile <- getUser alice charlie >>= getJSON 200
+    pStatus <- profile %. "legalhold_status" & asString
+    pStatus `shouldMatch` "enabled"
+
+    -- charlie tries to create a group and should fail when POSTing the add commit
+    _ <- createNewGroup charlie1
+
+    void
+      -- we try to add alice since adding charlie himself would trigger 2.1
+      -- since he'd try to claim his own keypackages
+      $ createAddCommit charlie1 [alice]
+      >>= \mp ->
+        postMLSCommitBundle mp.sender (mkBundle mp)
+          `bindResponse` assertLabel 409 "mls-legal-hold-not-allowed"
+
+    -- (unsurprisingly) this same thing should also work in the one2one case
+
+    respJson <- getMLSOne2OneConversation alice charlie >>= getJSON 200
+    resetGroup alice1 (respJson %. "conversation")
+
+    void
+      -- we try to add alice since adding charlie himself would trigger 2.1
+      -- since he'd try to claim his own keypackages
+      $ createAddCommit charlie1 [alice]
+      >>= \mp ->
+        postMLSCommitBundle mp.sender (mkBundle mp)
+          `bindResponse` assertLabel 409 "mls-legal-hold-not-allowed"