From ff1994101a3fb4a91e348189bb19b1caaec4e022 Mon Sep 17 00:00:00 2001 From: Caleb Roseland Date: Wed, 17 Apr 2024 06:38:05 -0500 Subject: [PATCH] =?UTF-8?q?MM-57376,=20MM-57374,=20MM-57368:=20=20Connecti?= =?UTF-8?q?on=20Invites=E2=80=94whitelist=20and=20refinements=20(#599)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * wip-3 * wip-4 * whitelist io & restrictions * guard against empty ids in notify-connect * webapp lint * webapp lint * refactor CanConnect, refine option texts * fix options * remove refactored left-behind * review comments * fix config name * review changes * store test * fix lint & test * cleanup whitelist io flow, fix canopenlyconnect --- plugin.json | 422 ++++++++++-------- server/api.go | 263 ++++++++--- server/command.go | 30 +- server/configuration.go | 1 + server/connect.go | 84 +++- server/helper_test.go | 2 +- server/plugin.go | 12 +- server/store/mocks/Store.go | 218 ++++++--- server/store/sqlstore/migrations.go | 72 +++ server/store/sqlstore/store.go | 332 +++++++++++--- server/store/sqlstore/store_test.go | 63 +-- server/store/store.go | 59 ++- server/store/storemodels/storemodels.go | 11 +- server/store/timerlayer/timerlayer.go | 146 ++++-- webapp/src/client.ts | 31 ++ .../admin_console/app_manifest_setting.tsx | 39 ++ .../get_connected_users_setting.tsx} | 0 .../invite_whitelist_setting.tsx | 126 ++++++ webapp/src/components/appManifestSetting.tsx | 43 -- webapp/src/index.tsx | 9 +- 20 files changed, 1411 insertions(+), 552 deletions(-) create mode 100644 webapp/src/components/admin_console/app_manifest_setting.tsx rename webapp/src/components/{getConnectedUsersSetting.tsx => admin_console/get_connected_users_setting.tsx} (100%) create mode 100644 webapp/src/components/admin_console/invite_whitelist_setting.tsx delete mode 100644 webapp/src/components/appManifestSetting.tsx diff --git a/plugin.json b/plugin.json index c1a986b09..3485c56cc 100644 --- a/plugin.json +++ b/plugin.json @@ -18,188 +18,244 @@ "settings_schema": { "header": "", "footer": "", - "settings": [{ - "key": "tenantId", - "display_name": "Tenant ID", - "type": "text", - "help_text": "Microsoft Teams Tenant ID", - "default": "" - },{ - "key": "clientId", - "display_name": "Client ID", - "type": "text", - "help_text": "Microsoft Teams Client ID", - "default": "" - },{ - "key": "clientSecret", - "display_name": "Client Secret", - "type": "text", - "help_text": "Microsoft Teams Client Secret", - "default": "" - },{ - "key": "encryptionKey", - "display_name": "At Rest Encryption Key:", - "type": "generated", - "help_text": "The AES encryption key used to encrypt stored access tokens" - },{ - "key": "webhookSecret", - "display_name": "Webhook secret", - "type": "generated", - "help_text": "Microsoft Teams will use this secret to send messages to Mattermost" - },{ - "key": "certificatePublic", - "display_name": "Certificate Public", - "type": "longtext", - "help_text": "Certificate public part" - },{ - "key": "certificateKey", - "display_name": "Certificate Key", - "type": "longtext", - "help_text": "Certificate private key" - },{ - "key": "evaluationAPI", - "display_name": "Use the evaluation API pay model", - "type": "bool", - "help_text": "The evaluation API pay model only allows you to get a limited number of change notifications. Be sure your system keeps them low if you enable this setting", - "default": false - },{ - "key": "syncUsers", - "display_name": "Sync users", - "type": "number", - "help_text": "Set the number of minutes between users sync (Leave it empty to disable users sync)", - "default": 0 - },{ - "key": "syncGuestUsers", - "display_name": "Sync guest users", - "type": "bool", - "help_text": "Set the value to 'true' to sync MS Teams guest users", - "default": false - },{ - "key": "syncDirectMessages", - "display_name": "Sync direct and group messages", - "type": "bool", - "help_text": "Sync direct and group messages where any of the user in the conversation is a real Mattermost user connected to MS Teams account", - "default": false - },{ - "key": "selectiveSync", - "display_name": "Selective sync", - "type": "bool", - "help_text": "Skip syncing messages between users on the same platform.", - "default": false - },{ - "key": "syncLinkedChannels", - "display_name": "Sync linked channels", - "type": "bool", - "help_text": "Sync messages from channels linked between Mattermost and MS Teams", - "default": false - },{ - "key": "syncReactions", - "display_name": "Sync reactions", - "type": "bool", - "help_text": "Sync reactions on messages", - "default": false - },{ - "key": "syncFileAttachments", - "display_name": "Sync file attachments", - "type": "bool", - "help_text": "Sync file attachments on messages", - "default": false - },{ - "key": "enabledTeams", - "display_name": "Enabled Teams", - "type": "text", - "help_text": "Mattermost teams where sync is enabled (comma separated Mattermost team names, empty means all enabled)", - "default": "" - },{ - "key": "maxSizeForCompleteDownload", - "display_name": "Maximum size of attachments to support complete one time download (in MB)", - "type": "number", - "help_text": "Set the maximum size for attachments that can be loaded into the memory. Attachments bigger than this size will be streamed from MS Teams to Mattermost", - "default": 20 - },{ - "key": "bufferSizeForFileStreaming", - "display_name": "Buffer size for streaming files (in MB)", - "type": "number", - "help_text": "Set the buffer size for streaming files from MS Teams to Mattermost", - "default": 20 - },{ - "key": "connectedUsersAllowed", - "display_name": "Max Connected Users", - "type": "number", - "help_text": "The maximum number of users that may connect their MS Teams account. Once connected, the user is added to a white-list and may reconnect at any time.", - "default": 1000 - },{ - "key": "connectedUsersInvitePoolSize", - "display_name": "Connection Invites: Max Pending Invitations", - "type": "number", - "help_text": "Invite pool size: the maximum number of connection invites that may be pending at a given time. When specified, connection invite direct messages will be sent to users as they login, up to the maximum specified here. As invited users connect, spaces in the invite pool will open up and more invites will be sent out. (Set to 0 or leave empty to disable connection invites.)", - "default": 0 - },{ - "key": "automaticallyPromoteSyntheticUsers", - "display_name": "Automatically Promote Synthetic Users", - "type": "bool", - "help_text": "When true, synthetic users will be converted to members when they login for the first time.", - "default": false - },{ - "key": "disableSyncMsg", - "display_name": "Disable using the sync msg infrastructure for tracking message changes", - "type": "bool", - "help_text": "When true, the plugin will not enable any sync msg infrastructure.", - "default": false - },{ - "key": "disableCheckCredentials", - "display_name": "Disable periodically checking the validity of the application credentials", - "type": "bool", - "help_text": "When true, the plugin will not periodically check the validity of the application credentials.", - "default": false - },{ - "key": "syntheticUserAuthService", - "display_name": "Synthetic User Auth Service", - "type": "dropdown", - "help_text": "Select the authentication service to use when creating synthetic users. This should match the service used for member user access to Mattermost. Default is 'SAML'.", - "default": "saml", - "options": [ - { - "display_name": "SAML", - "value": "saml" - }, - { - "display_name": "AD/LDAP", - "value": "ldap" - } - ] - },{ - "key": "syntheticUserAuthData", - "display_name": "Synthetic User Auth Data", - "type": "dropdown", - "help_text": "Select the MS Teams user property to use as the authentication identifier. For AD/LDAP and SAML, the identifier's value should match the value provided by the ID Attribute. ", - "default": "ID", - "options": [ - { - "display_name": "ID", - "value": "ID" - }, - { - "display_name": "Mail", - "value": "Mail" - }, - { - "display_name": "User Principal Name", - "value": "UserPrincipalName" - } - ] - },{ - "key": "appManifestDownload", - "display_name": "Download Manifest", - "type": "custom", - "help_text": "", - "default": "" - },{ - "key": "ConnectedUsersReportDownload", - "display_name": "Download Report", - "type": "custom", - "help_text": "", - "default": "" - }] + "settings": [ + { + "key": "tenantId", + "display_name": "Tenant ID", + "type": "text", + "help_text": "Microsoft Teams Tenant ID", + "default": "" + }, + { + "key": "clientId", + "display_name": "Client ID", + "type": "text", + "help_text": "Microsoft Teams Client ID", + "default": "" + }, + { + "key": "clientSecret", + "display_name": "Client Secret", + "type": "text", + "help_text": "Microsoft Teams Client Secret", + "default": "" + }, + { + "key": "encryptionKey", + "display_name": "At Rest Encryption Key:", + "type": "generated", + "help_text": "The AES encryption key used to encrypt stored access tokens" + }, + { + "key": "webhookSecret", + "display_name": "Webhook secret", + "type": "generated", + "help_text": "Microsoft Teams will use this secret to send messages to Mattermost" + }, + { + "key": "certificatePublic", + "display_name": "Certificate Public", + "type": "longtext", + "help_text": "Certificate public part" + }, + { + "key": "certificateKey", + "display_name": "Certificate Key", + "type": "longtext", + "help_text": "Certificate private key" + }, + { + "key": "evaluationAPI", + "display_name": "Use the evaluation API pay model", + "type": "bool", + "help_text": "The evaluation API pay model only allows you to get a limited number of change notifications. Be sure your system keeps them low if you enable this setting", + "default": false + }, + { + "key": "syncUsers", + "display_name": "Sync users", + "type": "number", + "help_text": "Set the number of minutes between users sync (Leave it empty to disable users sync)", + "default": 0 + }, + { + "key": "syncGuestUsers", + "display_name": "Sync guest users", + "type": "bool", + "help_text": "Set the value to 'true' to sync MS Teams guest users", + "default": false + }, + { + "key": "syncDirectMessages", + "display_name": "Sync direct and group messages", + "type": "bool", + "help_text": "Sync direct and group messages where any of the user in the conversation is a real Mattermost user connected to MS Teams account", + "default": false + }, + { + "key": "selectiveSync", + "display_name": "Selective sync", + "type": "bool", + "help_text": "Skip syncing messages between users on the same platform.", + "default": false + }, + { + "key": "syncLinkedChannels", + "display_name": "Sync linked channels", + "type": "bool", + "help_text": "Sync messages from channels linked between Mattermost and MS Teams", + "default": false + }, + { + "key": "syncReactions", + "display_name": "Sync reactions", + "type": "bool", + "help_text": "Sync reactions on messages", + "default": false + }, + { + "key": "syncFileAttachments", + "display_name": "Sync file attachments", + "type": "bool", + "help_text": "Sync file attachments on messages", + "default": false + }, + { + "key": "enabledTeams", + "display_name": "Enabled Teams", + "type": "text", + "help_text": "Mattermost teams where sync is enabled (comma separated Mattermost team names, empty means all enabled)", + "default": "" + }, + { + "key": "maxSizeForCompleteDownload", + "display_name": "Maximum size of attachments to support complete one time download (in MB)", + "type": "number", + "help_text": "Set the maximum size for attachments that can be loaded into the memory. Attachments bigger than this size will be streamed from MS Teams to Mattermost", + "default": 20 + }, + { + "key": "bufferSizeForFileStreaming", + "display_name": "Buffer size for streaming files (in MB)", + "type": "number", + "help_text": "Set the buffer size for streaming files from MS Teams to Mattermost", + "default": 20 + }, + { + "key": "newUserConnections", + "display_name": "New User Connections", + "type": "dropdown", + "help_text": "When enabled, any user who has not connected will be allowed to connect their account. When set to Rollout, the bot will incrementally send connection invite direct messages to users as they login or become active; as invited users connect, spaces in the invite pool will open up and more invites will be sent out. \nRollout (open) — all users may connect and receive invites. \nRollout (open-restricted) — all users may connect, but only whitelisted users may receive invites.", + "default": "enabled", + "options": [ + { + "display_name": "Enabled: allow all users to connect, and do not send connection invites", + "value": "enabled" + }, + { + "display_name": "Rollout (open): allow all users to connect, and send connection invites to any user as they login or become active", + "value": "rolloutOpen" + }, + { + "display_name": "Rollout (open-restricted): allow all users to connect, and send connection invites to whitelisted users as they login or become active", + "value": "rolloutOpenRestricted" + } + ] + }, + { + "key": "connectedUsersAllowed", + "display_name": "Max Connected Users", + "type": "number", + "help_text": "The maximum number of users that may connect their MS Teams account. Once connected, users may reconnect at any time if they become disconnected.", + "default": 1000 + }, + { + "key": "connectedUsersInvitePoolSize", + "display_name": "Rollout Max. Pending Invitations", + "type": "number", + "help_text": "Set the maximum number of connection invites that may be pending at a given time during Rollout. As invited users connect, spaces in the invite pool will open up and more invites will be sent out as configured above.", + "default": 10 + }, + { + "key": "inviteWhitelistUpload", + "display_name": "Connection Invites: Whitelist", + "type": "custom", + "help_text": "", + "default": "" + }, + { + "key": "automaticallyPromoteSyntheticUsers", + "display_name": "Automatically Promote Synthetic Users", + "type": "bool", + "help_text": "When true, synthetic users will be converted to members when they login for the first time.", + "default": false + }, + { + "key": "disableSyncMsg", + "display_name": "Disable using the sync msg infrastructure for tracking message changes", + "type": "bool", + "help_text": "When true, the plugin will not enable any sync msg infrastructure.", + "default": false + }, + { + "key": "disableCheckCredentials", + "display_name": "Disable periodically checking the validity of the application credentials", + "type": "bool", + "help_text": "When true, the plugin will not periodically check the validity of the application credentials.", + "default": false + }, + { + "key": "syntheticUserAuthService", + "display_name": "Synthetic User Auth Service", + "type": "dropdown", + "help_text": "Select the authentication service to use when creating synthetic users. This should match the service used for member user access to Mattermost. Default is 'SAML'.", + "default": "saml", + "options": [ + { + "display_name": "SAML", + "value": "saml" + }, + { + "display_name": "AD/LDAP", + "value": "ldap" + } + ] + }, + { + "key": "syntheticUserAuthData", + "display_name": "Synthetic User Auth Data", + "type": "dropdown", + "help_text": "Select the MS Teams user property to use as the authentication identifier. For AD/LDAP and SAML, the identifier's value should match the value provided by the ID Attribute. ", + "default": "ID", + "options": [ + { + "display_name": "ID", + "value": "ID" + }, + { + "display_name": "Mail", + "value": "Mail" + }, + { + "display_name": "User Principal Name", + "value": "UserPrincipalName" + } + ] + }, + { + "key": "appManifestDownload", + "display_name": "Download Manifest", + "type": "custom", + "help_text": "", + "default": "" + }, + { + "key": "ConnectedUsersReportDownload", + "display_name": "Download Report", + "type": "custom", + "help_text": "", + "default": "" + } + ] } } diff --git a/server/api.go b/server/api.go index 88f234835..6c621f7b9 100644 --- a/server/api.go +++ b/server/api.go @@ -14,6 +14,7 @@ import ( "encoding/csv" "encoding/json" "fmt" + "io" "net/http" "path/filepath" "strconv" @@ -42,15 +43,23 @@ type Activities struct { } const ( - DefaultPage = 0 - MaxPerPage = 100 - QueryParamPage = "page" - QueryParamPerPage = "per_page" - QueryParamPrimaryPlatform = "primary_platform" + DefaultPage = 0 + MaxPerPage = 100 + UpdateWhitelistCsvParseErrThreshold = 0 + UpdateWhitelistNotFoundEmailsErrThreshold = 10 + QueryParamPage = "page" + QueryParamPerPage = "per_page" + QueryParamPrimaryPlatform = "primary_platform" APIChoosePrimaryPlatform = "/choose-primary-platform" ) +type UpdateWhitelistResult struct { + Count int `json:"count"` + Failed []string `json:"failed"` + FailedLines []string `json:"failedLines"` +} + func NewAPI(p *Plugin, store store.Store) *API { router := mux.NewRouter() p.handleStaticFiles(router) @@ -70,6 +79,8 @@ func NewAPI(p *Plugin, store store.Store) *API { router.HandleFunc("/oauth-redirect", api.oauthRedirectHandler).Methods("GET", "OPTIONS") router.HandleFunc("/connected-users", api.getConnectedUsers).Methods(http.MethodGet) router.HandleFunc("/connected-users/download", api.getConnectedUsersFile).Methods(http.MethodGet) + router.HandleFunc("/whitelist", api.updateWhitelist).Methods(http.MethodPut) + router.HandleFunc("/whitelist/download", api.getWhitelistEmailsFile).Methods(http.MethodGet) router.HandleFunc("/notify-connect", api.notifyConnect).Methods("GET") router.HandleFunc(APIChoosePrimaryPlatform, api.choosePrimaryPlatform).Methods(http.MethodGet) router.HandleFunc("/stats/site", api.siteStats).Methods("GET") @@ -360,6 +371,12 @@ func (a *API) connect(w http.ResponseWriter, r *http.Request) { func (a *API) notifyConnect(w http.ResponseWriter, r *http.Request) { userID := r.Header.Get("Mattermost-User-ID") + if userID == "" { + a.p.API.LogWarn("Not authorized") + http.Error(w, "not authorized", http.StatusUnauthorized) + return + } + if inviteWasSent, err := a.p.MaybeSendInviteMessage(userID); err != nil { a.p.API.LogWarn("Error in connection invite flow", "user_id", userID, "error", err.Error()) } else if inviteWasSent { @@ -457,66 +474,47 @@ func (a *API) oauthRedirectHandler(w http.ResponseWriter, r *http.Request) { return } - if err = a.p.store.SetUserInfo(mmUserID, msteamsUser.ID, token); err != nil { - a.p.API.LogWarn("Unable to store the token", "error", err.Error()) - http.Error(w, "failed to store the token", http.StatusInternalServerError) - return - } - - a.p.whitelistClusterMutex.Lock() - defer a.p.whitelistClusterMutex.Unlock() - - inWhitelist, err := a.p.store.IsUserPresentInWhitelist(mmUserID) - if err != nil { - a.p.API.LogWarn("Error in checking whitelist", "user_id", mmUserID, "error", err.Error()) - http.Error(w, "Something went wrong", http.StatusInternalServerError) - return - } - - whitelistSize, err := a.p.store.GetSizeOfWhitelist() - if err != nil { - a.p.API.LogWarn("Unable to get whitelist size", "error", err.Error()) - http.Error(w, "Something went wrong", http.StatusInternalServerError) - return - } - - invitedSize, err := a.p.store.GetSizeOfInvitedUsers() - if err != nil { - a.p.API.LogWarn("Unable to get invited size", "error", err.Error()) - http.Error(w, "Something went wrong", http.StatusInternalServerError) - return - } + a.p.connectClusterMutex.Lock() + defer a.p.connectClusterMutex.Unlock() - invitedUser, err := a.p.store.GetInvitedUser(mmUserID) + hasRightToConnect, err := a.p.UserHasRightToConnect(mmUserID) if err != nil { - a.p.API.LogWarn("Error in getting invited user", "user_id", mmUserID, "error", err.Error()) + a.p.API.LogWarn("Unable to check if user has the right to connect", "error", err.Error()) http.Error(w, "Something went wrong", http.StatusInternalServerError) return } - if !inWhitelist && (whitelistSize+invitedSize) >= a.p.getConfiguration().ConnectedUsersAllowed && invitedUser == nil { - if err = a.p.store.SetUserInfo(mmUserID, msteamsUser.ID, nil); err != nil { - a.p.API.LogWarn("Unable to delete the OAuth token for user", "user_id", mmUserID, "error", err.Error()) + if !hasRightToConnect { + canOpenlyConnect, openConnectErr := a.p.UserCanOpenlyConnect(mmUserID) + if openConnectErr != nil { + a.p.API.LogWarn("Unable to check if user can openly connect", "error", openConnectErr.Error()) + http.Error(w, "Something went wrong", http.StatusInternalServerError) + return } - http.Error(w, "You cannot connect your account because the maximum limit of users allowed to connect has been reached. Please contact your system administrator.", http.StatusBadRequest) - return - } - if err = a.p.store.StoreUserInWhitelist(mmUserID); err != nil { - a.p.API.LogWarn("Unable to store the user in whitelist", "user_id", mmUserID, "error", err.Error()) - if err = a.p.store.SetUserInfo(mmUserID, msteamsUser.ID, nil); err != nil { - a.p.API.LogWarn("Unable to delete the OAuth token for user", "user_id", mmUserID, "error", err.Error()) + if !canOpenlyConnect { + if err = a.p.store.SetUserInfo(mmUserID, msteamsUser.ID, nil); err != nil { + a.p.API.LogWarn("Unable to delete the OAuth token for user", "user_id", mmUserID, "error", err.Error()) + } + http.Error(w, "You cannot connect your account because the maximum limit of users allowed to connect has been reached. Please contact your system administrator.", http.StatusBadRequest) + return } + } - http.Error(w, "Something went wrong.", http.StatusInternalServerError) + if err = a.p.store.SetUserInfo(mmUserID, msteamsUser.ID, token); err != nil { + a.p.API.LogWarn("Unable to store the token", "error", err.Error()) + http.Error(w, "failed to store the token", http.StatusInternalServerError) return } - err = a.p.store.DeleteUserInvite(mmUserID) - if err != nil { + if err = a.p.store.DeleteUserInvite(mmUserID); err != nil { a.p.API.LogWarn("Unable to clear user invite", "user_id", mmUserID, "error", err.Error()) } + if err = a.p.store.DeleteUserFromWhitelist(mmUserID); err != nil { + a.p.API.LogWarn("Unable to remove user from whitelist", "user_id", mmUserID, "error", err.Error()) + } + w.Header().Add("Content-Type", "text/html") if mmUser.Id == a.p.GetBotUserID() { connectionMessage := "The bot account has been connected" @@ -554,7 +552,7 @@ func (a *API) oauthRedirectHandler(w http.ResponseWriter, r *http.Request) { } func (a *API) getConnectedUsers(w http.ResponseWriter, r *http.Request) { - userID := r.Header.Get("Mattermost-User-Id") + userID := r.Header.Get("Mattermost-User-ID") if userID == "" { a.p.API.LogWarn("Not authorized") http.Error(w, "not authorized", http.StatusUnauthorized) @@ -594,7 +592,7 @@ func (a *API) getConnectedUsers(w http.ResponseWriter, r *http.Request) { } func (a *API) getConnectedUsersFile(w http.ResponseWriter, r *http.Request) { - userID := r.Header.Get("Mattermost-User-Id") + userID := r.Header.Get("Mattermost-User-ID") if userID == "" { a.p.API.LogWarn("Not authorized") http.Error(w, "not authorized", http.StatusUnauthorized) @@ -645,8 +643,148 @@ func (a *API) getConnectedUsersFile(w http.ResponseWriter, r *http.Request) { } } +func (a *API) getWhitelistEmailsFile(w http.ResponseWriter, r *http.Request) { + userID := r.Header.Get("Mattermost-User-ID") + if userID == "" { + a.p.API.LogWarn("Not authorized") + http.Error(w, "not authorized", http.StatusUnauthorized) + return + } + + if !a.p.API.HasPermissionTo(userID, model.PermissionManageSystem) { + a.p.API.LogWarn("Insufficient permissions", "user_id", userID) + http.Error(w, "not able to authorize the user", http.StatusForbidden) + return + } + + whitelist, err := a.p.getWhitelistEmails() + if err != nil { + a.p.API.LogWarn("Unable to get whitelist", "error", err.Error()) + http.Error(w, "unable to get whitelist", http.StatusInternalServerError) + return + } + + b := &bytes.Buffer{} + csvWriter := csv.NewWriter(b) + if err := csvWriter.Write([]string{"Email"}); err != nil { + a.p.API.LogWarn("Unable to write headers in CSV file", "error", err.Error()) + http.Error(w, "unable to write data in CSV file", http.StatusInternalServerError) + return + } + + for _, email := range whitelist { + if err := csvWriter.Write([]string{email}); err != nil { + a.p.API.LogWarn("Unable to write data in CSV file", "error", err.Error()) + http.Error(w, "unable to write data in CSV file", http.StatusInternalServerError) + return + } + } + + csvWriter.Flush() + if err := csvWriter.Error(); err != nil { + a.p.API.LogWarn("Unable to flush the data in writer", "error", err.Error()) + http.Error(w, "unable to write data in CSV file", http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "text/csv") + w.Header().Set("Content-Disposition", "attachment;filename=invite-whitelist.csv") + if _, err := w.Write(b.Bytes()); err != nil { + a.p.API.LogWarn("Unable to write the data", "error", err.Error()) + http.Error(w, "unable to write the data", http.StatusInternalServerError) + } +} + +func (a *API) updateWhitelist(w http.ResponseWriter, r *http.Request) { + userID := r.Header.Get("Mattermost-User-ID") + if userID == "" { + a.p.API.LogWarn("Not authorized") + http.Error(w, "not authorized", http.StatusUnauthorized) + return + } + + if !a.p.API.HasPermissionTo(userID, model.PermissionManageSystem) { + a.p.API.LogWarn("Insufficient permissions", "user_id", userID) + http.Error(w, "not able to authorize the user", http.StatusForbidden) + return + } + + file, _, err := r.FormFile("file") + if err != nil { + a.p.API.LogWarn("Error reading whitelist file") + http.Error(w, "error reading whitelist", http.StatusBadRequest) + return + } + defer file.Close() + + reader := csv.NewReader(file) + columns, err := reader.Read() + if err != nil || strings.ToLower(columns[0]) != "email" || len(columns) != 1 { + a.p.API.LogWarn("Error parsing whitelist csv header") + http.Error(w, "error parsing whitelist - please check header and try again", http.StatusBadRequest) + return + } + + var ids []string + var failed []string + + var csvLineErrs []string + var i = 1 // offset, start line 1 + for { + i++ + row, readErr := reader.Read() + if readErr == io.EOF { + break + } + if readErr != nil { + csvLineErrs = append(csvLineErrs, strconv.Itoa(i)) + continue + } + if len(csvLineErrs) > UpdateWhitelistCsvParseErrThreshold { + break + } + email := row[0] + user, err := a.p.API.GetUserByEmail(email) + if err != nil { + a.p.API.LogWarn("Error could not find user with email", "line", i) + failed = append(failed, email) + continue + } + + ids = append(ids, user.Id) + } + + if len(csvLineErrs) > UpdateWhitelistCsvParseErrThreshold { + a.p.API.LogWarn("Error parsing whitelist csv data", "lines", csvLineErrs) + http.Error(w, "error parsing whitelist - please check data at line(s) "+strings.Join(csvLineErrs, ", ")+" and try again", http.StatusBadRequest) + return + } + + if len(failed) > UpdateWhitelistNotFoundEmailsErrThreshold { + a.p.API.LogWarn("Error: too many users not found", "threshold", UpdateWhitelistNotFoundEmailsErrThreshold, "failed", len(failed)) + http.Error(w, "error - could not find user(s): "+strings.Join(failed, ", "), http.StatusInternalServerError) + return + } + + if err := a.p.store.SetWhitelist(ids, MaxPerPage); err != nil { + a.p.API.LogWarn("Error processing whitelist", "error", err.Error()) + http.Error(w, "error processing whitelist - please check data and try again", http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(&UpdateWhitelistResult{ + Count: len(ids), + Failed: failed, + FailedLines: csvLineErrs, + }); err != nil { + a.p.API.LogWarn("Error writing update whitelist response") + } +} + func (a *API) choosePrimaryPlatform(w http.ResponseWriter, r *http.Request) { - userID := r.Header.Get("Mattermost-User-Id") + userID := r.Header.Get("Mattermost-User-ID") if userID == "" { a.p.API.LogWarn("Not authorized") http.Error(w, "not authorized", http.StatusUnauthorized) @@ -692,6 +830,27 @@ func (p *Plugin) getConnectedUsersList() ([]*storemodels.ConnectedUser, error) { return connectedUserList, nil } +func (p *Plugin) getWhitelistEmails() ([]string, error) { + page := DefaultPage + perPage := MaxPerPage + var result []string + for { + emails, err := p.store.GetWhitelistEmails(page, perPage) + if err != nil { + return nil, err + } + + result = append(result, emails...) + if len(emails) < perPage { + break + } + + page++ + } + + return result, nil +} + // handleStaticFiles handles the static files under the assets directory. func (p *Plugin) handleStaticFiles(r *mux.Router) { bundlePath, err := p.API.GetBundlePath() diff --git a/server/command.go b/server/command.go index 5b45d7e77..6f1d23f0e 100644 --- a/server/command.go +++ b/server/command.go @@ -457,20 +457,21 @@ func (p *Plugin) executeConnectCommand(args *model.CommandArgs) (*model.CommandR } genericErrorMessage := "Error in trying to connect the account, please try again." - presentInWhitelist, err := p.store.IsUserPresentInWhitelist(args.UserId) + + hasRightToConnect, err := p.UserHasRightToConnect(args.UserId) if err != nil { - p.API.LogWarn("Error in checking if a user is present in whitelist", "user_id", args.UserId, "error", err.Error()) + p.API.LogWarn("Error in checking if the user has the right to connect", "user_id", args.UserId, "error", err.Error()) return p.cmdError(args, genericErrorMessage) } - if !presentInWhitelist { - whitelistSize, err := p.store.GetSizeOfWhitelist() - if err != nil { - p.API.LogWarn("Error in getting the size of whitelist", "error", err.Error()) + if !hasRightToConnect { + canOpenlyConnect, openConnectErr := p.UserCanOpenlyConnect(args.UserId) + if openConnectErr != nil { + p.API.LogWarn("Error in checking if the user can openly connect", "user_id", args.UserId, "error", openConnectErr.Error()) return p.cmdError(args, genericErrorMessage) } - if whitelistSize >= p.getConfiguration().ConnectedUsersAllowed { + if !canOpenlyConnect { return p.cmdError(args, "You cannot connect your account because the maximum limit of users allowed to connect has been reached. Please contact your system administrator.") } } @@ -489,20 +490,21 @@ func (p *Plugin) executeConnectBotCommand(args *model.CommandArgs) (*model.Comma } genericErrorMessage := "Error in trying to connect the bot account, please try again." - presentInWhitelist, err := p.store.IsUserPresentInWhitelist(p.userID) + + hasRightToConnect, err := p.UserHasRightToConnect(p.userID) if err != nil { - p.API.LogWarn("Error in checking if the bot user is present in whitelist", "bot_user_id", p.userID, "error", err.Error()) + p.API.LogWarn("Error in checking if the bot user has the right to connect", "bot_user_id", p.userID, "error", err.Error()) return p.cmdError(args, genericErrorMessage) } - if !presentInWhitelist { - whitelistSize, err := p.store.GetSizeOfWhitelist() - if err != nil { - p.API.LogWarn("Error in getting the size of whitelist", "error", err.Error()) + if !hasRightToConnect { + canOpenlyConnect, openConnectErr := p.UserCanOpenlyConnect(p.userID) + if openConnectErr != nil { + p.API.LogWarn("Error in checking if the bot user can openly connect", "bot_user_id", p.userID, "error", openConnectErr.Error()) return p.cmdError(args, genericErrorMessage) } - if whitelistSize >= p.getConfiguration().ConnectedUsersAllowed { + if !canOpenlyConnect { return p.cmdError(args, "You cannot connect the bot account because the maximum limit of users allowed to connect has been reached.") } } diff --git a/server/configuration.go b/server/configuration.go index 13ed15611..f848ad915 100644 --- a/server/configuration.go +++ b/server/configuration.go @@ -39,6 +39,7 @@ type configuration struct { MaxSizeForCompleteDownload int `json:"maxSizeForCompleteDownload"` BufferSizeForFileStreaming int `json:"bufferSizeForFileStreaming"` ConnectedUsersAllowed int `json:"connectedUsersAllowed"` + NewUserConnections string `json:"newUserConnections"` ConnectedUsersInvitePoolSize int `json:"connectedUsersInvitePoolSize"` SyntheticUserAuthService string `json:"syntheticUserAuthService"` SyntheticUserAuthData string `json:"syntheticUserAuthData"` diff --git a/server/connect.go b/server/connect.go index 397f7d531..a9f097b27 100644 --- a/server/connect.go +++ b/server/connect.go @@ -9,6 +9,12 @@ import ( "github.com/pkg/errors" ) +const ( + NewConnectionsEnabled = "enabled" + NewConnectionsRolloutOpen = "rolloutOpen" + NewConnectionsRolloutOpenRestricted = "rolloutOpenRestricted" +) + func (p *Plugin) botSendDirectMessage(userID, message string) error { channel, err := p.apiClient.Channel.GetDirect(userID, p.userID) if err != nil { @@ -23,8 +29,8 @@ func (p *Plugin) botSendDirectMessage(userID, message string) error { } func (p *Plugin) MaybeSendInviteMessage(userID string) (bool, error) { - if p.getConfiguration().ConnectedUsersInvitePoolSize == 0 { - // connection invites disabled + if p.getConfiguration().NewUserConnections == NewConnectionsEnabled { + // new connections allowed, but invites disabled return false, nil } @@ -33,15 +39,27 @@ func (p *Plugin) MaybeSendInviteMessage(userID string) (bool, error) { return false, errors.Wrapf(err, "error getting user") } - p.whitelistClusterMutex.Lock() - defer p.whitelistClusterMutex.Unlock() + if p.getConfiguration().NewUserConnections == NewConnectionsRolloutOpenRestricted { + // new connections allowed, but invites restricted to whitelist + isWhitelisted, whitelistErr := p.store.IsUserWhitelisted(userID) + if whitelistErr != nil { + return false, errors.Wrapf(whitelistErr, "error getting user in whitelist") + } - userInWhitelist, err := p.store.IsUserPresentInWhitelist(user.Id) + if !isWhitelisted { + return false, nil + } + } + + p.connectClusterMutex.Lock() + defer p.connectClusterMutex.Unlock() + + hasConnected, err := p.store.UserHasConnected(user.Id) if err != nil { - return false, errors.Wrapf(err, "error getting user in whitelist") + return false, errors.Wrapf(err, "error checking user connected status") } - if userInWhitelist { + if hasConnected { // user already connected return false, nil } @@ -118,19 +136,59 @@ func (p *Plugin) shouldSendInviteMessage( } func (p *Plugin) moreInvitesAllowed() (bool, int, error) { - nWhitelisted, err := p.store.GetSizeOfWhitelist() + nConnected, err := p.store.GetHasConnectedCount() if err != nil { - return false, 0, errors.Wrapf(err, "error in getting the size of whitelist") + return false, 0, errors.Wrapf(err, "error in getting has-connected count") } - nInvited, err := p.store.GetSizeOfInvitedUsers() + nInvited, err := p.store.GetInvitedCount() if err != nil { - return false, 0, errors.Wrapf(err, "error in getting the number of invited users") + return false, 0, errors.Wrapf(err, "error in getting invited count") } - if (nWhitelisted + nInvited) >= p.getConfiguration().ConnectedUsersAllowed { + if (nConnected + nInvited) >= p.getConfiguration().ConnectedUsersAllowed { // only invite up to max connected return false, 0, nil } - return nInvited < p.getConfiguration().ConnectedUsersInvitePoolSize, nWhitelisted, nil + return nInvited < p.getConfiguration().ConnectedUsersInvitePoolSize, nConnected, nil +} + +func (p *Plugin) UserHasRightToConnect(mmUserID string) (bool, error) { + hasConnected, err := p.store.UserHasConnected(mmUserID) + if err != nil { + return false, errors.Wrapf(err, "error in checking if user has connected or not") + } + + if hasConnected { + return true, nil + } + + invitedUser, err := p.store.GetInvitedUser(mmUserID) + if err != nil { + return false, errors.Wrapf(err, "error in getting user invite") + } + + if invitedUser != nil { + return true, nil + } + + return false, nil +} + +func (p *Plugin) UserCanOpenlyConnect(mmUserID string) (bool, error) { + numHasConnected, err := p.store.GetHasConnectedCount() + if err != nil { + return false, errors.Wrapf(err, "error in getting has connected count") + } + + numInvited, err := p.store.GetInvitedCount() + if err != nil { + return false, errors.Wrapf(err, "error in getting invited count") + } + + if (numHasConnected + numInvited) >= p.getConfiguration().ConnectedUsersAllowed { + return false, nil + } + + return true, nil } diff --git a/server/helper_test.go b/server/helper_test.go index e2f057e80..e7404502f 100644 --- a/server/helper_test.go +++ b/server/helper_test.go @@ -136,7 +136,7 @@ func (th *testHelper) clearDatabase(t *testing.T) { require.NoError(t, err) _, err = db.Exec("DELETE FROM msteamssync_users") require.NoError(t, err) - _, err = db.Exec("DELETE FROM msteamssync_whitelisted_users") + _, err = db.Exec("DELETE FROM msteamssync_whitelist") require.NoError(t, err) } diff --git a/server/plugin.go b/server/plugin.go index e3214b99b..6cefc7b5a 100644 --- a/server/plugin.go +++ b/server/plugin.go @@ -42,7 +42,7 @@ const ( botDisplayName = "MS Teams" pluginID = "com.mattermost.msteams-sync" subscriptionsClusterMutexKey = "subscriptions_cluster_mutex" - whitelistClusterMutexKey = "whitelist_cluster_mutex" + connectClusterMutexKey = "connect_cluster_mutex" msteamsUserTypeGuest = "Guest" syncUsersJobName = "sync_users" metricsJobName = "metrics" @@ -74,7 +74,7 @@ type Plugin struct { store store.Store subscriptionsClusterMutex *cluster.Mutex - whitelistClusterMutex *cluster.Mutex + connectClusterMutex *cluster.Mutex monitor *monitor.Monitor syncUserJob *cluster.Job checkCredentialsJob *cluster.Job @@ -482,7 +482,7 @@ func (p *Plugin) onActivate() error { return err } - p.whitelistClusterMutex, err = cluster.NewMutex(p.API, whitelistClusterMutexKey) + p.connectClusterMutex, err = cluster.NewMutex(p.API, connectClusterMutexKey) if err != nil { return err } @@ -575,12 +575,6 @@ func (p *Plugin) onActivate() error { p.API.LogError("Recovering from panic", "panic", r, "stack", string(debug.Stack())) } }() - - p.whitelistClusterMutex.Lock() - defer p.whitelistClusterMutex.Unlock() - if err2 := p.store.PrefillWhitelist(); err2 != nil { - p.API.LogWarn("Error in populating the whitelist with already connected users", "error", err2.Error()) - } }() go p.start(false) diff --git a/server/store/mocks/Store.go b/server/store/mocks/Store.go index bad68bd83..2e34afb49 100644 --- a/server/store/mocks/Store.go +++ b/server/store/mocks/Store.go @@ -58,6 +58,20 @@ func (_m *Store) DeleteSubscription(subscriptionID string) error { return r0 } +// DeleteUserFromWhitelist provides a mock function with given fields: userID +func (_m *Store) DeleteUserFromWhitelist(userID string) error { + ret := _m.Called(userID) + + var r0 error + if rf, ok := ret.Get(0).(func(string) error); ok { + r0 = rf(userID) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // DeleteUserInfo provides a mock function with given fields: mmUserID func (_m *Store) DeleteUserInfo(mmUserID string) error { ret := _m.Called(mmUserID) @@ -201,6 +215,48 @@ func (_m *Store) GetGlobalSubscription(subscriptionID string) (*storemodels.Glob return r0, r1 } +// GetHasConnectedCount provides a mock function with given fields: +func (_m *Store) GetHasConnectedCount() (int, error) { + ret := _m.Called() + + var r0 int + if rf, ok := ret.Get(0).(func() int); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int) + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetInvitedCount provides a mock function with given fields: +func (_m *Store) GetInvitedCount() (int, error) { + ret := _m.Called() + + var r0 int + if rf, ok := ret.Get(0).(func() int); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int) + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetInvitedUser provides a mock function with given fields: mmUserID func (_m *Store) GetInvitedUser(mmUserID string) (*storemodels.InvitedUser, error) { ret := _m.Called(mmUserID) @@ -316,48 +372,6 @@ func (_m *Store) GetPostInfoByMattermostID(postID string) (*storemodels.PostInfo return r0, r1 } -// GetSizeOfInvitedUsers provides a mock function with given fields: -func (_m *Store) GetSizeOfInvitedUsers() (int, error) { - ret := _m.Called() - - var r0 int - if rf, ok := ret.Get(0).(func() int); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(int) - } - - var r1 error - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetSizeOfWhitelist provides a mock function with given fields: -func (_m *Store) GetSizeOfWhitelist() (int, error) { - ret := _m.Called() - - var r0 int - if rf, ok := ret.Get(0).(func() int); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(int) - } - - var r1 error - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - // GetStats provides a mock function with given fields: func (_m *Store) GetStats() (*storemodels.Stats, error) { ret := _m.Called() @@ -471,6 +485,73 @@ func (_m *Store) GetTokenForMattermostUser(userID string) (*oauth2.Token, error) return r0, r1 } +// GetUserConnectStatus provides a mock function with given fields: mmUserID +func (_m *Store) GetUserConnectStatus(mmUserID string) (*storemodels.UserConnectStatus, error) { + ret := _m.Called(mmUserID) + + var r0 *storemodels.UserConnectStatus + if rf, ok := ret.Get(0).(func(string) *storemodels.UserConnectStatus); ok { + r0 = rf(mmUserID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*storemodels.UserConnectStatus) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(mmUserID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetWhitelistCount provides a mock function with given fields: +func (_m *Store) GetWhitelistCount() (int, error) { + ret := _m.Called() + + var r0 int + if rf, ok := ret.Get(0).(func() int); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int) + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetWhitelistEmails provides a mock function with given fields: page, perPage +func (_m *Store) GetWhitelistEmails(page int, perPage int) ([]string, error) { + ret := _m.Called(page, perPage) + + var r0 []string + if rf, ok := ret.Get(0).(func(int, int) []string); ok { + r0 = rf(page, perPage) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]string) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(int, int) error); ok { + r1 = rf(page, perPage) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // Init provides a mock function with given fields: remoteID func (_m *Store) Init(remoteID string) error { ret := _m.Called(remoteID) @@ -485,8 +566,8 @@ func (_m *Store) Init(remoteID string) error { return r0 } -// IsUserPresentInWhitelist provides a mock function with given fields: userID -func (_m *Store) IsUserPresentInWhitelist(userID string) (bool, error) { +// IsUserWhitelisted provides a mock function with given fields: userID +func (_m *Store) IsUserWhitelisted(userID string) (bool, error) { ret := _m.Called(userID) var r0 bool @@ -702,20 +783,6 @@ func (_m *Store) MattermostToTeamsUserID(userID string) (string, error) { return r0, r1 } -// PrefillWhitelist provides a mock function with given fields: -func (_m *Store) PrefillWhitelist() error { - ret := _m.Called() - - var r0 error - if rf, ok := ret.Get(0).(func() error); ok { - r0 = rf() - } else { - r0 = ret.Error(0) - } - - return r0 -} - // RecoverPost provides a mock function with given fields: postID func (_m *Store) RecoverPost(postID string) error { ret := _m.Called(postID) @@ -814,6 +881,20 @@ func (_m *Store) SetUserInfo(userID string, msTeamsUserID string, token *oauth2. return r0 } +// SetWhitelist provides a mock function with given fields: userIDs, batchSize +func (_m *Store) SetWhitelist(userIDs []string, batchSize int) error { + ret := _m.Called(userIDs, batchSize) + + var r0 error + if rf, ok := ret.Get(0).(func([]string, int) error); ok { + r0 = rf(userIDs, batchSize) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // StoreChannelLink provides a mock function with given fields: link func (_m *Store) StoreChannelLink(link *storemodels.ChannelLink) error { ret := _m.Called(link) @@ -919,6 +1000,27 @@ func (_m *Store) UpdateSubscriptionLastActivityAt(subscriptionID string, lastAct return r0 } +// UserHasConnected provides a mock function with given fields: mmUserID +func (_m *Store) UserHasConnected(mmUserID string) (bool, error) { + ret := _m.Called(mmUserID) + + var r0 bool + if rf, ok := ret.Get(0).(func(string) bool); ok { + r0 = rf(mmUserID) + } else { + r0 = ret.Get(0).(bool) + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(mmUserID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // VerifyOAuth2State provides a mock function with given fields: state func (_m *Store) VerifyOAuth2State(state string) error { ret := _m.Called(state) diff --git a/server/store/sqlstore/migrations.go b/server/store/sqlstore/migrations.go index 13991037e..3738fb376 100644 --- a/server/store/sqlstore/migrations.go +++ b/server/store/sqlstore/migrations.go @@ -3,6 +3,7 @@ package sqlstore import ( "database/sql" "fmt" + "time" sq "github.com/Masterminds/squirrel" ) @@ -109,6 +110,59 @@ func (s *SQLStore) runMSTeamUserIDDedup() error { return err } +func (s *SQLStore) ensureMigrationWhitelistedUsers() error { + oldWhitelistToProcess, err := s.tableExist(whitelistedUsersLegacyTableName) + if err != nil { + return err + } + + if !oldWhitelistToProcess { + // migration already done, no rows to process + return nil + } + + s.api.LogInfo("Migrating old whitelist rows") + + now := time.Now() + + // all presently-whitelisted users should already in the users table, + // as being added to the old whitelist only happened after successful connection. + + // has-connected users (presently and previously) + _, err = s.getQueryBuilder(). + Update(usersTableName). + Set("lastConnectAt", now.UnixMicro()). + Where(sq.Or{ + sq.And{sq.NotEq{"token": ""}, sq.NotEq{"token": nil}}, + sq.Expr("mmUserID IN (SELECT mmUserID FROM " + whitelistedUsersLegacyTableName + ")"), + }). + Exec() + if err != nil { + return err + } + + // only previously-connected + _, err = s.getQueryBuilder(). + Update(usersTableName). + Set("lastDisconnectAt", now.UnixMicro()). + Where(sq.And{ + sq.Or{sq.Eq{"token": ""}, sq.Eq{"token": nil}}, + sq.Expr("mmUserID IN (SELECT mmUserID FROM " + whitelistedUsersLegacyTableName + ")"), + }). + Exec() + if err != nil { + return err + } + + err = s.deleteTable(whitelistedUsersLegacyTableName) + + if err != nil { + return err + } + + return nil +} + func (s *SQLStore) createTable(tableName, columnList string) error { if _, err := s.db.Exec(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (%s)", tableName, columnList)); err != nil { return err @@ -117,6 +171,14 @@ func (s *SQLStore) createTable(tableName, columnList string) error { return nil } +func (s *SQLStore) deleteTable(tableName string) error { + if _, err := s.db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)); err != nil { + return err + } + + return nil +} + func (s *SQLStore) createIndex(tableName, indexName, columnList string) error { if _, err := s.db.Exec(fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s (%s)", indexName, tableName, columnList)); err != nil { return err @@ -151,6 +213,16 @@ func (s *SQLStore) indexExist(tableName, indexName string) (bool, error) { return rows.Next(), nil } +func (s *SQLStore) tableExist(tableName string) (bool, error) { + rows, err := s.db.Query(fmt.Sprintf("SELECT 1 FROM pg_tables WHERE schemaname = 'public' AND tablename = '%s'", tableName)) + if err != nil { + return false, err + } + + defer rows.Close() + return rows.Next(), nil +} + func (s *SQLStore) addPrimaryKey(tableName, columnList string) error { rows, err := s.db.Query(fmt.Sprintf("SELECT constraint_name from information_schema.table_constraints where table_name = '%s' and constraint_type='PRIMARY KEY'", tableName)) if err != nil { diff --git a/server/store/sqlstore/store.go b/server/store/sqlstore/store.go index 2d56ee4f2..565246bfe 100644 --- a/server/store/sqlstore/store.go +++ b/server/store/sqlstore/store.go @@ -16,22 +16,24 @@ import ( ) const ( - connectionPromptKey = "connect_" - subscriptionRefreshTimeLimit = 5 * time.Minute - maxLimitForLinks = 100 - subscriptionTypeUser = "user" - subscriptionTypeChannel = "channel" - subscriptionTypeAllChats = "allChats" - oAuth2StateTimeToLive = 300 // seconds - oAuth2KeyPrefix = "oauth2_" - backgroundJobPrefix = "background_job" - usersTableName = "msteamssync_users" - linksTableName = "msteamssync_links" - postsTableName = "msteamssync_posts" - subscriptionsTableName = "msteamssync_subscriptions" - whitelistedUsersTableName = "msteamssync_whitelisted_users" - invitedUsersTableName = "msteamssync_invited_users" - PGUniqueViolationErrorCode = "23505" // See https://github.com/lib/pq/blob/master/error.go#L178 + connectionPromptKey = "connect_" + subscriptionRefreshTimeLimit = 5 * time.Minute + maxLimitForLinks = 100 + setWhitelistFailureThreshold = 0 + subscriptionTypeUser = "user" + subscriptionTypeChannel = "channel" + subscriptionTypeAllChats = "allChats" + oAuth2StateTimeToLive = 300 // seconds + oAuth2KeyPrefix = "oauth2_" + backgroundJobPrefix = "background_job" + usersTableName = "msteamssync_users" + linksTableName = "msteamssync_links" + postsTableName = "msteamssync_posts" + subscriptionsTableName = "msteamssync_subscriptions" + whitelistedUsersLegacyTableName = "msteamssync_whitelisted_users" // LEGACY-UNUSED + whitelistTableName = "msteamssync_whitelist" + invitedUsersTableName = "msteamssync_invited_users" + PGUniqueViolationErrorCode = "23505" // See https://github.com/lib/pq/blob/master/error.go#L178 ) type SQLStore struct { @@ -96,10 +98,6 @@ func (s *SQLStore) Init(remoteID string) error { return err } - if err := s.createTable(whitelistedUsersTableName, "mmUserID VARCHAR(255) PRIMARY KEY"); err != nil { - return err - } - if err := s.createTable(invitedUsersTableName, "mmUserID VARCHAR(255) PRIMARY KEY"); err != nil { return err } @@ -112,6 +110,18 @@ func (s *SQLStore) Init(remoteID string) error { return err } + if err := s.addColumn(usersTableName, "lastConnectAt", "BIGINT NOT NULL DEFAULT 0"); err != nil { + return err + } + + if err := s.addColumn(usersTableName, "lastDisconnectAt", "BIGINT NOT NULL DEFAULT 0"); err != nil { + return err + } + + if err := s.createTable(whitelistTableName, "mmUserID VARCHAR(255) PRIMARY KEY"); err != nil { + return err + } + if remoteID != "" { if err := s.runMigrationRemoteID(remoteID); err != nil { return err @@ -137,6 +147,10 @@ func (s *SQLStore) Init(remoteID string) error { } } + if err := s.ensureMigrationWhitelistedUsers(); err != nil { + return err + } + return nil } @@ -379,6 +393,82 @@ func (s *SQLStore) GetTokenForMSTeamsUser(userID string) (*oauth2.Token, error) return &token, nil } +func (s *SQLStore) UserHasConnected(mmUserID string) (bool, error) { + connectStatus, err := s.GetUserConnectStatus(mmUserID) + + if err != nil { + return false, err + } + + return !connectStatus.LastConnectAt.IsZero(), nil +} + +func (s *SQLStore) GetUserConnectStatus(mmUserID string) (*storemodels.UserConnectStatus, error) { + query := s.getQueryBuilder(). + Select("mmUserID", "token", "lastConnectAt", "lastDisconnectAt"). + From(usersTableName). + Where(sq.Eq{"mmUserID": mmUserID}) + + rows, err := query.Query() + if err != nil { + return nil, err + } + defer rows.Close() + + result := &storemodels.UserConnectStatus{} + if rows.Next() { + var encryptedToken string + var lastConnectAt int64 + var lastDisconnectAt int64 + + if scanErr := rows.Scan(&result.ID, &encryptedToken, &lastConnectAt, &lastDisconnectAt); scanErr != nil { + return nil, scanErr + } + + if encryptedToken != "" { + result.Connected = true + } + + if lastConnectAt != 0 { + result.LastConnectAt = time.UnixMicro(lastConnectAt) + } + + if lastDisconnectAt != 0 { + result.LastDisconnectAt = time.UnixMicro(lastDisconnectAt) + } + } + + return result, nil +} + +func computeStatusTimes(status *storemodels.UserConnectStatus, nextIsConnected bool) (int64, int64, error) { + var lastConnectAt int64 + var lastDisconnectAt int64 + + now := time.Now() + + if nextIsConnected { + // connected + lastConnectAt = now.UnixMicro() // bump always if new token + + if !status.LastDisconnectAt.IsZero() { + lastDisconnectAt = status.LastDisconnectAt.UnixMicro() // no change, pass-through + } + } else { + if !status.LastConnectAt.IsZero() { + lastConnectAt = status.LastConnectAt.UnixMicro() // pass-through + } + + if status.Connected { + lastDisconnectAt = now.UnixMicro() // bump only on actual disconnect + } else if !status.LastDisconnectAt.IsZero() { + lastDisconnectAt = status.LastDisconnectAt.UnixMicro() // no change, pass-through + } + } + + return lastConnectAt, lastDisconnectAt, nil +} + func (s *SQLStore) SetUserInfo(userID string, msTeamsUserID string, token *oauth2.Token) error { var encryptedToken string if token != nil { @@ -395,11 +485,21 @@ func (s *SQLStore) SetUserInfo(userID string, msTeamsUserID string, token *oauth } } + currentConnectStatus, err := s.GetUserConnectStatus(userID) + if err != nil { + return err + } + + lastConnectAt, lastDisconnectAt, err := computeStatusTimes(currentConnectStatus, encryptedToken != "") + if err != nil { + return err + } + if err := s.DeleteUserInfo(userID); err != nil { return err } - if _, err := s.getQueryBuilder().Insert(usersTableName).Columns("mmUserID, msTeamsUserID, token").Values(userID, msTeamsUserID, encryptedToken).Suffix("ON CONFLICT (mmUserID, msTeamsUserID) DO UPDATE SET token = EXCLUDED.token").Exec(); err != nil { + if _, err := s.getQueryBuilder().Insert(usersTableName).Columns("mmUserID, msTeamsUserID, token, lastConnectAt, lastDisconnectAt").Values(userID, msTeamsUserID, encryptedToken, lastConnectAt, lastDisconnectAt).Suffix("ON CONFLICT (mmUserID, msTeamsUserID) DO UPDATE SET token = EXCLUDED.token, lastConnectAt = EXCLUDED.lastConnectAt, lastDisconnectAt = EXCLUDED.lastDisconnectAt").Exec(); err != nil { return err } return nil @@ -822,43 +922,11 @@ func (s *SQLStore) GetConnectedUsers(page, perPage int) ([]*storemodels.Connecte return connectedUsers, nil } -func (s *SQLStore) PrefillWhitelist() error { - page := 0 - perPage := 100 - for { - query := s.getQueryBuilder().Select("mmuserid").From(usersTableName).Where(sq.NotEq{"token": ""}).Offset(uint64(page * perPage)).Limit(uint64(perPage)) - rows, err := query.Query() - if err != nil { - return err - } - - count := 0 - for rows.Next() { - count++ - var connectedUserID string - if err := rows.Scan(&connectedUserID); err != nil { - s.api.LogDebug("Unable to scan the result", "Error", err.Error()) - continue - } - - if err := s.StoreUserInWhitelist(connectedUserID); err != nil { - s.api.LogDebug("Unable to store user in whitelist", "UserID", connectedUserID, "Error", err.Error()) - } - } - - rows.Close() - if count < perPage { - break - } - - page++ - } - - return nil -} - -func (s *SQLStore) GetSizeOfWhitelist() (int, error) { - query := s.getQueryBuilder().Select("count(*)").From(whitelistedUsersTableName) +func (s *SQLStore) GetHasConnectedCount() (int, error) { + query := s.getQueryBuilder(). + Select("count(*)"). + From(usersTableName). + Where(sq.And{sq.NotEq{"lastConnectAt": 0}}) rows, err := query.Query() if err != nil { return 0, err @@ -876,7 +944,7 @@ func (s *SQLStore) GetSizeOfWhitelist() (int, error) { } func (s *SQLStore) StoreUserInWhitelist(userID string) error { - query := s.getQueryBuilder().Insert(whitelistedUsersTableName).Columns("mmUserID").Values(userID) + query := s.getQueryBuilder().Insert(whitelistTableName).Columns("mmUserID").Values(userID) if _, err := query.Exec(); err != nil { if isDuplicate(err) { s.api.LogDebug("UserID already present in whitelist", "UserID", userID) @@ -889,8 +957,68 @@ func (s *SQLStore) StoreUserInWhitelist(userID string) error { return nil } -func (s *SQLStore) IsUserPresentInWhitelist(userID string) (bool, error) { - query := s.getQueryBuilder().Select("mmUserID").From(whitelistedUsersTableName).Where(sq.Eq{"mmUserID": userID}) +func (s *SQLStore) storeUsersInWhitelist(userIDs []string, tx *sql.Tx) error { + query := s.getQueryBuilder(). + Insert(whitelistTableName). + Columns("mmUserID"). + RunWith(tx) + + for _, userID := range userIDs { + query = query.Values(userID) + } + + if _, err := query.Exec(); err != nil { + // TODO handle duplicates + return err + } + + return nil +} + +func (s *SQLStore) SetWhitelist(userIDs []string, batchSize int) error { + var err error + tx, err := s.db.Begin() + if err != nil { + return err + } + defer func() { + if err != nil { + s.api.LogDebug("Error processing whitelist, rolling back tx", "error", err.Error()) + err = tx.Rollback() + if err != nil { + s.api.LogError("Error rolling back add whitelist tx", "error", err.Error()) + } + return + } + err = tx.Commit() + if err != nil { + s.api.LogDebug("Error committing tx", "error", err.Error()) + } + }() + + if err = s.deleteWhitelist(tx); err != nil { + s.api.LogDebug("Error deleting whitelist") + return err + } + + var currentBatch []string + + for i, id := range userIDs { + currentBatch = append(currentBatch, id) + if len(currentBatch) >= batchSize || i == len(userIDs)-1 { + if err = s.storeUsersInWhitelist(currentBatch, tx); err != nil { + s.api.LogDebug("Error adding batched users to whitelist", "error", err.Error(), "userIds", currentBatch) + return err + } + clear(currentBatch) + } + } + + return nil +} + +func (s *SQLStore) IsUserWhitelisted(userID string) (bool, error) { + query := s.getQueryBuilder().Select("mmUserID").From(whitelistTableName).Where(sq.Eq{"mmUserID": userID}) rows, err := query.Query() if err != nil { return false, err @@ -907,6 +1035,67 @@ func (s *SQLStore) IsUserPresentInWhitelist(userID string) (bool, error) { return result != "", nil } +func (s *SQLStore) DeleteUserFromWhitelist(mmUserID string) error { + if _, err := s.getQueryBuilder().Delete(whitelistTableName).Where(sq.Eq{"mmUserID": mmUserID}).Exec(); err != nil { + return err + } + + return nil +} + +func (s *SQLStore) GetWhitelistCount() (int, error) { + query := s.getQueryBuilder().Select("count(*)").From(whitelistTableName) + rows, err := query.Query() + if err != nil { + return 0, err + } + defer rows.Close() + + var result int + if rows.Next() { + if scanErr := rows.Scan(&result); scanErr != nil { + return 0, scanErr + } + } + + return result, nil +} + +func (s *SQLStore) GetWhitelistEmails(page, perPage int) ([]string, error) { + query := s.getQueryBuilder(). + Select("Users.Email"). + From(whitelistTableName). + LeftJoin("Users ON Users.Id = msteamssync_whitelist.mmuserid"). + Offset(uint64(page * perPage)). + Limit(uint64(perPage)) + rows, err := query.Query() + if err != nil { + return nil, err + } + defer rows.Close() + + var result []string + for rows.Next() { + var email string + if err := rows.Scan(&email); err != nil { + s.api.LogDebug("Unable to scan the result", "Error", err.Error()) + continue + } + + result = append(result, email) + } + + return result, nil +} + +func (s *SQLStore) deleteWhitelist(tx *sql.Tx) error { + if _, err := s.getQueryBuilder().Delete(whitelistTableName).RunWith(tx).Exec(); err != nil { + return err + } + + return nil +} + func (s *SQLStore) StoreInvitedUser(invitedUser *storemodels.InvitedUser) error { pendingSince := invitedUser.InvitePendingSince.UnixMicro() lastSentAt := invitedUser.InviteLastSentAt.UnixMicro() @@ -936,24 +1125,27 @@ func (s *SQLStore) GetInvitedUser(mmUserID string) (*storemodels.InvitedUser, er } defer rows.Close() - var result *storemodels.InvitedUser if rows.Next() { - var id string + var result = &storemodels.InvitedUser{} var pendingSince int64 var lastSentAt int64 - if scanErr := rows.Scan(&id, &pendingSince, &lastSentAt); scanErr != nil { + if scanErr := rows.Scan(&result.ID, &pendingSince, &lastSentAt); scanErr != nil { return nil, scanErr } - result = &storemodels.InvitedUser{ - ID: id, - InvitePendingSince: time.UnixMicro(pendingSince), - InviteLastSentAt: time.UnixMicro(pendingSince), + if pendingSince != 0 { + result.InvitePendingSince = time.UnixMicro(pendingSince) + } + + if lastSentAt != 0 { + result.InvitePendingSince = time.UnixMicro(lastSentAt) } + + return result, nil } - return result, nil + return nil, nil } func (s *SQLStore) DeleteUserInvite(mmUserID string) error { @@ -964,7 +1156,7 @@ func (s *SQLStore) DeleteUserInvite(mmUserID string) error { return nil } -func (s *SQLStore) GetSizeOfInvitedUsers() (int, error) { +func (s *SQLStore) GetInvitedCount() (int, error) { query := s.getQueryBuilder().Select("count(*)").From(invitedUsersTableName) rows, err := query.Query() if err != nil { diff --git a/server/store/sqlstore/store_test.go b/server/store/sqlstore/store_test.go index 0d9486037..e9d86a4db 100644 --- a/server/store/sqlstore/store_test.go +++ b/server/store/sqlstore/store_test.go @@ -971,79 +971,48 @@ func TestListConnectedUsers(t *testing.T) { assert.Nil(delErr) } -func TestStoreUserAndIsUserPresentAndGetSizeOfWhitelist(t *testing.T) { +func TestWhitelistIO(t *testing.T) { store, _ := setupTestStore(t) assert := assert.New(t) - count, getErr := store.GetSizeOfWhitelist() + count, getErr := store.GetWhitelistCount() assert.Equal(0, count) assert.Nil(getErr) - storeErr := store.StoreUserInWhitelist(testutils.GetUserID()) + storeErr := store.StoreUserInWhitelist(testutils.GetUserID() + "1") assert.Nil(storeErr) - count, getErr = store.GetSizeOfWhitelist() + count, getErr = store.GetWhitelistCount() assert.Equal(1, count) assert.Nil(getErr) - present, presentErr := store.IsUserPresentInWhitelist(testutils.GetUserID()) + present, presentErr := store.IsUserWhitelisted(testutils.GetUserID() + "1") assert.Equal(true, present) assert.Nil(presentErr) - present, presentErr = store.IsUserPresentInWhitelist(testutils.GetTeamsUserID()) + present, presentErr = store.IsUserWhitelisted(testutils.GetTeamsUserID() + "1") assert.Equal(false, present) assert.Nil(presentErr) - storeErr = store.StoreUserInWhitelist(testutils.GetTeamsUserID()) + storeErr = store.StoreUserInWhitelist(testutils.GetUserID() + "2") assert.Nil(storeErr) - count, getErr = store.GetSizeOfWhitelist() + count, getErr = store.GetWhitelistCount() assert.Equal(2, count) assert.Nil(getErr) - present, presentErr = store.IsUserPresentInWhitelist(testutils.GetTeamsUserID()) + present, presentErr = store.IsUserWhitelisted(testutils.GetUserID() + "2") assert.Equal(true, present) assert.Nil(presentErr) - _, err := store.getQueryBuilder().Delete(whitelistedUsersTableName).Exec() - assert.Nil(err) -} - -func TestPrefillWhitelist(t *testing.T) { - store, _ := setupTestStore(t) - assert := assert.New(t) - store.encryptionKey = func() []byte { - return make([]byte, 16) - } - - token := &oauth2.Token{ - AccessToken: "mockAccessToken-1", - RefreshToken: "mockRefreshToken-1", - } - - storeErr := store.SetUserInfo(testutils.GetID()+"1", testutils.GetTeamsUserID()+"1", token) - assert.Nil(storeErr) - - storeErr = store.SetUserInfo(testutils.GetID()+"2", testutils.GetTeamsUserID()+"2", nil) - assert.Nil(storeErr) + tx, txErr := store.db.Begin() + assert.Nil(txErr) + delErr := store.deleteWhitelist(tx) + assert.Nil(delErr) + txCommitErr := tx.Commit() + assert.Nil(txCommitErr) - count, getErr := store.GetSizeOfWhitelist() + count, getErr = store.GetWhitelistCount() assert.Equal(0, count) assert.Nil(getErr) - - prefillErr := store.PrefillWhitelist() - assert.Nil(prefillErr) - - count, getErr = store.GetSizeOfWhitelist() - assert.Equal(1, count) - assert.Nil(getErr) - - _, err := store.getQueryBuilder().Delete(whitelistedUsersTableName).Exec() - assert.Nil(err) - - delErr := store.DeleteUserInfo(testutils.GetID() + "1") - assert.Nil(delErr) - - delErr = store.DeleteUserInfo(testutils.GetID() + "2") - assert.Nil(delErr) } diff --git a/server/store/store.go b/server/store/store.go index 61d3a2d4d..ec0659802 100644 --- a/server/store/store.go +++ b/server/store/store.go @@ -11,6 +11,42 @@ import ( type Store interface { Init(remoteID string) error + + // teams + CheckEnabledTeamByTeamID(teamID string) bool + + // users + TeamsToMattermostUserID(userID string) (string, error) + MattermostToTeamsUserID(userID string) (string, error) + GetTokenForMattermostUser(userID string) (*oauth2.Token, error) + GetTokenForMSTeamsUser(userID string) (*oauth2.Token, error) + GetConnectedUsers(page, perPage int) ([]*storemodels.ConnectedUser, error) + UserHasConnected(mmUserID string) (bool, error) + GetUserConnectStatus(mmUserID string) (*storemodels.UserConnectStatus, error) + GetHasConnectedCount() (int, error) + SetUserInfo(userID string, msTeamsUserID string, token *oauth2.Token) error + DeleteUserInfo(mmUserID string) error + + // auth + StoreOAuth2State(state string) error + VerifyOAuth2State(state string) error + + // invites & whitelist + StoreInvitedUser(invitedUser *storemodels.InvitedUser) error + GetInvitedUser(mmUserID string) (*storemodels.InvitedUser, error) + DeleteUserInvite(mmUserID string) error + GetInvitedCount() (int, error) + StoreUserInWhitelist(userID string) error + IsUserWhitelisted(userID string) (bool, error) + DeleteUserFromWhitelist(userID string) error + GetWhitelistCount() (int, error) + GetWhitelistEmails(page int, perPage int) ([]string, error) + SetWhitelist(userIDs []string, batchSize int) error + + // stats + GetStats() (*storemodels.Stats, error) + + // links, channels, posts GetLinkByChannelID(channelID string) (*storemodels.ChannelLink, error) ListChannelLinks() ([]storemodels.ChannelLink, error) ListChannelLinksWithNames() ([]*storemodels.ChannelLink, error) @@ -22,13 +58,9 @@ type Store interface { LinkPosts(postInfo storemodels.PostInfo) error SetPostLastUpdateAtByMattermostID(postID string, lastUpdateAt time.Time) error SetPostLastUpdateAtByMSTeamsID(postID string, lastUpdateAt time.Time) error - GetTokenForMattermostUser(userID string) (*oauth2.Token, error) - GetTokenForMSTeamsUser(userID string) (*oauth2.Token, error) - SetUserInfo(userID string, msTeamsUserID string, token *oauth2.Token) error - DeleteUserInfo(mmUserID string) error - TeamsToMattermostUserID(userID string) (string, error) - MattermostToTeamsUserID(userID string) (string, error) - CheckEnabledTeamByTeamID(teamID string) bool + RecoverPost(postID string) error + + // subscriptions ListGlobalSubscriptions() ([]*storemodels.GlobalSubscription, error) ListGlobalSubscriptionsToRefresh(certificate string) ([]*storemodels.GlobalSubscription, error) ListChatSubscriptionsToCheck() ([]storemodels.ChatSubscription, error) @@ -44,19 +76,6 @@ type Store interface { GetChatSubscription(subscriptionID string) (*storemodels.ChatSubscription, error) GetGlobalSubscription(subscriptionID string) (*storemodels.GlobalSubscription, error) GetSubscriptionType(subscriptionID string) (string, error) - RecoverPost(postID string) error - StoreOAuth2State(state string) error - VerifyOAuth2State(state string) error - GetStats() (*storemodels.Stats, error) - GetConnectedUsers(page, perPage int) ([]*storemodels.ConnectedUser, error) - PrefillWhitelist() error - GetSizeOfWhitelist() (int, error) - StoreUserInWhitelist(userID string) error - IsUserPresentInWhitelist(userID string) (bool, error) - StoreInvitedUser(invitedUser *storemodels.InvitedUser) error - GetInvitedUser(mmUserID string) (*storemodels.InvitedUser, error) - DeleteUserInvite(mmUserID string) error - GetSizeOfInvitedUsers() (int, error) UpdateSubscriptionLastActivityAt(subscriptionID string, lastActivityAt time.Time) error GetSubscriptionsLastActivityAt() (map[string]time.Time, error) } diff --git a/server/store/storemodels/storemodels.go b/server/store/storemodels/storemodels.go index db86e5002..9e81d1dd8 100644 --- a/server/store/storemodels/storemodels.go +++ b/server/store/storemodels/storemodels.go @@ -1,6 +1,8 @@ package storemodels -import "time" +import ( + "time" +) type Stats struct { ConnectedUsers int64 @@ -58,6 +60,13 @@ type ConnectedUser struct { Email string } +type UserConnectStatus struct { + ID string + Connected bool + LastConnectAt time.Time + LastDisconnectAt time.Time +} + type InvitedUser struct { ID string InvitePendingSince time.Time diff --git a/server/store/timerlayer/timerlayer.go b/server/store/timerlayer/timerlayer.go index 5a6416b0c..6b5532ff0 100644 --- a/server/store/timerlayer/timerlayer.go +++ b/server/store/timerlayer/timerlayer.go @@ -63,6 +63,20 @@ func (s *TimerLayer) DeleteSubscription(subscriptionID string) error { return err } +func (s *TimerLayer) DeleteUserFromWhitelist(userID string) error { + start := time.Now() + + err := s.Store.DeleteUserFromWhitelist(userID) + + elapsed := float64(time.Since(start)) / float64(time.Second) + success := "false" + if err == nil { + success = "true" + } + s.metrics.ObserveStoreMethodDuration("Store.DeleteUserFromWhitelist", success, elapsed) + return err +} + func (s *TimerLayer) DeleteUserInfo(mmUserID string) error { start := time.Now() @@ -161,101 +175,101 @@ func (s *TimerLayer) GetGlobalSubscription(subscriptionID string) (*storemodels. return result, err } -func (s *TimerLayer) GetInvitedUser(mmUserID string) (*storemodels.InvitedUser, error) { +func (s *TimerLayer) GetHasConnectedCount() (int, error) { start := time.Now() - result, err := s.Store.GetInvitedUser(mmUserID) + result, err := s.Store.GetHasConnectedCount() elapsed := float64(time.Since(start)) / float64(time.Second) success := "false" if err == nil { success = "true" } - s.metrics.ObserveStoreMethodDuration("Store.GetInvitedUser", success, elapsed) + s.metrics.ObserveStoreMethodDuration("Store.GetHasConnectedCount", success, elapsed) return result, err } -func (s *TimerLayer) GetLinkByChannelID(channelID string) (*storemodels.ChannelLink, error) { +func (s *TimerLayer) GetInvitedCount() (int, error) { start := time.Now() - result, err := s.Store.GetLinkByChannelID(channelID) + result, err := s.Store.GetInvitedCount() elapsed := float64(time.Since(start)) / float64(time.Second) success := "false" if err == nil { success = "true" } - s.metrics.ObserveStoreMethodDuration("Store.GetLinkByChannelID", success, elapsed) + s.metrics.ObserveStoreMethodDuration("Store.GetInvitedCount", success, elapsed) return result, err } -func (s *TimerLayer) GetLinkByMSTeamsChannelID(teamID string, channelID string) (*storemodels.ChannelLink, error) { +func (s *TimerLayer) GetInvitedUser(mmUserID string) (*storemodels.InvitedUser, error) { start := time.Now() - result, err := s.Store.GetLinkByMSTeamsChannelID(teamID, channelID) + result, err := s.Store.GetInvitedUser(mmUserID) elapsed := float64(time.Since(start)) / float64(time.Second) success := "false" if err == nil { success = "true" } - s.metrics.ObserveStoreMethodDuration("Store.GetLinkByMSTeamsChannelID", success, elapsed) + s.metrics.ObserveStoreMethodDuration("Store.GetInvitedUser", success, elapsed) return result, err } -func (s *TimerLayer) GetPostInfoByMSTeamsID(chatID string, postID string) (*storemodels.PostInfo, error) { +func (s *TimerLayer) GetLinkByChannelID(channelID string) (*storemodels.ChannelLink, error) { start := time.Now() - result, err := s.Store.GetPostInfoByMSTeamsID(chatID, postID) + result, err := s.Store.GetLinkByChannelID(channelID) elapsed := float64(time.Since(start)) / float64(time.Second) success := "false" if err == nil { success = "true" } - s.metrics.ObserveStoreMethodDuration("Store.GetPostInfoByMSTeamsID", success, elapsed) + s.metrics.ObserveStoreMethodDuration("Store.GetLinkByChannelID", success, elapsed) return result, err } -func (s *TimerLayer) GetPostInfoByMattermostID(postID string) (*storemodels.PostInfo, error) { +func (s *TimerLayer) GetLinkByMSTeamsChannelID(teamID string, channelID string) (*storemodels.ChannelLink, error) { start := time.Now() - result, err := s.Store.GetPostInfoByMattermostID(postID) + result, err := s.Store.GetLinkByMSTeamsChannelID(teamID, channelID) elapsed := float64(time.Since(start)) / float64(time.Second) success := "false" if err == nil { success = "true" } - s.metrics.ObserveStoreMethodDuration("Store.GetPostInfoByMattermostID", success, elapsed) + s.metrics.ObserveStoreMethodDuration("Store.GetLinkByMSTeamsChannelID", success, elapsed) return result, err } -func (s *TimerLayer) GetSizeOfInvitedUsers() (int, error) { +func (s *TimerLayer) GetPostInfoByMSTeamsID(chatID string, postID string) (*storemodels.PostInfo, error) { start := time.Now() - result, err := s.Store.GetSizeOfInvitedUsers() + result, err := s.Store.GetPostInfoByMSTeamsID(chatID, postID) elapsed := float64(time.Since(start)) / float64(time.Second) success := "false" if err == nil { success = "true" } - s.metrics.ObserveStoreMethodDuration("Store.GetSizeOfInvitedUsers", success, elapsed) + s.metrics.ObserveStoreMethodDuration("Store.GetPostInfoByMSTeamsID", success, elapsed) return result, err } -func (s *TimerLayer) GetSizeOfWhitelist() (int, error) { +func (s *TimerLayer) GetPostInfoByMattermostID(postID string) (*storemodels.PostInfo, error) { start := time.Now() - result, err := s.Store.GetSizeOfWhitelist() + result, err := s.Store.GetPostInfoByMattermostID(postID) elapsed := float64(time.Since(start)) / float64(time.Second) success := "false" if err == nil { success = "true" } - s.metrics.ObserveStoreMethodDuration("Store.GetSizeOfWhitelist", success, elapsed) + s.metrics.ObserveStoreMethodDuration("Store.GetPostInfoByMattermostID", success, elapsed) return result, err } @@ -329,6 +343,48 @@ func (s *TimerLayer) GetTokenForMattermostUser(userID string) (*oauth2.Token, er return result, err } +func (s *TimerLayer) GetUserConnectStatus(mmUserID string) (*storemodels.UserConnectStatus, error) { + start := time.Now() + + result, err := s.Store.GetUserConnectStatus(mmUserID) + + elapsed := float64(time.Since(start)) / float64(time.Second) + success := "false" + if err == nil { + success = "true" + } + s.metrics.ObserveStoreMethodDuration("Store.GetUserConnectStatus", success, elapsed) + return result, err +} + +func (s *TimerLayer) GetWhitelistCount() (int, error) { + start := time.Now() + + result, err := s.Store.GetWhitelistCount() + + elapsed := float64(time.Since(start)) / float64(time.Second) + success := "false" + if err == nil { + success = "true" + } + s.metrics.ObserveStoreMethodDuration("Store.GetWhitelistCount", success, elapsed) + return result, err +} + +func (s *TimerLayer) GetWhitelistEmails(page int, perPage int) ([]string, error) { + start := time.Now() + + result, err := s.Store.GetWhitelistEmails(page, perPage) + + elapsed := float64(time.Since(start)) / float64(time.Second) + success := "false" + if err == nil { + success = "true" + } + s.metrics.ObserveStoreMethodDuration("Store.GetWhitelistEmails", success, elapsed) + return result, err +} + func (s *TimerLayer) Init(remoteID string) error { start := time.Now() @@ -343,17 +399,17 @@ func (s *TimerLayer) Init(remoteID string) error { return err } -func (s *TimerLayer) IsUserPresentInWhitelist(userID string) (bool, error) { +func (s *TimerLayer) IsUserWhitelisted(userID string) (bool, error) { start := time.Now() - result, err := s.Store.IsUserPresentInWhitelist(userID) + result, err := s.Store.IsUserWhitelisted(userID) elapsed := float64(time.Since(start)) / float64(time.Second) success := "false" if err == nil { success = "true" } - s.metrics.ObserveStoreMethodDuration("Store.IsUserPresentInWhitelist", success, elapsed) + s.metrics.ObserveStoreMethodDuration("Store.IsUserWhitelisted", success, elapsed) return result, err } @@ -483,20 +539,6 @@ func (s *TimerLayer) MattermostToTeamsUserID(userID string) (string, error) { return result, err } -func (s *TimerLayer) PrefillWhitelist() error { - start := time.Now() - - err := s.Store.PrefillWhitelist() - - elapsed := float64(time.Since(start)) / float64(time.Second) - success := "false" - if err == nil { - success = "true" - } - s.metrics.ObserveStoreMethodDuration("Store.PrefillWhitelist", success, elapsed) - return err -} - func (s *TimerLayer) RecoverPost(postID string) error { start := time.Now() @@ -595,6 +637,20 @@ func (s *TimerLayer) SetUserInfo(userID string, msTeamsUserID string, token *oau return err } +func (s *TimerLayer) SetWhitelist(userIDs []string, batchSize int) error { + start := time.Now() + + err := s.Store.SetWhitelist(userIDs, batchSize) + + elapsed := float64(time.Since(start)) / float64(time.Second) + success := "false" + if err == nil { + success = "true" + } + s.metrics.ObserveStoreMethodDuration("Store.SetWhitelist", success, elapsed) + return err +} + func (s *TimerLayer) StoreChannelLink(link *storemodels.ChannelLink) error { start := time.Now() @@ -693,6 +749,20 @@ func (s *TimerLayer) UpdateSubscriptionLastActivityAt(subscriptionID string, las return err } +func (s *TimerLayer) UserHasConnected(mmUserID string) (bool, error) { + start := time.Now() + + result, err := s.Store.UserHasConnected(mmUserID) + + elapsed := float64(time.Since(start)) / float64(time.Second) + success := "false" + if err == nil { + success = "true" + } + s.metrics.ObserveStoreMethodDuration("Store.UserHasConnected", success, elapsed) + return result, err +} + func (s *TimerLayer) VerifyOAuth2State(state string) error { start := time.Now() diff --git a/webapp/src/client.ts b/webapp/src/client.ts index d3470c031..e7ff90e69 100644 --- a/webapp/src/client.ts +++ b/webapp/src/client.ts @@ -21,6 +21,10 @@ class ClientClass { await this.doGet(`${this.url}/notify-connect`); }; + uploadWhitelist = async (fileData: File) => { + return this.uploadFile(`${this.url}/whitelist`, fileData); + }; + fetchSiteStats = async (): Promise => { const data = await this.doGet(`${this.url}/stats/site`); if (!data) { @@ -122,6 +126,33 @@ class ClientClass { url, }); }; + + uploadFile = async (url: string, fileData: File, headers: {[key: string]: any} = {}) => { + headers['X-Timezone-Offset'] = new Date().getTimezoneOffset(); + + const formData = new FormData(); + formData.append('file', fileData); + + const options = { + method: 'put', + body: formData, + headers, + }; + + const response = await fetch(url, Client4.getOptions(options)); + + if (response.ok) { + return response.json(); + } + + const text = await response.text(); + + throw new ClientError(Client4.url, { + message: text || '', + status_code: response.status, + url, + }); + }; } const Client = new ClientClass(); diff --git a/webapp/src/components/admin_console/app_manifest_setting.tsx b/webapp/src/components/admin_console/app_manifest_setting.tsx new file mode 100644 index 000000000..225a9973f --- /dev/null +++ b/webapp/src/components/admin_console/app_manifest_setting.tsx @@ -0,0 +1,39 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +import React from 'react'; + +type Props = { + label: string; + disabled: boolean; +}; + +export default function MSTeamsAppManifestSetting(props: Props) { + return ( +
+

+ {'To embed Mattermost within Microsoft Teams, an application manifest can be downloaded and installed as a MS Teams app. '} + {'Clicking the Download button below will generate an application manifest that will embed this instance of Mattermost. '} +

+

+ {'Mattermost embedded in MS Teams can be used together with MSTeams Sync, or independently.'} +

+ + {props.label} + +
+ ); +} + +const styles = { + buttonBorder: { + marginTop: '8px', + }, +}; diff --git a/webapp/src/components/getConnectedUsersSetting.tsx b/webapp/src/components/admin_console/get_connected_users_setting.tsx similarity index 100% rename from webapp/src/components/getConnectedUsersSetting.tsx rename to webapp/src/components/admin_console/get_connected_users_setting.tsx diff --git a/webapp/src/components/admin_console/invite_whitelist_setting.tsx b/webapp/src/components/admin_console/invite_whitelist_setting.tsx new file mode 100644 index 000000000..9161f82ef --- /dev/null +++ b/webapp/src/components/admin_console/invite_whitelist_setting.tsx @@ -0,0 +1,126 @@ +import React, {ChangeEvent, useRef, useState} from 'react'; + +import Client from 'client'; + +type Props = { + label: string; + disabled: boolean; +}; + +const InviteWhitelistSetting = ({label, disabled}: Props) => { + const fileInputRef = useRef(null); + const [pendingFile, setPendingFile] = useState(); + const [statusMsg, setStatusMsg] = useState(''); + + const onChange = (e: ChangeEvent) => { + const file = e.target.files?.[0]; + + if (file) { + setPendingFile(file); + setStatusMsg(''); + } + + if (fileInputRef.current) { + fileInputRef.current.value = ''; + } + }; + + const doUpload = async (e: React.MouseEvent) => { + e.preventDefault(); + + if (!pendingFile) { + return; + } + + setStatusMsg('Uploading'); + + try { + const {count, failed, failedLines} = await Client.uploadWhitelist(pendingFile); + + let msg = `Upload successful - whitelist replaced, size: ${count}`; + + if (failed || failedLines) { + if (failed?.length) { + msg += `\n Could not find emails: ${failed.join(', ')}`; + } + if (failedLines?.length) { + msg += `\n Could not parse lines: ${failedLines.join(', ')}`; + } + msg += '\n Please verify data and try again'; + } + + setStatusMsg(msg); + } catch (err: any) { + if (err.message) { + setStatusMsg(err.message); + } else { + setStatusMsg('error while uploading'); + } + } + + // eslint-disable-next-line no-undefined + setPendingFile(undefined); + }; + + return ( +
+ +
+
+ + +
+ +
+ {statusMsg || pendingFile?.name} +
+

+ {'Upload a CSV file containing mattermost user-emails that may receive invites to connect to MS Teams. NOTE: This will replace the entire whitelist, not add to it.'} +

+ + +
+
+ ); +}; + +const styles = { + divMargin: { + marginTop: '20px', + }, + buttonMargin: { + marginTop: '8px', + }, +}; + +export default InviteWhitelistSetting; diff --git a/webapp/src/components/appManifestSetting.tsx b/webapp/src/components/appManifestSetting.tsx deleted file mode 100644 index bdce525bd..000000000 --- a/webapp/src/components/appManifestSetting.tsx +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -import React from 'react'; - -type Props = { - label: string; - disabled: boolean; -}; - -export default class MSTeamsAppManifestSetting extends React.PureComponent { - handleClick = () => { - window.location.href = '/plugins/com.mattermost.msteams-sync/iframe-manifest'; - }; - - render() { - return ( -
-

- {'To embed Mattermost within Microsoft Teams, an application manifest can be downloaded and installed as a MS Teams app. '} - {'Clicking the Download button below will generate an application manifest that will embed this instance of Mattermost. '} -

-

- {'Mattermost embedded in MS Teams can be used together with MSTeams Sync, or independently.'} -

- -
- ); - } -} - -const styles = { - buttonBorder: { - marginTop: '8px', - }, -}; \ No newline at end of file diff --git a/webapp/src/index.tsx b/webapp/src/index.tsx index b7a1cc1d7..da64968bd 100644 --- a/webapp/src/index.tsx +++ b/webapp/src/index.tsx @@ -2,10 +2,12 @@ import {Store, Action} from 'redux'; import {GlobalState} from 'mattermost-redux/types/store'; -import manifest from './manifest'; +import ListConnectedUsers from 'components/admin_console/get_connected_users_setting'; +import InviteWhitelistSetting from 'components/admin_console/invite_whitelist_setting'; +import MSTeamsAppManifestSetting from 'components/admin_console/app_manifest_setting'; + import Client from './client'; -import ListConnectedUsers from './components/getConnectedUsersSetting'; -import MSTeamsAppManifestSetting from './components/appManifestSetting'; +import manifest from './manifest'; // eslint-disable-next-line import/no-unresolved import {PluginRegistry} from './types/mattermost-webapp'; @@ -61,6 +63,7 @@ export default class Plugin { registry.registerAdminConsoleCustomSetting('appManifestDownload', MSTeamsAppManifestSetting); registry.registerAdminConsoleCustomSetting('ConnectedUsersReportDownload', ListConnectedUsers); + registry.registerAdminConsoleCustomSetting('inviteWhitelistUpload', InviteWhitelistSetting); this.userActivityWatch(); // let settingsEnabled = (state as any)[`plugins-${manifest.id}`]?.connectedStateSlice?.connected || false; //TODO use connected selector from https://github.com/mattermost/mattermost-plugin-msteams/pull/438