From aad15945b329230b38569f0e5b0543b3578f1261 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Tue, 2 Jan 2024 23:43:35 +1300 Subject: [PATCH 01/13] Fix: Log total deleted messages when deleting all messages from search --- internal/storage/database.go | 4 ++-- internal/storage/search.go | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/storage/database.go b/internal/storage/database.go index e4f88976a..b2d1b768c 100644 --- a/internal/storage/database.go +++ b/internal/storage/database.go @@ -686,11 +686,11 @@ func DeleteAllMessages() error { logger.Log().Debugf("[db] deleted %d messages in %s", total, elapsed) } - logMessagesDeleted(total) - dbLastAction = time.Now() dbDataDeleted = false + logMessagesDeleted(total) + websockets.Broadcast("prune", nil) BroadcastMailboxStats() diff --git a/internal/storage/search.go b/internal/storage/search.go index ecd873e79..8750d786f 100644 --- a/internal/storage/search.go +++ b/internal/storage/search.go @@ -193,6 +193,8 @@ func DeleteSearch(search string) error { dbLastAction = time.Now() dbDataDeleted = true + logMessagesDeleted(total) + BroadcastMailboxStats() } From cdab59b295142ef64dd4256788dcc6095b65a06e Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Wed, 3 Jan 2024 12:06:36 +1300 Subject: [PATCH 02/13] Feature: Add option to only allow SMTP recipients matching a regular expression (disable open-relay behaviour #219) --- cmd/root.go | 4 +++ config/config.go | 20 ++++++++++-- internal/stats/stats.go | 40 +++++++++++------------ server/smtpd/smtpd.go | 22 +++++++++++-- server/ui-src/components/AboutMailpit.vue | 14 +++++--- server/ui/api/v1/swagger.json | 16 ++++----- 6 files changed, 78 insertions(+), 38 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index e972a9597..03a38785b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -103,6 +103,7 @@ func init() { rootCmd.Flags().BoolVar(&config.SMTPAuthAllowInsecure, "smtp-auth-allow-insecure", config.SMTPAuthAllowInsecure, "Enable insecure PLAIN & LOGIN authentication") rootCmd.Flags().BoolVar(&config.SMTPStrictRFCHeaders, "smtp-strict-rfc-headers", config.SMTPStrictRFCHeaders, "Return SMTP error if message headers contain ") rootCmd.Flags().IntVar(&config.SMTPMaxRecipients, "smtp-max-recipients", config.SMTPMaxRecipients, "Maximum SMTP recipients allowed") + rootCmd.Flags().StringVar(&config.SMTPAllowedRecipients, "smtp-allowed-recipients", config.SMTPAllowedRecipients, "Only allow SMTP recipients matching a regular expression (default allow all)") rootCmd.Flags().StringVar(&config.SMTPRelayConfigFile, "smtp-relay-config", config.SMTPRelayConfigFile, "SMTP configuration file to allow releasing messages") rootCmd.Flags().BoolVar(&config.SMTPRelayAllIncoming, "smtp-relay-all", config.SMTPRelayAllIncoming, "Relay all incoming messages via external SMTP server (caution!)") @@ -170,6 +171,9 @@ func initConfigFromEnv() { if len(os.Getenv("MP_SMTP_MAX_RECIPIENTS")) > 0 { config.SMTPMaxRecipients, _ = strconv.Atoi(os.Getenv("MP_SMTP_MAX_RECIPIENTS")) } + if len(os.Getenv("MP_SMTP_ALLOWED_RECIPIENTS")) > 0 { + config.SMTPAllowedRecipients = os.Getenv("MP_SMTP_ALLOWED_RECIPIENTS") + } // Relay server config config.SMTPRelayConfigFile = os.Getenv("MP_SMTP_RELAY_CONFIG") diff --git a/config/config.go b/config/config.go index c8e701190..15aa2867e 100644 --- a/config/config.go +++ b/config/config.go @@ -93,6 +93,12 @@ var ( // @see https://github.com/axllent/mailpit/issues/87 & https://github.com/axllent/mailpit/issues/153 SMTPStrictRFCHeaders bool + // SMTPAllowedRecipients if set, will only accept recipients matching this regular expression + SMTPAllowedRecipients string + + // SMTPAllowedRecipientsRegexp is the compiled version of SMTPAllowedRecipients + SMTPAllowedRecipientsRegexp *regexp.Regexp + // ReleaseEnabled is whether message releases are enabled, requires a valid SMTPRelayConfigFile ReleaseEnabled = false @@ -262,6 +268,16 @@ func VerifyConfig() error { } } + if SMTPAllowedRecipients != "" { + restrictRegexp, err := regexp.Compile(SMTPAllowedRecipients) + if err != nil { + return fmt.Errorf("Failed to compile smtp-allowed-recipients regexp: %s", err.Error()) + } + + SMTPAllowedRecipientsRegexp = restrictRegexp + logger.Log().Infof("[smtp] only allowing recipients matching the following regexp: %s", SMTPAllowedRecipients) + } + if err := parseRelayConfig(SMTPRelayConfigFile); err != nil { return err } @@ -335,11 +351,11 @@ func parseRelayConfig(c string) error { if SMTPRelayConfig.RecipientAllowlist != "" { if err != nil { - return fmt.Errorf("failed to compile recipient allowlist regexp: %e", err) + return fmt.Errorf("Failed to compile relay recipient allowlist regexp: %s", err.Error()) } SMTPRelayConfig.RecipientAllowlistRegexp = allowlistRegexp - logger.Log().Infof("[smtp] recipient allowlist is active with the following regexp: %s", SMTPRelayConfig.RecipientAllowlist) + logger.Log().Infof("[smtp] relay recipient allowlist is active with the following regexp: %s", SMTPRelayConfig.RecipientAllowlist) } diff --git a/internal/stats/stats.go b/internal/stats/stats.go index e31ed0771..c10792661 100644 --- a/internal/stats/stats.go +++ b/internal/stats/stats.go @@ -21,9 +21,9 @@ var ( mu sync.RWMutex - smtpReceived int - smtpReceivedSize int - smtpErrors int + smtpAccepted int + smtpAcceptedSize int + smtpRejected int smtpIgnored int ) @@ -52,13 +52,13 @@ type AppInformation struct { Memory uint64 // Messages deleted MessagesDeleted int - // SMTP messages received via since run - SMTPReceived int - // Total size in bytes of received messages since run - SMTPReceivedSize int - // SMTP errors since run - SMTPErrors int - // SMTP messages ignored since run (duplicate IDs) + // SMTP accepted messages since run + SMTPAccepted int + // Total size in bytes of accepted messages since run + SMTPAcceptedSize int + // SMTP rejected messages since run + SMTPRejected int + // SMTP ignored messages since run (duplicate IDs) SMTPIgnored int } } @@ -75,9 +75,9 @@ func Load() AppInformation { info.RuntimeStats.Uptime = int(time.Since(startedAt).Seconds()) info.RuntimeStats.MessagesDeleted = storage.StatsDeleted - info.RuntimeStats.SMTPReceived = smtpReceived - info.RuntimeStats.SMTPReceivedSize = smtpReceivedSize - info.RuntimeStats.SMTPErrors = smtpErrors + info.RuntimeStats.SMTPAccepted = smtpAccepted + info.RuntimeStats.SMTPAcceptedSize = smtpAcceptedSize + info.RuntimeStats.SMTPRejected = smtpRejected info.RuntimeStats.SMTPIgnored = smtpIgnored if latestVersionCache != "" { @@ -116,18 +116,18 @@ func Track() { startedAt = time.Now() } -// LogSMTPReceived logs a successfully SMTP transaction -func LogSMTPReceived(size int) { +// LogSMTPAccepted logs a successful SMTP transaction +func LogSMTPAccepted(size int) { mu.Lock() - smtpReceived = smtpReceived + 1 - smtpReceivedSize = smtpReceivedSize + size + smtpAccepted = smtpAccepted + 1 + smtpAcceptedSize = smtpAcceptedSize + size mu.Unlock() } -// LogSMTPError logs a failed SMTP transaction -func LogSMTPError() { +// LogSMTPRejected logs a rejected SMTP transaction +func LogSMTPRejected() { mu.Lock() - smtpErrors = smtpErrors + 1 + smtpRejected = smtpRejected + 1 mu.Unlock() } diff --git a/server/smtpd/smtpd.go b/server/smtpd/smtpd.go index ba09058d4..70d886753 100644 --- a/server/smtpd/smtpd.go +++ b/server/smtpd/smtpd.go @@ -28,7 +28,7 @@ func mailHandler(origin net.Addr, from string, to []string, data []byte) error { msg, err := mail.ReadMessage(bytes.NewReader(data)) if err != nil { logger.Log().Errorf("[smtpd] error parsing message: %s", err.Error()) - stats.LogSMTPError() + stats.LogSMTPRejected() return err } @@ -121,11 +121,10 @@ func mailHandler(origin net.Addr, from string, to []string, data []byte) error { _, err = storage.Store(&data) if err != nil { logger.Log().Errorf("[db] error storing message: %s", err.Error()) - stats.LogSMTPError() return err } - stats.LogSMTPReceived(len(data)) + stats.LogSMTPAccepted(len(data)) data = nil // avoid memory leaks @@ -153,6 +152,22 @@ func authHandlerAny(remoteAddr net.Addr, mechanism string, username []byte, _ [] return true, nil } +// HandlerRcpt used to optionally restrict recipients based on `--smtp-allowed-recipients` +func handlerRcpt(remoteAddr net.Addr, from string, to string) bool { + if config.SMTPAllowedRecipientsRegexp == nil { + return true + } + + result := config.SMTPAllowedRecipientsRegexp.MatchString(to) + + if !result { + logger.Log().Warnf("[smtpd] rejected message to %s from %s (%s)", to, from, cleanIP(remoteAddr)) + stats.LogSMTPRejected() + } + + return result +} + // Listen starts the SMTPD server func Listen() error { if config.SMTPAuthAllowInsecure { @@ -178,6 +193,7 @@ func listenAndServe(addr string, handler smtpd.Handler, authHandler smtpd.AuthHa srv := &smtpd.Server{ Addr: addr, Handler: handler, + HandlerRcpt: handlerRcpt, Appname: "Mailpit", Hostname: "", AuthHandler: nil, diff --git a/server/ui-src/components/AboutMailpit.vue b/server/ui-src/components/AboutMailpit.vue index 06991ed2c..b2af8888f 100644 --- a/server/ui-src/components/AboutMailpit.vue +++ b/server/ui-src/components/AboutMailpit.vue @@ -240,19 +240,23 @@ export default { - SMTP messages received + SMTP messages accepted - {{ formatNumber(mailbox.appInfo.RuntimeStats.SMTPReceived) }} - ({{ getFileSize(mailbox.appInfo.RuntimeStats.SMTPReceivedSize) }}) + {{ formatNumber(mailbox.appInfo.RuntimeStats.SMTPAccepted) }} + + ({{ + getFileSize(mailbox.appInfo.RuntimeStats.SMTPAcceptedSize) + }}) + - SMTP errors + SMTP messages rejected - {{ formatNumber(mailbox.appInfo.RuntimeStats.SMTPErrors) }} + {{ formatNumber(mailbox.appInfo.RuntimeStats.SMTPRejected) }} diff --git a/server/ui/api/v1/swagger.json b/server/ui/api/v1/swagger.json index afa07735c..8aa7adff4 100644 --- a/server/ui/api/v1/swagger.json +++ b/server/ui/api/v1/swagger.json @@ -762,23 +762,23 @@ "type": "integer", "format": "int64" }, - "SMTPErrors": { - "description": "SMTP errors since run", + "SMTPAccepted": { + "description": "SMTP accepted messages since run", "type": "integer", "format": "int64" }, - "SMTPIgnored": { - "description": "SMTP messages ignored since run (duplicate IDs)", + "SMTPAcceptedSize": { + "description": "Total size in bytes of accepted messages since run", "type": "integer", "format": "int64" }, - "SMTPReceived": { - "description": "SMTP messages received via since run", + "SMTPIgnored": { + "description": "SMTP ignored messages since run (duplicate IDs)", "type": "integer", "format": "int64" }, - "SMTPReceivedSize": { - "description": "Total size in bytes of received messages since run", + "SMTPRejected": { + "description": "SMTP rejected messages since run", "type": "integer", "format": "int64" }, From f4c703b6861a493adfd94c954cd739452d672d4e Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Wed, 3 Jan 2024 12:21:00 +1300 Subject: [PATCH 03/13] Chore: Standardize error logging & formatting --- internal/htmlcheck/css.go | 6 ++---- internal/linkcheck/status.go | 2 +- internal/storage/database.go | 12 ++++++------ internal/storage/migrationTasks.go | 2 +- internal/storage/reindex.go | 10 +++++----- internal/storage/search.go | 6 +++--- internal/storage/tags.go | 6 +++--- server/apiv1/thumbnails.go | 6 +++--- server/handlers/proxy.go | 2 +- server/websockets/client.go | 2 +- 10 files changed, 26 insertions(+), 28 deletions(-) diff --git a/internal/htmlcheck/css.go b/internal/htmlcheck/css.go index 53a404e48..84a570850 100644 --- a/internal/htmlcheck/css.go +++ b/internal/htmlcheck/css.go @@ -25,14 +25,12 @@ func runCSSTests(html string) ([]Warning, int, error) { inlined, err := inlineRemoteCSS(html) if err != nil { - // logger.Log().Warn(err) inlined = html } // merge all CSS inline merged, err := mergeInlineCSS(inlined) if err != nil { - // logger.Log().Warn(err) merged = inlined } @@ -157,7 +155,7 @@ func inlineRemoteCSS(h string) (string, error) { resp, err := downloadToBytes(a.Val) if err != nil { - logger.Log().Warningf("html check failed to download %s", a.Val) + logger.Log().Warnf("[html-check] failed to download %s", a.Val) continue } @@ -179,7 +177,7 @@ func inlineRemoteCSS(h string) (string, error) { newDoc, err := doc.Html() if err != nil { - logger.Log().Warning(err) + logger.Log().Warnf("[html-check] failed to download %s", err.Error()) return h, err } diff --git a/internal/linkcheck/status.go b/internal/linkcheck/status.go index cc3f00451..102f61931 100644 --- a/internal/linkcheck/status.go +++ b/internal/linkcheck/status.go @@ -79,7 +79,7 @@ func doHead(link string, followRedirects bool) (int, error) { req, err := http.NewRequest("HEAD", link, nil) if err != nil { - logger.Log().Error(err) + logger.Log().Errorf("[link-check] %s", err.Error()) return 0, err } diff --git a/internal/storage/database.go b/internal/storage/database.go index b2d1b768c..a9fc53977 100644 --- a/internal/storage/database.go +++ b/internal/storage/database.go @@ -110,7 +110,7 @@ func InitDB() error { func Close() { if db != nil { if err := db.Close(); err != nil { - logger.Log().Warning("[db] error closing database, ignoring") + logger.Log().Warn("[db] error closing database, ignoring") } } @@ -128,7 +128,7 @@ func Store(body *[]byte) (string, error) { // Parse message body with enmime env, err := enmime.ReadEnvelope(bytes.NewReader(*body)) if err != nil { - logger.Log().Warningf("[db] %s", err.Error()) + logger.Log().Warnf("[message] %s", err.Error()) return "", nil } @@ -271,12 +271,12 @@ func List(start, limit int) ([]MessageSummary, error) { em := MessageSummary{} if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &snippet); err != nil { - logger.Log().Error(err) + logger.Log().Errorf("[db] %s", err.Error()) return } if err := json.Unmarshal([]byte(metadata), &em); err != nil { - logger.Log().Error(err) + logger.Log().Errorf("[json] %s", err.Error()) return } @@ -349,7 +349,7 @@ func GetMessage(id string) (*Message, error) { var created int64 if err := row.Scan(&created); err != nil { - logger.Log().Error(err) + logger.Log().Errorf("[db] %s", err.Error()) return } @@ -357,7 +357,7 @@ func GetMessage(id string) (*Message, error) { date = time.UnixMilli(created) }); err != nil { - logger.Log().Error(err) + logger.Log().Errorf("[db] %s", err.Error()) } } diff --git a/internal/storage/migrationTasks.go b/internal/storage/migrationTasks.go index 94ebff30b..756a76220 100644 --- a/internal/storage/migrationTasks.go +++ b/internal/storage/migrationTasks.go @@ -36,7 +36,7 @@ func migrateTagsToManyMany() { tags := []string{} if err := json.Unmarshal([]byte(jsonTags), &tags); err != nil { - logger.Log().Error(err) + logger.Log().Errorf("[json] %s", err.Error()) return } diff --git a/internal/storage/reindex.go b/internal/storage/reindex.go index bc5c3c6d5..282f8572c 100644 --- a/internal/storage/reindex.go +++ b/internal/storage/reindex.go @@ -29,7 +29,7 @@ func ReindexAll() { }) if err != nil { - logger.Log().Error(err) + logger.Log().Errorf("[db] %s", err.Error()) os.Exit(1) } @@ -59,7 +59,7 @@ func ReindexAll() { env, err := enmime.ReadEnvelope(r) if err != nil { - logger.Log().Error(err) + logger.Log().Errorf("[message] %s", err.Error()) continue } @@ -77,7 +77,7 @@ func ReindexAll() { ctx := context.Background() tx, err := db.BeginTx(ctx, nil) if err != nil { - logger.Log().Error(err) + logger.Log().Errorf("[db] %s", err.Error()) continue } @@ -88,13 +88,13 @@ func ReindexAll() { for _, u := range updates { _, err = tx.Exec("UPDATE mailbox SET SearchText = ?, Snippet = ? WHERE ID = ?", u.SearchText, u.Snippet, u.ID) if err != nil { - logger.Log().Error(err) + logger.Log().Errorf("[db] %s", err.Error()) continue } } if err := tx.Commit(); err != nil { - logger.Log().Error(err) + logger.Log().Errorf("[db] %s", err.Error()) continue } diff --git a/internal/storage/search.go b/internal/storage/search.go index 8750d786f..061727537 100644 --- a/internal/storage/search.go +++ b/internal/storage/search.go @@ -43,12 +43,12 @@ func Search(search string, start, limit int) ([]MessageSummary, int, error) { em := MessageSummary{} if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &snippet, &ignore, &ignore, &ignore, &ignore); err != nil { - logger.Log().Error(err) + logger.Log().Errorf("[db] %s", err.Error()) return } if err := json.Unmarshal([]byte(metadata), &em); err != nil { - logger.Log().Error(err) + logger.Log().Errorf("[db] %s", err.Error()) return } @@ -114,7 +114,7 @@ func DeleteSearch(search string) error { var ignore string if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &snippet, &ignore, &ignore, &ignore, &ignore); err != nil { - logger.Log().Error(err) + logger.Log().Errorf("[db] %s", err.Error()) return } diff --git a/internal/storage/tags.go b/internal/storage/tags.go index 8d3ef6b83..7bcffe8d2 100644 --- a/internal/storage/tags.go +++ b/internal/storage/tags.go @@ -149,7 +149,7 @@ func GetAllTags() []string { QueryAndClose(nil, db, func(row *sql.Rows) { tags = append(tags, name) }); err != nil { - logger.Log().Error(err) + logger.Log().Errorf("[db] %s", err.Error()) } return tags @@ -172,7 +172,7 @@ func GetAllTagsCount() map[string]int64 { tags[name] = total // tags = append(tags, name) }); err != nil { - logger.Log().Error(err) + logger.Log().Errorf("[db] %s", err.Error()) } return tags @@ -193,7 +193,7 @@ func pruneUnusedTags() error { var c int if err := row.Scan(&id, &n, &c); err != nil { - logger.Log().Error("[tags]", err) + logger.Log().Errorf("[tags] %s", err.Error()) return } diff --git a/server/apiv1/thumbnails.go b/server/apiv1/thumbnails.go index 8eb872868..8a1291ead 100644 --- a/server/apiv1/thumbnails.go +++ b/server/apiv1/thumbnails.go @@ -77,7 +77,7 @@ func Thumbnail(w http.ResponseWriter, r *http.Request) { img, err := imaging.Decode(buf) if err != nil { // it's not an image, return default - logger.Log().Warning(err) + logger.Log().Warnf("[image] %s", err.Error()) blankImage(a, w) return } @@ -99,7 +99,7 @@ func Thumbnail(w http.ResponseWriter, r *http.Request) { dst = imaging.OverlayCenter(dst, dstImageFill, 1.0) if err := jpeg.Encode(foo, dst, &jpeg.Options{Quality: 70}); err != nil { - logger.Log().Warning(err) + logger.Log().Warnf("[image] %s", err.Error()) blankImage(a, w) return } @@ -120,7 +120,7 @@ func blankImage(a *enmime.Part, w http.ResponseWriter) { dstImageFill := imaging.Fill(img, thumbWidth, thumbHeight, imaging.Center, imaging.Lanczos) if err := jpeg.Encode(foo, dstImageFill, &jpeg.Options{Quality: 70}); err != nil { - logger.Log().Warning(err) + logger.Log().Warnf("[image] %s", err.Error()) } fileName := a.FileName diff --git a/server/handlers/proxy.go b/server/handlers/proxy.go index 010c4c434..0dd360b38 100644 --- a/server/handlers/proxy.go +++ b/server/handlers/proxy.go @@ -95,7 +95,7 @@ func ProxyHandler(w http.ResponseWriter, r *http.Request) { address, err := absoluteURL(parts[3], uri) if err != nil { - logger.Log().Error(err) + logger.Log().Errorf("[proxy] %s", err.Error()) return []byte(parts[3]) } diff --git a/server/websockets/client.go b/server/websockets/client.go index 547ea0e2d..c4523c0e2 100644 --- a/server/websockets/client.go +++ b/server/websockets/client.go @@ -132,7 +132,7 @@ func ServeWs(hub *Hub, w http.ResponseWriter, r *http.Request) { conn, err := upgrader.Upgrade(w, r, nil) if err != nil { - logger.Log().Error(err) + logger.Log().Errorf("[websocket] %s", err.Error()) return } From a25c7e359ab89feda032c87746e1b7f367bdaea8 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Wed, 3 Jan 2024 12:24:33 +1300 Subject: [PATCH 04/13] Libs: Update node modules --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 315fb1d3d..6ccc47d64 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2218,9 +2218,9 @@ "optional": true }, "node_modules/sass": { - "version": "1.69.6", - "resolved": "https://registry.npmjs.org/sass/-/sass-1.69.6.tgz", - "integrity": "sha512-qbRr3k9JGHWXCvZU77SD2OTwUlC+gNT+61JOLcmLm+XqH4h/5D+p4IIsxvpkB89S9AwJOyb5+rWNpIucaFxSFQ==", + "version": "1.69.7", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.69.7.tgz", + "integrity": "sha512-rzj2soDeZ8wtE2egyLXgOOHQvaC2iosZrkF6v3EUG+tBwEvhqUCzm0VP3k9gHF9LXbSrRhT5SksoI56Iw8NPnQ==", "dev": true, "dependencies": { "chokidar": ">=3.0.0 <4.0.0", From 12cfb097743a9181aa9df1e48a8a89c6ec72fe09 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Wed, 3 Jan 2024 12:30:15 +1300 Subject: [PATCH 05/13] Update swagger docs --- internal/stats/stats.go | 10 +++++----- server/ui/api/v1/swagger.json | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/internal/stats/stats.go b/internal/stats/stats.go index c10792661..cca3f7e19 100644 --- a/internal/stats/stats.go +++ b/internal/stats/stats.go @@ -50,15 +50,15 @@ type AppInformation struct { Uptime int // Current memory usage in bytes Memory uint64 - // Messages deleted + // Database runtime messages deleted MessagesDeleted int - // SMTP accepted messages since run + // Accepted runtime SMTP messages SMTPAccepted int - // Total size in bytes of accepted messages since run + // Total runtime accepted messages size in bytes SMTPAcceptedSize int - // SMTP rejected messages since run + // Rejected runtime SMTP messages SMTPRejected int - // SMTP ignored messages since run (duplicate IDs) + // Ignored runtime SMTP messages (when using --ignore-duplicate-ids) SMTPIgnored int } } diff --git a/server/ui/api/v1/swagger.json b/server/ui/api/v1/swagger.json index 8aa7adff4..f1c61a30a 100644 --- a/server/ui/api/v1/swagger.json +++ b/server/ui/api/v1/swagger.json @@ -758,27 +758,27 @@ "format": "uint64" }, "MessagesDeleted": { - "description": "Messages deleted", + "description": "Database runtime messages deleted", "type": "integer", "format": "int64" }, "SMTPAccepted": { - "description": "SMTP accepted messages since run", + "description": "Accepted runtime SMTP messages", "type": "integer", "format": "int64" }, "SMTPAcceptedSize": { - "description": "Total size in bytes of accepted messages since run", + "description": "Total runtime accepted messages size in bytes", "type": "integer", "format": "int64" }, "SMTPIgnored": { - "description": "SMTP ignored messages since run (duplicate IDs)", + "description": "Ignored runtime SMTP messages (when using --ignore-duplicate-ids)", "type": "integer", "format": "int64" }, "SMTPRejected": { - "description": "SMTP rejected messages since run", + "description": "Rejected runtime SMTP messages", "type": "integer", "format": "int64" }, From dd57596fd1ff789f439d1cf778e76ce72e8f13f3 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Wed, 3 Jan 2024 12:54:12 +1300 Subject: [PATCH 06/13] UI: Automatically refresh connected browsers if Mailpit is upgraded (version change) --- internal/storage/notifications.go | 11 +++++++---- server/server.go | 2 +- server/ui-src/components/Notifications.vue | 11 +++++++++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/internal/storage/notifications.go b/internal/storage/notifications.go index 8b8ccbb5e..8c59c2f0e 100644 --- a/internal/storage/notifications.go +++ b/internal/storage/notifications.go @@ -3,6 +3,7 @@ package storage import ( "time" + "github.com/axllent/mailpit/config" "github.com/axllent/mailpit/server/websockets" ) @@ -23,11 +24,13 @@ func BroadcastMailboxStats() { time.Sleep(250 * time.Millisecond) bcStatsDelay = false b := struct { - Total int - Unread int + Total int + Unread int + Version string }{ - Total: CountTotal(), - Unread: CountUnread(), + Total: CountTotal(), + Unread: CountUnread(), + Version: config.Version, } websockets.Broadcast("stats", b) diff --git a/server/server.go b/server/server.go index ac954f9be..a91c68658 100644 --- a/server/server.go +++ b/server/server.go @@ -290,7 +290,7 @@ func index(w http.ResponseWriter, _ *http.Request) { -
+
diff --git a/server/ui-src/components/Notifications.vue b/server/ui-src/components/Notifications.vue index d10aa17bd..96ba6449d 100644 --- a/server/ui-src/components/Notifications.vue +++ b/server/ui-src/components/Notifications.vue @@ -15,10 +15,16 @@ export default { reconnectRefresh: false, socketURI: false, pauseNotifications: false, // prevent spamming + version: false } }, mounted() { + let d = document.getElementById('app') + if (d) { + this.version = d.dataset.version + } + let proto = location.protocol == 'https:' ? 'wss' : 'ws' this.socketURI = proto + "://" + document.location.host + this.resolve(`/api/events`) @@ -79,6 +85,11 @@ export default { // refresh mailbox stats mailbox.total = response.Data.Total mailbox.unread = response.Data.Unread + + // detect version updated, refresh is needed + if (self.version != response.Data.Version) { + location.reload(); + } } } From 381813fe6311d83fdc073711a6d02644666832b9 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Wed, 3 Jan 2024 13:09:06 +1300 Subject: [PATCH 07/13] Fix: Prevent rare error from websocket connection (unexpected non-whitespace character) --- server/ui-src/components/Notifications.vue | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/server/ui-src/components/Notifications.vue b/server/ui-src/components/Notifications.vue index 96ba6449d..7f4871b5a 100644 --- a/server/ui-src/components/Notifications.vue +++ b/server/ui-src/components/Notifications.vue @@ -41,10 +41,13 @@ export default { let ws = new WebSocket(this.socketURI) let self = this ws.onmessage = function (e) { - let response = JSON.parse(e.data) - if (!response) { + let response + try { + response = JSON.parse(e.data) + } catch (e) { return } + // new messages if (response.Type == "new" && response.Data) { if (!mailbox.searching) { @@ -88,7 +91,7 @@ export default { // detect version updated, refresh is needed if (self.version != response.Data.Version) { - location.reload(); + location.reload() } } } From 119e6a55d2a545d1c7cf2e67a22938c439a063b4 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Wed, 3 Jan 2024 13:13:43 +1300 Subject: [PATCH 08/13] Fix: Log total deleted messages when auto-pruning messages (--max) --- internal/storage/utils.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/storage/utils.go b/internal/storage/utils.go index 25d7aec92..359827bab 100644 --- a/internal/storage/utils.go +++ b/internal/storage/utils.go @@ -171,6 +171,8 @@ func dbCron() { elapsed := time.Since(start) logger.Log().Debugf("[db] auto-pruned %d messages in %s", len(ids), elapsed) + logMessagesDeleted(len(ids)) + websockets.Broadcast("prune", nil) } } From 669c1a747f4d3772208b0a9add5c07c23c5f13a3 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Wed, 3 Jan 2024 14:39:28 +1300 Subject: [PATCH 09/13] Chore: Significantly increase database performance using WAL (Write-Ahead-Log) --- internal/storage/database.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/storage/database.go b/internal/storage/database.go index a9fc53977..6ae12763d 100644 --- a/internal/storage/database.go +++ b/internal/storage/database.go @@ -76,6 +76,12 @@ func InitDB() error { // @see https://github.com/mattn/go-sqlite3#faq db.SetMaxOpenConns(1) + // SQLite performance tuning (https://phiresky.github.io/blog/2020/sqlite-performance-tuning/) + _, err = db.Exec("PRAGMA journal_mode = WAL; PRAGMA synchronous = normal;") + if err != nil { + return err + } + // create tables if necessary & apply migrations if err := dbApplyMigrations(); err != nil { return err From 4d2b6d6b4ae6f2329f76be514af1265aa4d84e47 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Wed, 3 Jan 2024 14:41:52 +1300 Subject: [PATCH 10/13] Tests: Run tests on Linux, Windows & Mac --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7c3b41eba..a6830226e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,7 +9,7 @@ jobs: strategy: matrix: go-version: [1.21.x] - os: [ubuntu-latest] + os: [ubuntu-latest, windows-latest, macos-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/setup-go@v5 From 8c86cc624e5dafec1b3898deedfdb49dc91057a1 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Wed, 3 Jan 2024 14:52:53 +1300 Subject: [PATCH 11/13] Limit testing for web UI build & swagger-editor-validate to Ubuntu --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a6830226e..92e507fc4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -30,6 +30,7 @@ jobs: # build the assets - name: Build web UI + if: startsWith(matrix.os, 'ubuntu') == true uses: actions/setup-node@v4 with: node-version: 18 @@ -39,6 +40,7 @@ jobs: # validate the swagger file - name: Validate OpenAPI definition + if: startsWith(matrix.os, 'ubuntu') == true uses: char0n/swagger-editor-validate@v1 with: definition-file: server/ui/api/v1/swagger.json From 0c3519cb0dc9c46120a87b5ace2669d46a549566 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Wed, 3 Jan 2024 14:58:35 +1300 Subject: [PATCH 12/13] Limit testing for web UI build & swagger-editor-validate to Ubuntu --- .github/workflows/tests.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 92e507fc4..360288863 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,6 +15,7 @@ jobs: - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} + cache: false - uses: actions/checkout@v4 - name: Run Go tests uses: actions/cache@v3 @@ -35,8 +36,10 @@ jobs: with: node-version: 18 cache: 'npm' - - run: npm install - - run: npm run package + - if: startsWith(matrix.os, 'ubuntu') == true + run: npm install + - if: startsWith(matrix.os, 'ubuntu') == true + run: npm run package # validate the swagger file - name: Validate OpenAPI definition From b8385dc18b70c77dfed8f02459c62dd10424a73d Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Wed, 3 Jan 2024 15:03:15 +1300 Subject: [PATCH 13/13] Release v1.12.1 --- CHANGELOG.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 300750190..990186962 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,30 @@ Notable changes to Mailpit will be documented in this file. +## [v1.12.1] + +### Chore +- Significantly increase database performance using WAL (Write-Ahead-Log) +- Standardize error logging & formatting + +### Feature +- Add option to only allow SMTP recipients matching a regular expression (disable open-relay behaviour [#219](https://github.com/axllent/mailpit/issues/219)) + +### Fix +- Log total deleted messages when auto-pruning messages (--max) +- Prevent rare error from websocket connection (unexpected non-whitespace character) +- Log total deleted messages when deleting all messages from search + +### Libs +- Update node modules + +### Tests +- Run tests on Linux, Windows & Mac + +### UI +- Automatically refresh connected browsers if Mailpit is upgraded (version change) + + ## [v1.12.0] ### Chore