From e243211140a0246b40ac9859eacb2206c6c50d6c Mon Sep 17 00:00:00 2001 From: "Jens L." Date: Sat, 23 Nov 2024 04:38:25 +0100 Subject: [PATCH] extconfig: make log_iml work correctly (#1310) --- Makefile | 4 +- pkg/extconfig/ip.go | 6 +- pkg/extconfig/log.go | 8 ++- pkg/extconfig/log_iml/log_iml.go | 49 +++++---------- pkg/roles/api/api_cluster_node_log.go | 16 +++-- schema.yml | 11 +++- web/package-lock.json | 25 -------- web/package.json | 1 - web/src/pages/cluster/ClusterNodeLogsPage.ts | 63 +++++--------------- 9 files changed, 61 insertions(+), 122 deletions(-) diff --git a/Makefile b/Makefile index e1d28b5f7..dcda64dfa 100644 --- a/Makefile +++ b/Makefile @@ -128,7 +128,7 @@ gen-client-ts: cd ${PWD}/gen-ts-api && npm i \cp -rf ${PWD}/gen-ts-api/* ${PWD}/web/node_modules/gravity-api -gen-client-ts-update: gen-client-ts +gen-client-ts-publish: gen-client-ts cd ${PWD}/gen-ts-api npm publish cd ${PWD}/web @@ -136,7 +136,7 @@ gen-client-ts-update: gen-client-ts npm version ${VERSION} || true git add package*.json -gen: gen-build gen-clean gen-client-go gen-client-ts-update gen-tag +gen: gen-build gen-clean gen-client-go gen-client-ts-publish gen-tag lint: web-lint golangci-lint run -v --timeout 5000s diff --git a/pkg/extconfig/ip.go b/pkg/extconfig/ip.go index a892e92a6..17b5eea5e 100644 --- a/pkg/extconfig/ip.go +++ b/pkg/extconfig/ip.go @@ -54,7 +54,7 @@ func (e *ExtConfig) GetInterfaceForIP(forIp net.IP) (*net.Interface, error) { for _, i := range ifaces { addrs, err := e.checkInterface(i) if err != nil { - e.Logger().Debug("failed to get IPs from interface", zap.Error(err), zap.String("if", i.Name)) + e.intLog().Debug("failed to get IPs from interface", zap.Error(err), zap.String("if", i.Name)) continue } for _, addr := range addrs { @@ -74,13 +74,13 @@ func (e *ExtConfig) GetIP() (net.IP, error) { for _, i := range ifaces { addrs, err := e.checkInterface(i) if err != nil { - e.Logger().Debug("failed to get IPs from interface", zap.Error(err), zap.String("if", i.Name)) + e.intLog().Debug("failed to get IPs from interface", zap.Error(err), zap.String("if", i.Name)) continue } if len(addrs) < 1 { continue } - e.Logger().Debug("Detected IP of instance", zap.String("ip", addrs[0].String())) + e.intLog().Debug("Detected IP of instance", zap.String("ip", addrs[0].String())) return addrs[0], nil } return nil, errors.New("failed to find IP, set `INSTANCE_IP`") diff --git a/pkg/extconfig/log.go b/pkg/extconfig/log.go index 548d6dbd2..2927d8eb4 100644 --- a/pkg/extconfig/log.go +++ b/pkg/extconfig/log.go @@ -1,7 +1,7 @@ package extconfig import ( - _ "beryju.io/gravity/pkg/extconfig/log_iml" + "beryju.io/gravity/pkg/extconfig/log_iml" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) @@ -10,6 +10,10 @@ func (e *ExtConfig) Logger() *zap.Logger { return e.logger } +func (e *ExtConfig) intLog() *zap.Logger { + return e.Logger().Named("extconfig") +} + func (e *ExtConfig) BuildLogger() *zap.Logger { l, err := zapcore.ParseLevel(e.LogLevel) if err != nil { @@ -45,5 +49,5 @@ func (e *ExtConfig) BuildLoggerWithLevel(l zapcore.Level) *zap.Logger { return log.With( zap.String("instance", e.Instance.Identifier), zap.String("version", FullVersion()), - ) + ).WithOptions(log_iml.Get().Hook()) } diff --git a/pkg/extconfig/log_iml/log_iml.go b/pkg/extconfig/log_iml/log_iml.go index 0f6432a38..40ceea8a6 100644 --- a/pkg/extconfig/log_iml/log_iml.go +++ b/pkg/extconfig/log_iml/log_iml.go @@ -1,44 +1,31 @@ package log_iml import ( - "bytes" - "net/url" - "strings" "sync" "go.uber.org/zap" + "go.uber.org/zap/zapcore" ) type inMemoryLogger struct { - msgs []string - msgM sync.RWMutex - lineBuf *bytes.Buffer - max int + msgs []zapcore.Entry + msgM sync.RWMutex + max int } -func (iml *inMemoryLogger) Close() error { return nil } -func (iml *inMemoryLogger) Sync() error { return nil } -func (iml *inMemoryLogger) flush() { - go func() { +func (iml *inMemoryLogger) Hook() zap.Option { + return zap.Hooks(func(e zapcore.Entry) error { iml.msgM.Lock() - iml.msgs = append(iml.msgs, iml.lineBuf.String()) - iml.lineBuf.Reset() + defer iml.msgM.Unlock() + iml.msgs = append(iml.msgs, e) if len(iml.msgs) > iml.max { iml.msgs = iml.msgs[1:] } - iml.msgM.Unlock() - }() -} - -func (iml *inMemoryLogger) Write(log []byte) (int, error) { - n, err := iml.lineBuf.Write(log) - if strings.Contains(string(log), "\n") { - iml.flush() - } - return n, err + return nil + }) } -func (iml *inMemoryLogger) Messages() []string { +func (iml *inMemoryLogger) Messages() []zapcore.Entry { iml.msgM.RLock() defer iml.msgM.RUnlock() return iml.msgs @@ -48,20 +35,14 @@ var iml *inMemoryLogger func init() { iml = &inMemoryLogger{ - msgs: make([]string, 0), - max: 300, - lineBuf: new(bytes.Buffer), - } - err := zap.RegisterSink("gravity-in-memory", func(u *url.URL) (zap.Sink, error) { - return iml, nil - }) - if err != nil { - panic(err) + msgs: make([]zapcore.Entry, 0), + max: 300, } } type InMemoryLogger interface { - Messages() []string + Messages() []zapcore.Entry + Hook() zap.Option } func Get() InMemoryLogger { diff --git a/pkg/roles/api/api_cluster_node_log.go b/pkg/roles/api/api_cluster_node_log.go index 2494d9809..d0a6604f6 100644 --- a/pkg/roles/api/api_cluster_node_log.go +++ b/pkg/roles/api/api_cluster_node_log.go @@ -2,6 +2,7 @@ package api import ( "context" + "time" "beryju.io/gravity/pkg/extconfig" "beryju.io/gravity/pkg/extconfig/log_iml" @@ -10,21 +11,26 @@ import ( ) type APILogMessage struct { - Message string `json:"message"` - Node string `json:"node"` + Message string `json:"message"` + Time time.Time `json:"time"` + Level string `json:"level"` + Logger string `json:"logger"` + + Node string `json:"node"` } type APILogMessages struct { - IsJSON bool `json:"isJSON"` Messages []APILogMessage `json:"messages"` } func (r *Role) APIClusterNodeLogMessages() usecase.Interactor { u := usecase.NewInteractor(func(ctx context.Context, input struct{}, output *APILogMessages) error { - output.IsJSON = !extconfig.Get().Debug for _, lm := range log_iml.Get().Messages() { output.Messages = append(output.Messages, APILogMessage{ - Message: lm, + Message: lm.Message, + Level: lm.Level.CapitalString(), + Time: lm.Time, + Logger: lm.LoggerName, Node: extconfig.Get().Instance.Identifier, }) } diff --git a/schema.yml b/schema.yml index 476bca493..65ec73ea1 100644 --- a/schema.yml +++ b/schema.yml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: gravity - version: 0.14.0 + version: 0.14.1 paths: /api/v1/auth/config: get: @@ -1579,15 +1579,20 @@ components: type: object ApiAPILogMessage: properties: + level: + type: string + logger: + type: string message: type: string node: type: string + time: + format: date-time + type: string type: object ApiAPILogMessages: properties: - isJSON: - type: boolean messages: items: $ref: '#/components/schemas/ApiAPILogMessage' diff --git a/web/package-lock.json b/web/package-lock.json index e372e9c54..a78ac65fe 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -24,7 +24,6 @@ "@typescript-eslint/eslint-plugin": "^8.15.0", "@typescript-eslint/parser": "^8.15.0", "@webcomponents/webcomponentsjs": "^2.8.0", - "ansi-to-html": "^0.7.2", "chart.js": "^4.4.6", "chartjs-adapter-moment": "^1.0.1", "codemirror": "^6.0.1", @@ -2748,21 +2747,6 @@ "node": ">=4" } }, - "node_modules/ansi-to-html": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/ansi-to-html/-/ansi-to-html-0.7.2.tgz", - "integrity": "sha512-v6MqmEpNlxF+POuyhKkidusCHWWkaLcGRURzivcU3I9tv7k4JVhFcnukrM5Rlk2rUywdZuzYAZ+kbZqWCnfN3g==", - "license": "MIT", - "dependencies": { - "entities": "^2.2.0" - }, - "bin": { - "ansi-to-html": "bin/ansi-to-html" - }, - "engines": { - "node": ">=8.0.0" - } - }, "node_modules/apg-lite": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/apg-lite/-/apg-lite-1.0.4.tgz", @@ -3225,15 +3209,6 @@ "once": "^1.4.0" } }, - "node_modules/entities": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz", - "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==", - "license": "BSD-2-Clause", - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, "node_modules/es-module-lexer": { "version": "1.5.4", "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.5.4.tgz", diff --git a/web/package.json b/web/package.json index f6a653186..cd771d572 100644 --- a/web/package.json +++ b/web/package.json @@ -27,7 +27,6 @@ "@typescript-eslint/eslint-plugin": "^8.15.0", "@typescript-eslint/parser": "^8.15.0", "@webcomponents/webcomponentsjs": "^2.8.0", - "ansi-to-html": "^0.7.2", "chart.js": "^4.4.6", "chartjs-adapter-moment": "^1.0.1", "codemirror": "^6.0.1", diff --git a/web/src/pages/cluster/ClusterNodeLogsPage.ts b/web/src/pages/cluster/ClusterNodeLogsPage.ts index 2c5198e79..3da3a3f81 100644 --- a/web/src/pages/cluster/ClusterNodeLogsPage.ts +++ b/web/src/pages/cluster/ClusterNodeLogsPage.ts @@ -1,9 +1,7 @@ -import Convert from "ansi-to-html"; import { ApiAPILogMessage, RolesApiApi } from "gravity-api"; import { TemplateResult, html } from "lit"; -import { customElement, state } from "lit/decorators.js"; -import { unsafeHTML } from "lit/directives/unsafe-html.js"; +import { customElement } from "lit/decorators.js"; import { DEFAULT_CONFIG } from "../../api/Config"; import "../../elements/PageHeader"; @@ -15,10 +13,6 @@ import { PaginationWrapper } from "../../utils"; @customElement("gravity-cluster-node-logs") export class ClusterNodeLogsPage extends TablePage { - converter = new Convert(); - - @state() - isStructured = false; pageTitle(): string { return "Node logs"; @@ -31,7 +25,6 @@ export class ClusterNodeLogsPage extends TablePage { } async apiEndpoint(): Promise> { const logs = await new RolesApiApi(DEFAULT_CONFIG).apiGetLogMessages(); - this.isStructured = logs.isJSON || true; if (!logs.messages) { logs.messages = []; } @@ -39,46 +32,22 @@ export class ClusterNodeLogsPage extends TablePage { return PaginationWrapper(logs.messages); } columns(): TableColumn[] { - if (this.isStructured) { - return [ - new TableColumn("Level"), - new TableColumn("Timestamp"), - new TableColumn("Logger"), - new TableColumn("Message"), - new TableColumn("Fields"), - ]; - } - return [new TableColumn("Message")]; + return [ + new TableColumn("Level"), + new TableColumn("Timestamp"), + new TableColumn("Logger"), + new TableColumn("Message"), + new TableColumn("Fields"), + ]; } + row(item: ApiAPILogMessage): TemplateResult[] { - if (this.isStructured) { - try { - const payload: LogMessage = JSON.parse(item.message || ""); - const otherFields: { [key: string]: unknown } = { ...payload }; - delete otherFields.level; - delete otherFields.logger; - delete otherFields.msg; - delete otherFields.ts; - return [ - html`${payload.level}`, - html`${new Date(payload.ts * 1000).toLocaleTimeString()}`, - html`${payload.logger}`, - html`${payload.msg}`, - html`
${JSON.stringify(otherFields)}
`, - ]; - } catch (error) { - console.log(error); - this.isStructured = false; - } - } - return [html`${unsafeHTML(this.converter.toHtml(item.message || ""))}`]; + return [ + html`${item.level}`, + html`${item.time?.toLocaleString()}`, + html`
${item.logger}
`, + html`
${item.message}
`, + html``, + ]; } } - -interface LogMessage { - level: string; - ts: number; - logger: string; - msg: string; - [key: string]: unknown; -}