diff --git a/cmd/metal-api/internal/grpc/grpc-server.go b/cmd/metal-api/internal/grpc/grpc-server.go index bb865ed3..d0d8a991 100644 --- a/cmd/metal-api/internal/grpc/grpc-server.go +++ b/cmd/metal-api/internal/grpc/grpc-server.go @@ -52,7 +52,7 @@ type ServerConfig struct { ResponseInterval time.Duration CheckInterval time.Duration BMCSuperUserPasswordFile string - Auditing auditing.Auditing + Auditing []auditing.Auditing IPMISuperUser metal.MachineIPMISuperUser } @@ -121,7 +121,7 @@ func Run(cfg *ServerConfig) error { logging.UnaryServerInterceptor(interceptorLogger(log)), recovery.UnaryServerInterceptor(recovery.WithRecoveryHandler(grpcPanicRecoveryHandler)), } - if cfg.Auditing != nil { + if len(cfg.Auditing) > 0 { shouldAudit := func(fullMethod string) bool { switch fullMethod { case "/api.v1.BootService/Register": @@ -130,16 +130,19 @@ func Run(cfg *ServerConfig) error { return false } } - auditStreamInterceptor, err := auditing.StreamServerInterceptor(cfg.Auditing, log.WithGroup("auditing-grpc"), shouldAudit) - if err != nil { - return err - } - auditUnaryInterceptor, err := auditing.UnaryServerInterceptor(cfg.Auditing, log.WithGroup("auditing-grpc"), shouldAudit) - if err != nil { - return err + + for _, backend := range cfg.Auditing { + auditStreamInterceptor, err := auditing.StreamServerInterceptor(backend, log.WithGroup("auditing-grpc"), shouldAudit) + if err != nil { + return err + } + auditUnaryInterceptor, err := auditing.UnaryServerInterceptor(backend, log.WithGroup("auditing-grpc"), shouldAudit) + if err != nil { + return err + } + streamInterceptors = append(streamInterceptors, auditStreamInterceptor) + unaryInterceptors = append(unaryInterceptors, auditUnaryInterceptor) } - streamInterceptors = append(streamInterceptors, auditStreamInterceptor) - unaryInterceptors = append(unaryInterceptors, auditUnaryInterceptor) } unaryInterceptors = append(unaryInterceptors, metrics.GrpcMetrics, recovery.UnaryServerInterceptor(recoveryOpt)) diff --git a/cmd/metal-api/internal/service/audit-service.go b/cmd/metal-api/internal/service/audit-service.go index 2532b6ab..49b4b82c 100644 --- a/cmd/metal-api/internal/service/audit-service.go +++ b/cmd/metal-api/internal/service/audit-service.go @@ -64,7 +64,7 @@ func (r *auditResource) find(request *restful.Request, response *restful.Respons return } - backendResult, err := r.a.Search(auditing.EntryFilter{ + backendResult, err := r.a.Search(request.Request.Context(), auditing.EntryFilter{ Limit: requestPayload.Limit, From: requestPayload.From, To: requestPayload.To, diff --git a/cmd/metal-api/main.go b/cmd/metal-api/main.go index 31514a80..343e3a4e 100644 --- a/cmd/metal-api/main.go +++ b/cmd/metal-api/main.go @@ -287,11 +287,19 @@ func init() { rootCmd.Flags().StringP("masterdata-certkeypath", "", "", "the tls certificate key to talk to the masterdata-api") rootCmd.Flags().Bool("auditing-enabled", false, "enable auditing") - rootCmd.Flags().String("auditing-url", "http://localhost:7700", "url of the auditing service") - rootCmd.Flags().String("auditing-api-key", "secret", "api key for the auditing service") - rootCmd.Flags().String("auditing-index-prefix", "auditing", "auditing index prefix") - rootCmd.Flags().String("auditing-index-interval", "@daily", "auditing index creation interval, can be one of @hourly|@daily|@monthly") - rootCmd.Flags().Int64("auditing-keep", 14, "the amount of indexes to keep until cleanup") + + rootCmd.Flags().String("auditing-meili-url", "http://localhost:7700", "url of the auditing service") + rootCmd.Flags().String("auditing-meili-api-key", "secret", "api key for the auditing service") + rootCmd.Flags().String("auditing-meili-index-prefix", "auditing", "auditing index prefix") + rootCmd.Flags().String("auditing-meili-index-interval", "@daily", "auditing index creation interval, can be one of @hourly|@daily|@monthly") + rootCmd.Flags().Int64("auditing-meili-keep", 14, "the amount of indexes to keep until cleanup") + + rootCmd.Flags().String("auditing-timescaledb-host", "", "host of the auditing service") + rootCmd.Flags().String("auditing-timescaledb-port", "", "port of the auditing service") + rootCmd.Flags().String("auditing-timescaledb-db", "", "database name of the auditing service") + rootCmd.Flags().String("auditing-timescaledb-user", "", "user for the auditing service") + rootCmd.Flags().String("auditing-timescaledb-password", "", "password for the auditing service") + rootCmd.Flags().String("auditing-timescaledb-retention", "", "the time until audit traces are cleaned up") rootCmd.Flags().String("headscale-addr", "", "address of headscale server") rootCmd.Flags().String("headscale-cp-addr", "", "address of headscale control plane") @@ -691,7 +699,7 @@ func initAuth(lg *slog.Logger) security.UserGetter { return security.NewCreds(auths...) } -func initRestServices(audit auditing.Auditing, withauth bool, ipmiSuperUser metal.MachineIPMISuperUser) *restfulspec.Config { +func initRestServices(audit []auditing.Auditing, withauth bool, ipmiSuperUser metal.MachineIPMISuperUser) *restfulspec.Config { service.BasePath = viper.GetString("base-path") if !strings.HasPrefix(service.BasePath, "/") || !strings.HasSuffix(service.BasePath, "/") { log.Fatal("base path must start and end with a slash") @@ -757,7 +765,7 @@ func initRestServices(audit auditing.Auditing, withauth bool, ipmiSuperUser meta releaseVersion = pointer.Pointer(viper.GetString("release-version")) } - restful.DefaultContainer.Add(service.NewAudit(logger.WithGroup("audit-service"), audit)) + restful.DefaultContainer.Add(service.NewAudit(logger.WithGroup("audit-service"), pointer.FirstOrZero(audit))) restful.DefaultContainer.Add(service.NewPartition(logger.WithGroup("partition-service"), ds, nsqer)) restful.DefaultContainer.Add(service.NewImage(logger.WithGroup("image-service"), ds)) restful.DefaultContainer.Add(service.NewSize(logger.WithGroup("size-service"), ds, mdc)) @@ -790,8 +798,8 @@ func initRestServices(audit auditing.Auditing, withauth bool, ipmiSuperUser meta restful.DefaultContainer.Filter(ensurer.EnsureAllowedTenantFilter) } - if audit != nil { - httpFilter, err := auditing.HttpFilter(audit, logger.WithGroup("audit-middleware")) + for _, backend := range audit { + httpFilter, err := auditing.HttpFilter(backend, logger.WithGroup("audit-middleware")) if err != nil { log.Fatalf("unable to create http filter for auditing: %s", err) } @@ -908,7 +916,7 @@ func evaluateVPNConnected() error { } // might return (nil, nil) if auditing is disabled! -func createAuditingClient(log *slog.Logger) (auditing.Auditing, error) { +func createAuditingClient(log *slog.Logger) ([]auditing.Auditing, error) { isEnabled := viper.GetBool("auditing-enabled") if !isEnabled { log.Warn("auditing is disabled, can be enabled by setting --auditing-enabled=true") @@ -916,15 +924,46 @@ func createAuditingClient(log *slog.Logger) (auditing.Auditing, error) { } c := auditing.Config{ - Component: "metal-api", - URL: viper.GetString("auditing-url"), - APIKey: viper.GetString("auditing-api-key"), - IndexPrefix: viper.GetString("auditing-index-prefix"), - RotationInterval: auditing.Interval(viper.GetString("auditing-index-interval")), - Keep: viper.GetInt64("auditing-keep"), - Log: log, //FIXME - } - return auditing.New(c) + Component: "metal-api", + Log: log, + } + + var backends []auditing.Auditing + + if viper.IsSet("auditing-timescaledb-host") { + backend, err := auditing.NewTimescaleDB(c, auditing.TimescaleDbConfig{ + Host: viper.GetString("auditing-timescaledb-host"), + Port: viper.GetString("auditing-timescaledb-port"), + DB: viper.GetString("auditing-timescaledb-db"), + User: viper.GetString("auditing-timescaledb-user"), + Password: viper.GetString("auditing-timescaledb-password"), + Retention: viper.GetString("auditing-timescaledb-retention"), + }) + + if err != nil { + return nil, err + } + + backends = append(backends, backend) + } + + if viper.IsSet("auditing-meili-api-key") { + backend, err := auditing.NewMeilisearch(c, auditing.MeilisearchConfig{ + URL: viper.GetString("auditing-meili-url"), + APIKey: viper.GetString("auditing-meili-api-key"), + IndexPrefix: viper.GetString("auditing-meili-index-prefix"), + RotationInterval: auditing.Interval(viper.GetString("auditing-meili-index-interval")), + Keep: viper.GetInt64("auditing-meili-keep"), + }) + + if err != nil { + return nil, err + } + + backends = append(backends, backend) + } + + return backends, nil } func run() error { diff --git a/go.mod b/go.mod index dfd9688d..1cb27e0f 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/metal-stack/metal-api go 1.22 +replace github.com/metal-stack/metal-lib => ../metal-lib + require ( connectrpc.com/connect v1.16.2 github.com/Masterminds/semver/v3 v3.2.1 @@ -20,7 +22,7 @@ require ( github.com/looplab/fsm v1.0.2 github.com/metal-stack/go-ipam v1.14.5 github.com/metal-stack/masterdata-api v0.11.4 - github.com/metal-stack/metal-lib v0.18.1 + github.com/metal-stack/metal-lib v0.18.2-0.20240826135712-504fccc29464 github.com/metal-stack/security v0.8.1 github.com/metal-stack/v v1.0.3 github.com/nsqio/go-nsq v1.1.0 @@ -38,6 +40,8 @@ require ( gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.2 ) +require github.com/lopezator/migrator v0.3.1 // indirect + replace ( // Newer versions do not export base entities which are used to composite other entities. // This breaks metalctl and friends diff --git a/go.sum b/go.sum index f8cb449d..dcbf650f 100644 --- a/go.sum +++ b/go.sum @@ -210,10 +210,22 @@ github.com/icza/dyno v0.0.0-20230330125955-09f820a8d9c0 h1:nHoRIX8iXob3Y2kdt9Ksj github.com/icza/dyno v0.0.0-20230330125955-09f820a8d9c0/go.mod h1:c1tRKs5Tx7E2+uHGSyyncziFjvGpgv4H2HrqXeUQ/Uk= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v1.11.0 h1:HiHArx4yFbwl91X3qqIHtUFoiIfLNJXCQRsnzkiwwaQ= +github.com/jackc/pgconn v1.11.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3/v2 v2.2.0 h1:r7JypeP2D3onoQTCxWdTpCtJ4D+qpKr0TxvoyMhZ5ns= +github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgtype v1.10.0 h1:ILnBWrRMSXGczYvmkYD6PsYyVFUNLTnIUJHHDLmqk38= +github.com/jackc/pgtype v1.10.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.15.0 h1:B7dTkXsdILD3MF987WGGCcg+tvLW6bZJdEcqVFeU//w= +github.com/jackc/pgx/v4 v4.15.0/go.mod h1:D/zyOyXiaM1TmVWnOM18p0xdDtdakRBa0RsVGI3U3bw= github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= @@ -272,6 +284,8 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/looplab/fsm v1.0.2 h1:f0kdMzr4CRpXtaKKRUxwLYJ7PirTdwrtNumeLN+mDx8= github.com/looplab/fsm v1.0.2/go.mod h1:PmD3fFvQEIsjMEfvZdrCDZ6y8VwKTwWNjlpEr6IKPO4= +github.com/lopezator/migrator v0.3.1 h1:ZFPT6aC7+nGWkqhleynABZ6ftycSf6hmHHLOaryq1Og= +github.com/lopezator/migrator v0.3.1/go.mod h1:X+lHDMZ9Ci3/KdbypJcQYFFwipVrJsX4fRCQ4QLauYk= github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae h1:dIZY4ULFcto4tAFlj1FYZl8ztUZ13bdq+PLY+NOfbyI= github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= @@ -299,8 +313,6 @@ github.com/metal-stack/go-ipam v1.14.5 h1:KSnftPoySufz/SSbAmtCqo/HzmlYuyVMSfMi53 github.com/metal-stack/go-ipam v1.14.5/go.mod h1:K/ax3O8oPYIClpEpSLmu0a2NfKM/9qNrNLa05cdYndY= github.com/metal-stack/masterdata-api v0.11.4 h1:bgRk7PbD5BjYbmAReaV7gTKKKrW5x/ZzCwj98VSWoJk= github.com/metal-stack/masterdata-api v0.11.4/go.mod h1:fD0AtsoNNaOLqRMBeZzDFljiQW9RlrOnxeZ20Pqhxas= -github.com/metal-stack/metal-lib v0.18.1 h1:Kjmf/Z/6pWemR8O6ttbNPQ9PjeT3ON60sBNu51Lgi1M= -github.com/metal-stack/metal-lib v0.18.1/go.mod h1:GJjipRpHmpd2vjBtsaw9gGk5ZFan7NlShyjIsTdY1x4= github.com/metal-stack/security v0.8.1 h1:4zmVUxZvDWShVvVIxM3XhIv7pTmPe9DvACRIHW6YTsk= github.com/metal-stack/security v0.8.1/go.mod h1:OO8ZilZO6fUV5QEmwc7HP/RAjqYrGQxXoYIddJ9TvqE= github.com/metal-stack/v v1.0.3 h1:Sh2oBlnxrCUD+mVpzfC8HiqL045YWkxs0gpTvkjppqs=