Add ability to configure agent metric port and add host tags during l…

…inux installation (#143)

cmd/host-agent/main.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strconv"
 
 	"github.com/middleware-labs/mw-agent/pkg/agent"
 	"github.com/prometheus/common/version"
@@ -53,6 +54,7 @@ func (p *program) Stop(s service.Service) error {
 func (p *program) run() {
 	if err := p.collector.Run(context.Background()); err != nil {
 		p.logger.Error("collector server run finished with error", zap.Error(err))
+		os.Exit(1)
 	}
 }
 
@@ -73,7 +75,7 @@ func getFlags(execPath string, cfg *agent.HostConfig) []cli.Flag {
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    "config-check-interval",
 			EnvVars: []string{"MW_CONFIG_CHECK_INTERVAL"},
-			Usage: "Duration string to periodically check for configuration updates." +
+			Usage: "Duration string to periodically check for configuration updates. " +
 				"Setting the value to 0 disables this feature.",
 			Destination: &cfg.ConfigCheckInterval,
 			DefaultText: "60s",
@@ -82,15 +84,15 @@ func getFlags(execPath string, cfg *agent.HostConfig) []cli.Flag {
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:        "fetch-account-otel-config",
 			EnvVars:     []string{"MW_FETCH_ACCOUNT_OTEL_CONFIG"},
-			Usage:       "Get the otel-config from Middleware backend",
+			Usage:       "Get the otel-config from Middleware backend.",
 			Destination: &cfg.FetchAccountOtelConfig,
 			DefaultText: "true",
 			Value:       true,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:        "docker-endpoint",
 			EnvVars:     []string{"MW_DOCKER_ENDPOINT"},
-			Usage:       "Set the endpoint for Docker socket if different from default",
+			Usage:       "Set the endpoint for Docker socket if different from default.",
 			Destination: &cfg.DockerEndpoint,
 			DefaultText: "unix:///var/run/docker.sock",
 			Value:       "unix:///var/run/docker.sock",
@@ -105,49 +107,53 @@ func getFlags(execPath string, cfg *agent.HostConfig) []cli.Flag {
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:        "host-tags",
-			Usage:       "Tags for this host",
+			Usage:       "Tags for this host.",
 			EnvVars:     []string{"MW_HOST_TAGS"},
 			Destination: &cfg.HostTags,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:        "fluent-port",
-			Usage:       "Fluent receiver will listen to this port",
+			Usage:       "Fluent receiver will listen to this port.",
 			EnvVars:     []string{"MW_FLUENT_PORT"},
 			Destination: &cfg.FluentPort,
+			DefaultText: "8006",
 			Value:       "8006",
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:        "logfile",
-			Usage:       "Log file to store Middleware agent logs",
+			Usage:       "Log file to store Middleware agent logs.",
 			EnvVars:     []string{"MW_LOGFILE"},
 			Destination: &cfg.Logfile,
 			DefaultText: "",
 			Value:       "",
 		}),
 		altsrc.NewIntFlag(&cli.IntFlag{
 			Name:        "logfile-size",
-			Usage:       "Log file size to store Middleware agent logs. This flag only applifes if logfile flag is specified",
+			Usage:       "Log file size to store Middleware agent logs. This flag only applifes if logfile flag is specified.",
 			EnvVars:     []string{"MW_LOGFILE_SIZE"},
 			Destination: &cfg.LogfileSize,
+			DefaultText: "1",
 			Value:       1, // default value is 1MB
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:        "agent-features.infra-monitoring",
-			Usage:       "Flag to enable or disable infrastructure monitoring",
+			Usage:       "Flag to enable or disable infrastructure monitoring.",
 			EnvVars:     []string{"MW_AGENT_FEATURES_INFRA_MONITORING"},
 			Destination: &cfg.AgentFeatures.InfraMonitoring,
+			DefaultText: "true",
 			Value:       true, // infra monitoring is enabled by default
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:        "mw-agent-self-profiling",
-			Usage:       "For Profiling the agent itself",
+			Usage:       "For Profiling MW Agent itself.",
 			EnvVars:     []string{"MW_AGENT_SELF_PROFILING"},
 			Destination: &cfg.SelfProfiling,
+			DefaultText: "false",
 			Value:       false,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:        "mw-profiling-server-url",
-			Usage:       "Server Address for redirecting profiling data",
+			Usage:       "Server Address for redirecting profiling data.",
 			EnvVars:     []string{"MW_PROFILING_SERVER_URL"},
 			Destination: &cfg.ProfilngServerURL,
 			Value:       "https://profiling.middleware.io",
@@ -191,6 +197,15 @@ func getFlags(execPath string, cfg *agent.HostConfig) []cli.Flag {
 				return ""
 			}(),
 		},
+
+		altsrc.NewUintFlag(&cli.UintFlag{
+			Name:        "agent-internal-metrics-port",
+			Usage:       "Port where mw-agent will expose its Prometheus metrics.",
+			EnvVars:     []string{"MW_AGENT_INTERNAL_METRICS_PORT"},
+			Destination: &cfg.InternalMetricsPort,
+			DefaultText: "8888",
+			Value:       8888,
+		}),
 	}
 }
 
@@ -266,7 +281,14 @@ func main() {
 						infraPlatform = agent.InfraPlatformECSFargate
 					}
 
-					logger.Info("starting host agent", zap.String("agent location", execPath))
+					hostname, err := os.Hostname()
+					if err != nil {
+						logger.Error("error getting hostname", zap.Error(err))
+						hostname = "unknown"
+					}
+
+					logger.Info("starting host agent", zap.String("agent location", execPath),
+						zap.String("hostname", hostname))
 
 					logger.Info("host agent config", zap.Stringer("config", cfg),
 						zap.String("version", agentVersion),
@@ -295,15 +317,6 @@ func main() {
 					ctx, cancel := context.WithCancel(c.Context)
 					defer cancel()
 
-					// Listen to the config changes provided by Middleware API
-					if cfg.ConfigCheckInterval != "0" {
-						err = hostAgent.ListenForConfigChanges(ctx)
-						if err != nil {
-							logger.Info("error for listening for config changes", zap.Error(err))
-							return err
-						}
-					}
-
 					u, err := url.Parse(cfg.Target)
 					if err != nil {
 						return err
@@ -318,6 +331,7 @@ func main() {
 					os.Setenv("MW_TARGET", target)
 					os.Setenv("MW_API_KEY", cfg.APIKey)
 					os.Setenv("MW_FLUENT_PORT", cfg.FluentPort)
+					os.Setenv("MW_AGENT_INTERNAL_METRICS_PORT", strconv.Itoa(int(cfg.InternalMetricsPort)))
 
 					// TODO: check if on Windows, socket scheme is different than "unix"
 					os.Setenv("MW_DOCKER_ENDPOINT", cfg.DockerEndpoint)
@@ -331,12 +345,19 @@ func main() {
 					}
 
 					if cfg.FetchAccountOtelConfig {
-						yamlPath, err := hostAgent.GetUpdatedYAMLPath()
+						otelConfigFile, err := hostAgent.GetOtelConfig()
 						if err != nil {
 							logger.Error("error getting config file path", zap.Error(err))
 							return err
 						}
-						logger.Info("yaml path loaded", zap.String("path", yamlPath))
+						logger.Info("otel config file", zap.String("path", otelConfigFile))
+
+						// Listen to the config changes provided by Middleware API
+						err = hostAgent.ListenForConfigChanges(ctx)
+						if err != nil {
+							logger.Info("error for listening for config changes", zap.Error(err))
+							return err
+						}
 					}
 
 					configProvider, err := otelcol.NewConfigProvider(otelcol.ConfigProviderSettings{

package-tooling/agent-config.yaml.sample

@@ -6,20 +6,22 @@ api-key:
 # target is your unique Middleware account URL
 target: 
 
-# Synthetic monitoring using the Middleware agent is disabled by default.
-# Set enable-synthetic-monitoring to true if you want this agent to be
-# able to perform synthetic tests.
-enable-synthetic-monitoring: false
-
 # The duration for the agent to check for configuration changes from the
 # backend. We recommend to keep this change to 5 minutes or higher. Keeping 
 # this duration to a smaller value will impact resource consumption on the
 # host.
 config-check-interval: "5m"
 
 # The tags required to identify / categorize the host in the Middleware UI.
-# The tags are comma separated key:value pairs.
+# The tags are comma separated key:value pairs. Empty host-tag should always
+# be a double quoted string ("").
+# Examples:
+# host-tags: "" 
 # host-tags: "name:my-machine,env:production"
+host-tags: ""
+
+# This is the port where MW Agent exposes its internal metrics.
+agent-internal-metrics-port: 8888
 
 # log file to store agent logs in a given file. If left empty,
 # the logs will be directed to stdout & stderr. 
@@ -33,5 +35,5 @@ config-check-interval: "5m"
 #
 # infra-monitoring: By setting this flag to false, you can disable infrastructure
 # monitoring from this agent. infra-monitoring is set to true by default.
-#agent-features:
+# agent-features:
 #  infra-monitoring: false

package-tooling/linux/postinst

@@ -20,17 +20,29 @@ restart_service() {
 }
 
 # Function to update configuration values
+
 update_config() {
     local key=$1
     local value=$2
     local file=$3
 
     # Update configuration file with the provided key and value
     sed -i "s|^\s*${key}:.*|${key}: ${value}|" "$file"
+
     if [ $? -ne 0 ]; then
         echo "Error: Failed to update configuration file."
         exit 1
     fi
+
+    # Check if the key was updated. If not, append the key-value pair to the file.
+    grep -q "^\s*${key}:" "$file"
+    if [ $? -ne 0 ]; then
+        echo "${key}: ${value}" >> "$file"
+        if [ $? -ne 0 ]; then
+            echo "Error: Failed to append to configuration file."
+            exit 1
+        fi
+    fi
 }
 
 # Function to extract environment variables from old execuable file
@@ -56,15 +68,26 @@ handle_variable() {
         MW_TARGET)
             update_config "target" "$value" "${CONFIG_FILE}"
             ;;
-        MW_ENABLE_SYNTHETIC_MONITORING)
-            update_config "enable-synthetic-monitoring" "$value" "${CONFIG_FILE}"
-            ;;
         MW_CONFIG_CHECK_INTERVAL)
             update_config "config-check-interval" "$value" "${CONFIG_FILE}"
             ;;
+        MW_HOST_TAGS)
+            update_config "host-tags" "$value" "${CONFIG_FILE}"
+            ;;
+        MW_AGENT_INTERNAL_METRICS_PORT)
+            update_config "agent-internal-metrics-port" "$value" "${CONFIG_FILE}"
+            ;;
+        # Advanced configuration options
         MW_API_URL_FOR_CONFIG_CHECK)
             update_config "api-url-for-config-check" "$value" "${CONFIG_FILE}"
             ;;
+        MW_FETCH_ACCOUNT_OTEL_CONFIG)
+            update_config "fetch-account-otel-config" "$value" "${CONFIG_FILE}"
+            ;; 
+        MW_ENABLE_SYNTHETIC_MONITORING)
+            # synthetic monitoring feature has been removed from the agent
+            ;;
+
         PATH)
             ;;
         *)
@@ -145,6 +168,31 @@ else
         # Update configuration file with environment variable values
         handle_variable "MW_API_KEY" "${MW_API_KEY}"
         handle_variable "MW_TARGET" "${MW_TARGET}"
+
+        if [ -n "${MW_CONFIG_CHECK_INTERVAL}" ]; then
+            handle_variable "MW_CONFIG_CHECK_INTERVAL" "${MW_CONFIG_CHECK_INTERVAL}"
+        fi
+
+        if [ -n "${MW_HOST_TAGS}" ]; then
+            handle_variable "MW_HOST_TAGS" "${MW_HOST_TAGS}"
+        fi
+
+        if [ -n "${MW_AGENT_INTERNAL_METRICS_PORT}" ]; then
+            handle_variable "MW_AGENT_INTERNAL_METRICS_PORT" "${MW_AGENT_INTERNAL_METRICS_PORT}"
+        fi
+
+        if [ -n "${MW_API_URL_FOR_CONFIG_CHECK}" ]; then
+            handle_variable "MW_API_URL_FOR_CONFIG_CHECK" "${MW_API_URL_FOR_CONFIG_CHECK}"
+        fi
+
+        if [ -n "${MW_ENABLE_SYNTHETIC_MONITORING}" ]; then
+            handle_variable "MW_ENABLE_SYNTHETIC_MONITORING" "${MW_ENABLE_SYNTHETIC_MONITORING}"
+        fi 
+
+        if [ -n "${MW_FETCH_ACCOUNT_OTEL_CONFIG}" ]; then
+            handle_variable "MW_FETCH_ACCOUNT_OTEL_CONFIG" "${MW_FETCH_ACCOUNT_OTEL_CONFIG}"
+        fi
+
     fi
 fi
 

package-tooling/otel-config.yaml.sample

@@ -198,7 +198,7 @@ receivers:
         scrape_interval: 5s
         static_configs:
         - targets:
-          - 127.0.0.1:8888
+          - 127.0.0.1:${MW_AGENT_INTERNAL_METRICS_PORT}
 service:
   pipelines:
     logs:
@@ -239,5 +239,7 @@ service:
       receivers:
       - otlp
   telemetry:
+    metrics:
+      address: 127.0.0.1:${MW_AGENT_INTERNAL_METRICS_PORT}
     logs:
       level: fatal

pkg/agent/definitions.go

@@ -81,6 +81,7 @@ type BaseConfig struct {
 	AgentFeatures             AgentFeatures
 	SelfProfiling             bool
 	ProfilngServerURL         string
+	InternalMetricsPort       uint
 }
 
 // String() implements stringer interface for BaseConfig

pkg/agent/hostagent.go

@@ -221,7 +221,7 @@ func (c *HostAgent) updateConfig(config map[string]interface{}, cnf integrationC
 	return config, nil
 }
 
-func (c *HostAgent) updateYAML(configType, yamlPath string) error {
+func (c *HostAgent) updateConfigFile(configType string) error {
 	// _, apiURLForYAML := checkForConfigURLOverrides()
 
 	hostname := getHostname()
@@ -334,14 +334,14 @@ func (c *HostAgent) updateYAML(configType, yamlPath string) error {
 }
 
 // GetUpdatedYAMLPath gets the correct otel configuration file
-func (c *HostAgent) GetUpdatedYAMLPath() (string, error) {
+func (c *HostAgent) GetOtelConfig() (string, error) {
 	configType := "docker"
 	dockerSocketPath := strings.Split(c.DockerEndpoint, "//")
 	if len(dockerSocketPath) != 2 || !isSocketFn(dockerSocketPath[1]) {
 		configType = "nodocker"
 	}
 
-	if err := c.updateYAML(configType, c.OtelConfigFile); err != nil {
+	if err := c.updateConfigFile(configType); err != nil {
 		return "", err
 	}
 
@@ -384,7 +384,7 @@ func (c *HostAgent) checkIntConfigValidity(integrationType IntegrationType, cnf
 }
 
 func (c *HostAgent) restartHostAgent() error {
-	c.GetUpdatedYAMLPath()
+	c.GetOtelConfig()
 	cmd := exec.Command("kill", "-SIGHUP", fmt.Sprintf("%d", os.Getpid()))
 	err := cmd.Run()
 	if err != nil {
@@ -435,7 +435,7 @@ func (c *HostAgent) callRestartStatusAPI() error {
 
 	if apiResponse.Restart {
 		c.logger.Info("restarting mw-agent")
-		if _, err := c.GetUpdatedYAMLPath(); err != nil {
+		if _, err := c.GetOtelConfig(); err != nil {
 			c.logger.Error("error getting Updated YAML", zap.Error(err))
 			return err
 		}