support file output plugin

MAINTAINERS

@@ -1,4 +1,4 @@
 Jizhong Jiang <[email protected]> (@jzwlqx)
 Bingshen Wang <[email protected]> (@BSWANG)
 Linhua Tan <[email protected]> (@toolchainX)
-
+Quanzhao Chen <[email protected]> (@chenqz1987)

docker-images/config.default

@@ -27,13 +27,20 @@ cat << EOF
 ${FLUENTD_BUFFER_TYPE:+buffer_type $FLUENTD_BUFFER_TYPE}
 ${FLUENTD_BUFFER_CHUNK_LIMIT:+buffer_chunk_limit $FLUENTD_BUFFER_CHUNK_LIMIT}
 ${FLUENTD_BUFFER_QUEUE_LIMIT:+buffer_queue_limit $FLUENTD_BUFFER_QUEUE_LIMIT}
+${FLUENTD_BUFFER_CHUNK_LIMIT_SIZE:+chunk_limit_size ${FLUENTD_BUFFER_CHUNK_LIMIT_SIZE}}
+${FLUENTD_BUFFER_TOTAL_LIMIT_SIZE:+total_limit_size ${FLUENTD_BUFFER_TOTAL_LIMIT_SIZE}}
+${FLUENTD_BUFFER_CHUNK_FULL_THRESHOLD:+chunk_full_threshold ${FLUENTD_BUFFER_CHUNK_FULL_THRESHOLD}}
+${FLUENTD_BUFFER_COMPRESS:+compress ${FLUENTD_BUFFER_COMPRESS}}
 ${FLUENTD_FLUSH_INTERVAL:+flush_interval $FLUENTD_FLUSH_INTERVAL}
+${FLUENTD_FLUSH_MODE:+flush_mode ${FLUENTD_FLUSH_MODE}}
+${FLUENTD_FLUSH_INTERVAL:+flush_interval ${FLUENTD_FLUSH_INTERVAL}}
+${FLUENTD_FLUSH_THREAD_COUNT:+flush_thread_count ${FLUENTD_FLUSH_THREAD_COUNT}}
+${FLUENTD_FLUSH_AT_SHUTDOWN:+flush_at_shutdown $FLUENTD_FLUSH_AT_SHUTDOWN}
 ${FLUENTD_DISABLE_RETRY_LIMIT:+disable_retry_limit $FLUENTD_DISABLE_RETRY_LIMIT}
 ${FLUENTD_RETRY_LIMIT:+retry_limit $FLUENTD_RETRY_LIMIT}
 ${FLUENTD_RETRY_WAIT:+retry_wait $FLUENTD_RETRY_WAIT}
 ${FLUENTD_MAX_RETRY_WAIT:+max_retry_wait $FLUENTD_MAX_RETRY_WAIT}
 ${FLUENTD_NUM_THREADS:+num_threads $FLUENTD_NUM_THREADS}
-${FLUENTD_FLUSH_AT_SHUTDOWN:+flush_at_shutdown $FLUENTD_FLUSH_AT_SHUTDOWN}
 EOF
 }
 
@@ -75,7 +82,20 @@ assert_not_empty "$FILE_PATH" "FILE_PATH required"
 cat >> $FLUENTD_CONFIG << EOF
 <match docker.**>
 @type file
-path $FILE_PATH
+path $FILE_PATH/\${tag[1]}/\${tag[2]}.%Y-%m-%d.%H%M
+append ${FILE_APPEND:=true}
+${FILE_COMPRESS:+compress ${FILE_COMPRESS}}
+<format>
+  @type ${FILE_FORMAT:=json}
+</format>
+<buffer tag,time>
+  @type ${FILE_BUFFER_TYPE:=file}
+  path $FILE_PATH/.buffer
+  timekey ${FILE_BUFFER_TIME_KEY:=1m}
+  timekey_wait ${FILE_BUFFER_TIME_KEY_WAIT:=1m}
+  timekey_use_utc ${FILE_BUFFER_TIME_KEY_USE_UTC:=false}
+  $(bufferd_output)
+</buffer>
 </match>
 EOF
 }

pilot/fluentd_wrapper.go

@@ -11,20 +11,27 @@ import (
 
 var fluentd *exec.Cmd
 
+const ERR_ALREADY_STARTED = "fluentd already started"
+
 func StartFluentd() error {
 	if fluentd != nil {
-		return fmt.Errorf("fluentd already started")
+		return fmt.Errorf(ERR_ALREADY_STARTED)
 	}
-	log.Warn("start fluentd")
+
+	log.Info("start fluentd")
 	fluentd = exec.Command("/usr/bin/fluentd", "-c", "/etc/fluentd/fluentd.conf", "-p", "/etc/fluentd/plugins")
 	fluentd.Stderr = os.Stderr
 	fluentd.Stdout = os.Stdout
 	err := fluentd.Start()
 	if err != nil {
-		go func() {
-			fluentd.Wait()
-		}()
+		log.Error(err)
 	}
+	go func() {
+		err := fluentd.Wait()
+		if err != nil {
+			log.Error(err)
+		}
+	}()
 	return err
 }
 
@@ -39,9 +46,12 @@ func shell(command string) string {
 
 func ReloadFluentd() error {
 	if fluentd == nil {
-		return fmt.Errorf("fluentd have not started")
+		err := fmt.Errorf("fluentd have not started")
+		log.Error(err)
+		return err
 	}
-	log.Warn("reload fluentd")
+
+	log.Info("reload fluentd")
 	ch := make(chan struct{})
 	go func(pid int) {
 		command := fmt.Sprintf("pgrep -P %d", pid)

pilot/pilot.go

@@ -72,9 +72,13 @@ func (p *Pilot) watch() error {
 	if err := p.processAllContainers(); err != nil {
 		return err
 	}
-	StartFluentd()
-	p.lastReload = time.Now()
 
+	err := StartFluentd()
+	if err != nil && ERR_ALREADY_STARTED != err.Error() {
+		return err
+	}
+
+	p.lastReload = time.Now()
 	go p.doReload()
 
 	ctx := context.Background()
@@ -242,7 +246,7 @@ func (p *Pilot) newContainer(containerJSON *types.ContainerJSON) error {
 		return err
 	}
 	//TODO validate config before save
-	log.Infof("container %s fluentd config: %s", id, fluentdConfig)
+	//log.Debugf("container %s fluentd config: %s", id, fluentdConfig)
 	if err = ioutil.WriteFile(p.pathOf(id), []byte(fluentdConfig), os.FileMode(0644)); err != nil {
 		return err
 	}

quickstart/busybox.yml

@@ -0,0 +1,7 @@
+busybox:
+  image: busybox
+  restart: always
+  labels:
+    aliyun.logs.busybox: stdout
+    aliyun.logs.busybox.tags: app=busybox,stage=test
+  command: ['ping', 'localhost']

quickstart/file.yml

@@ -0,0 +1,12 @@
+version: '2'
+services:
+  pilot:
+    image: pilot:latest
+    privileged: true
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /etc/localtime:/etc/localtime
+      - /:/host
+    environment:
+      FLUENTD_OUTPUT: file
+      FILE_PATH: /host/tmp/logs

quickstart/run_file

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+cd $(dirname $0)
+
+green(){
+    echo -e "\033[0;32m$*\033[0m"
+}
+
+blue(){
+    echo -e "\033[0;34m$*\033[0m"
+}
+
+blue "Cleanup"
+docker-compose -p quickstart -f file.yml down
+
+blue "Starting file+fluentd-pilot"
+docker-compose -p quickstart -f file.yml up -d
+
+host=127.0.0.1
+
+if [ -n "DOCKER_HOST" ]; then
+    host=$(echo $DOCKER_HOST|sed -e 's:^.*//::' -e 's/:.*$//')
+fi
+
+if [ -z "$host" ]; then
+    echo "Could not detect docker host."
+fi
+
+blue "\nCleanup"
+#docker-compose -p busybox -f busybox.yml down
+docker-compose -p tomcat -f tomcat.yml down
+
+blue "Starting application"
+#docker-compose -p busybox -f busybox.yml up -d
+docker-compose -p tomcat -f tomcat.yml up -d
+
+pwd=$(pwd)
+project=$(basename $pwd)
+
+echo 
+green "Start successfully!"
+echo
+
+cat << EOF
+Now enter the FILE_PATH and after 1 minutes you can see any logs.
+EOF