feat: support real-time reading of gzip log files

Author: silenceqi

lib/util/file.go

@@ -6,6 +6,9 @@ package util
 
 import (
 	"bufio"
+	"compress/gzip"
+	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 )
@@ -37,3 +40,33 @@ func ResolveSymlink(link string) (string, error) {
 	}
 	return absPath, nil
 }
+
+// UnpackGzipFile extracts a .gz file and writes the output to a regular file.
+func UnpackGzipFile(gzFile, outputFile string) error {
+	f, err := os.Open(gzFile)
+	if err != nil {
+		return fmt.Errorf("failed to open .gz file %s: %w", gzFile, err)
+	}
+	defer f.Close()
+
+	// Create a gzip reader
+	gzReader, err := gzip.NewReader(f)
+	if err != nil {
+		return fmt.Errorf("failed to create gzip reader with file %s: %w", gzFile, err)
+	}
+	defer gzReader.Close()
+
+	// Create the output file
+	outFile, err := os.Create(outputFile)
+	if err != nil {
+		return fmt.Errorf("failed to create output file %s: %w", outputFile, err)
+	}
+	defer outFile.Close()
+
+	// Copy decompressed content to output file
+	_, err = io.Copy(outFile, gzReader)
+	if err != nil {
+		return fmt.Errorf("failed to write to output file %s: %w", outputFile, err)
+	}
+	return nil
+}

plugin/api/log.go

@@ -37,28 +37,26 @@ func (handler *AgentAPI) getElasticLogFiles(w http.ResponseWriter, req *http.Req
 	}
 	var files []util.MapStr
 	for _, info := range fileInfos {
-		if strings.HasSuffix(info.Name(), ".log") || strings.HasSuffix(info.Name(), ".json") {
-			if info.IsDir() {
-				continue
-			}
-			fInfo, err := info.Info()
-			if err != nil {
-				log.Error(err)
-				continue
-			}
-			filePath := path.Join(reqBody.LogsPath, info.Name())
-			totalRows, err := util2.CountFileRows(filePath)
-			if err != nil {
-				log.Error(err)
-				continue
-			}
-			files = append(files, util.MapStr{
-				"name":          fInfo.Name(),
-				"size_in_bytes": fInfo.Size(),
-				"modify_time":   fInfo.ModTime(),
-				"total_rows":    totalRows,
-			})
+		if info.IsDir() {
+			continue
+		}
+		fInfo, err := info.Info()
+		if err != nil {
+			log.Error(err)
+			continue
+		}
+		filePath := path.Join(reqBody.LogsPath, info.Name())
+		totalRows, err := util2.CountFileRows(filePath)
+		if err != nil {
+			log.Error(err)
+			continue
 		}
+		files = append(files, util.MapStr{
+			"name":          fInfo.Name(),
+			"size_in_bytes": fInfo.Size(),
+			"modify_time":   fInfo.ModTime(),
+			"total_rows":    totalRows,
+		})
 	}
 
 	handler.WriteJSON(w, util.MapStr{
@@ -80,6 +78,28 @@ func (handler *AgentAPI) readElasticLogFile(w http.ResponseWriter, req *http.Req
 	if reqBody.StartLineNumber < 0 {
 		reqBody.StartLineNumber = 0
 	}
+	if strings.HasSuffix(reqBody.FileName, ".gz") {
+		// read gzip log file, and then unpack it to tmp file
+		tmpFilePath := filepath.Join(os.TempDir(), "agent", strings.TrimSuffix(reqBody.FileName, ".gz"))
+		if !util.FileExists(tmpFilePath) {
+			fileDir := filepath.Dir(tmpFilePath)
+			if !util.FileExists(fileDir) {
+				err = os.MkdirAll(fileDir, os.ModePerm)
+				if err != nil {
+					log.Error(err)
+					handler.WriteJSON(w, err.Error(), http.StatusInternalServerError)
+					return
+				}
+			}
+			err = util2.UnpackGzipFile(logFilePath, tmpFilePath)
+			if err != nil {
+				log.Error(err)
+				handler.WriteJSON(w, err.Error(), http.StatusInternalServerError)
+				return
+			}
+		}
+		logFilePath = tmpFilePath
+	}
 	r, err := linenumber.NewLinePlainTextReader(logFilePath, reqBody.StartLineNumber, io.SeekStart)
 	if err != nil {
 		log.Error(err)