Merge pull request #264 from projectdiscovery/issue-263-fileutil-clean

add file clean and SafeOpen utils

.github/workflows/build-test.yml

@@ -26,3 +26,6 @@ jobs:
       - name: Race Condition Tests
         if: ${{ matrix.os != 'windows-latest' }} # false positives in windows
         run: go test -race ./...
+
+      - name: Fuzz File Read  # fuzz tests need to be run separately  
+        run:  go test -fuzztime=10s -fuzz=FuzzSafeOpen -run "FuzzSafeOpen" ./file/...

file/clean.go

@@ -0,0 +1,128 @@
+package fileutil
+
+import (
+	"os"
+	"path/filepath"
+
+	errorutil "github.com/projectdiscovery/utils/errors"
+)
+
+var (
+	DefaultFilePermission = os.FileMode(0644)
+)
+
+// CleanPath cleans paths to migtigate any possible path traversal attacks.
+// and it always returns an absolute path
+func CleanPath(inputPath string) (string, error) {
+	// check if path is abs
+	if filepath.IsAbs(inputPath) {
+		// clean it using filepath.Abs
+		// Abs always calls Clean, so we don't need to call it again
+		return filepath.Abs(inputPath)
+	}
+	// get current working directory
+	cwd, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+	// join cwd with inputPath
+	joined := filepath.Join(cwd, inputPath)
+	// clean it using filepath.Abs
+	return filepath.Abs(joined)
+}
+
+// CleanPathOrDefault cleans paths to migtigate any possible path traversal attacks.
+func CleanPathOrDefault(inputPath string, defaultPath string) string {
+	if inputPath == "" {
+		return defaultPath
+	}
+	if val, err := CleanPath(inputPath); err == nil {
+		return val
+	}
+	return defaultPath
+}
+
+// ResolveNClean resolves the path and cleans it
+// ex: a nuclei template can be either abs or relative to a specified directory
+// this function uses given path as a base to resolve the path instead of cwd
+func ResolveNClean(inputPath string, baseDir ...string) (string, error) {
+	// check if path is abs
+	if filepath.IsAbs(inputPath) {
+		// clean it using filepath.Abs
+		// Abs always calls Clean, so we don't need to call it again
+		return filepath.Abs(inputPath)
+	}
+	for _, dir := range baseDir {
+		// join cwd with inputPath
+		joined := filepath.Join(dir, inputPath)
+		// clean it using filepath.Abs
+		abs, err := filepath.Abs(joined)
+		if err == nil && FileOrFolderExists(abs) {
+			return abs, nil
+		}
+	}
+	return "", errorutil.NewWithErr(os.ErrNotExist).Msgf("failed to resolve path: %s", inputPath)
+}
+
+// ResolveNCleanOrDefault resolves the path and cleans it
+func ResolveNCleanOrDefault(inputPath string, defaultPath string, baseDir ...string) string {
+	if inputPath == "" {
+		return defaultPath
+	}
+	if val, err := ResolveNClean(inputPath, baseDir...); err == nil {
+		return val
+	}
+	return defaultPath
+}
+
+// SafeOpen opens a file after cleaning the path
+// in read mode
+func SafeOpen(path string) (*os.File, error) {
+	abs, err := CleanPath(path)
+	if err != nil {
+		return nil, err
+	}
+	return os.Open(abs)
+}
+
+// SafeOpenAppend opens a file after cleaning the path
+// in append mode and creates any missing directories in chain /path/to/file
+func SafeOpenAppend(path string) (*os.File, error) {
+	abs, err := CleanPath(path)
+	if err != nil {
+		return nil, err
+	}
+	_ = FixMissingDirs(abs)
+	return os.OpenFile(abs, os.O_APPEND|os.O_CREATE|os.O_WRONLY, DefaultFilePermission)
+}
+
+// SafeOpenWrite opens a file after cleaning the path
+// in write mode and creates any missing directories in chain /path/to/file
+func SafeOpenWrite(path string) (*os.File, error) {
+	abs, err := CleanPath(path)
+	if err != nil {
+		return nil, err
+	}
+	_ = FixMissingDirs(abs)
+	return os.OpenFile(abs, os.O_CREATE|os.O_WRONLY, DefaultFilePermission)
+}
+
+// SafeWriteFile writes data to a file after cleaning the path
+// in write mode and creates any missing directories in chain /path/to/file
+func SafeWriteFile(path string, data []byte) error {
+	abs, err := CleanPath(path)
+	if err != nil {
+		return err
+	}
+	_ = FixMissingDirs(abs)
+	return os.WriteFile(abs, data, DefaultFilePermission)
+}
+
+// FixMissingDirs creates any missing directories in chain /path/to/file
+func FixMissingDirs(path string) error {
+	abs, err := CleanPath(path)
+	if err != nil {
+		return err
+	}
+	return os.MkdirAll(filepath.Dir(abs), os.ModePerm)
+}

file/clean_test.go

@@ -0,0 +1,62 @@
+package fileutil
+
+import (
+	"io"
+	"os"
+	"strings"
+	"testing"
+)
+
+func FuzzSafeOpen(f *testing.F) {
+
+	// ==========setup==========
+
+	bin, err := os.ReadFile("tests/path-traversal.txt")
+	if err != nil {
+		f.Fatalf("failed to read file: %s", err)
+	}
+
+	fuzzPayloads := strings.Split(string(bin), "\n")
+
+	file, err := os.CreateTemp("", "*")
+	if err != nil {
+		f.Fatal(err)
+	}
+	_, _ = file.WriteString("pwned!")
+	_ = file.Close()
+
+	defer func(tmp string) {
+		if err = os.Remove(tmp); err != nil {
+			panic(err)
+		}
+	}(file.Name())
+
+	// ==========fuzzing==========
+
+	for _, payload := range fuzzPayloads {
+		f.Add(strings.ReplaceAll(payload, "{FILE}", f.Name()), f.Name())
+
+	}
+	f.Fuzz(func(t *testing.T, fuzzPath string, targetPath string) {
+		cleaned, err := CleanPath(fuzzPath)
+		if err != nil {
+			// Ignore errors
+			return
+		}
+		if cleaned != targetPath {
+			// cleaned path is different from target file
+			// so verify if 'path' is actually valid and not random chars
+			result, err := SafeOpen(cleaned)
+			if err != nil {
+				// Ignore errors
+				return
+			}
+			defer result.Close()
+			bin, _ := io.ReadAll(result)
+			if string(bin) == "pwned!" {
+				t.Fatalf("pwned! cleaned=%s ,input=%s", cleaned, fuzzPath)
+			}
+		}
+
+	})
+}

file/file_test.go

@@ -56,8 +56,8 @@ func TestDeleteFilesOlderThan(t *testing.T) {
 	// create a temporary folder with a couple of files
 	fo, err := os.MkdirTemp("", "")
 	require.Nil(t, err, "couldn't create folder: %s", err)
-	ttl := time.Duration(5 * time.Second)
-	sleepTime := time.Duration(10 * time.Second)
+	ttl := time.Duration(1 * time.Second)
+	sleepTime := time.Duration(3 * time.Second)
 
 	// defer temporary folder removal
 	defer os.RemoveAll(fo)