From db75f9e237ddafb20ef6990eab75b8dea2d557f9 Mon Sep 17 00:00:00 2001 From: Punith C K Date: Tue, 3 Sep 2024 17:05:40 +0530 Subject: [PATCH 1/4] fix: Conftest encounters errors on Windows when loading file paths that include drive letters (e.g., `C:/path/to/data.yaml`). Even when using a file URL (e.g., `file:///C:/path/to/data.yaml`), we still face issues. With these code changes, Conftest can now successfully load files using a file URL (e.g., `file:///C:/path/to/data.yaml`). We opted for file URLs instead of paths with drive letters (e.g., `C:/path/to/data.yaml`) because OPA does not support file paths with drive letters. For more details, see [this issue comment](https://github.com/open-policy-agent/opa/pull/6922#issuecomment-2312969000). Resolves: https://github.com/open-policy-agent/conftest/issues/979 Signed-off-by: Punith C K --- policy/engine.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/policy/engine.go b/policy/engine.go index 2fcf16aabf..84b9500835 100644 --- a/policy/engine.go +++ b/policy/engine.go @@ -134,7 +134,9 @@ func LoadWithData(policyPaths []string, dataPaths []string, capabilities string, return nil, fmt.Errorf("filter data paths: %w", err) } - documents, err := loader.NewFileLoader().All(allDocumentPaths) + documents, err := loader.NewFileLoader().WithProcessAnnotation(true).Filtered(dataPaths, func(_ string, info os.FileInfo, _ int) bool { + return !info.IsDir() && !contains([]string{".yaml", ".yml", ".json"}, filepath.Ext(info.Name())) + }) if err != nil { return nil, fmt.Errorf("load documents: %w", err) } From 13619ee47976018b8ea37a81185e5b477e2dd556 Mon Sep 17 00:00:00 2001 From: Punith C K Date: Thu, 5 Sep 2024 16:21:26 +0530 Subject: [PATCH 2/4] fix: Conftest can now successfully load files using a file URL (e.g., file:///C:/path/to/data.yaml) on windows Conftest encounters errors on Windows when loading file paths that include drive letters (e.g., C:/path/to/data.yaml). Even when using a file URL (e.g., file:///C:/path/to/data.yaml), we still face issues. We opted for file URLs(e.g., file:///C:/path/to/data.yaml) instead of paths with drive letters (e.g., C:/path/to/data.yaml) because OPA does not support file paths with drive letters. Resolves #979 Signed-off-by: Punith C K --- internal/commands/push.go | 2 +- policy/engine.go | 30 +++++------------------------- 2 files changed, 6 insertions(+), 26 deletions(-) diff --git a/internal/commands/push.go b/internal/commands/push.go index e84f126469..7213b3df6b 100644 --- a/internal/commands/push.go +++ b/internal/commands/push.go @@ -202,7 +202,7 @@ func pushLayers(ctx context.Context, pusher content.Pusher, policyPath, dataPath } for path, contents := range engine.Documents() { - data := []byte(contents) + data := []byte(contents.(string)) desc := content.NewDescriptorFromBytes(openPolicyAgentDataLayerMediaType, data) desc.Annotations = map[string]string{ ocispec.AnnotationTitle: path, diff --git a/policy/engine.go b/policy/engine.go index 84b9500835..997db16d23 100644 --- a/policy/engine.go +++ b/policy/engine.go @@ -31,7 +31,7 @@ type Engine struct { compiler *ast.Compiler store storage.Store policies map[string]string - docs map[string]string + docs map[string]any } type compilerOptions struct { @@ -121,22 +121,14 @@ func LoadWithData(policyPaths []string, dataPaths []string, capabilities string, return nil, fmt.Errorf("loading policies: %w", err) } } - - // FilteredPaths will recursively find all file paths that contain a valid document - // extension from the given list of data paths. - allDocumentPaths, err := loader.FilteredPaths(dataPaths, func(_ string, info os.FileInfo, _ int) bool { + filter := func(_ string, info os.FileInfo, _ int) bool { if info.IsDir() { return false } return !contains([]string{".yaml", ".yml", ".json"}, filepath.Ext(info.Name())) - }) - if err != nil { - return nil, fmt.Errorf("filter data paths: %w", err) } - documents, err := loader.NewFileLoader().WithProcessAnnotation(true).Filtered(dataPaths, func(_ string, info os.FileInfo, _ int) bool { - return !info.IsDir() && !contains([]string{".yaml", ".yml", ".json"}, filepath.Ext(info.Name())) - }) + documents, err := loader.NewFileLoader().Filtered(dataPaths, filter) if err != nil { return nil, fmt.Errorf("load documents: %w", err) } @@ -145,20 +137,8 @@ func LoadWithData(policyPaths []string, dataPaths []string, capabilities string, return nil, fmt.Errorf("get documents store: %w", err) } - documentContents := make(map[string]string) - for _, documentPath := range allDocumentPaths { - contents, err := os.ReadFile(documentPath) - if err != nil { - return nil, fmt.Errorf("read file: %w", err) - } - - documentPath = filepath.Clean(documentPath) - documentPath = filepath.ToSlash(documentPath) - documentContents[documentPath] = string(contents) - } - engine.store = store - engine.docs = documentContents + engine.docs = documents.Documents return engine, nil } @@ -244,7 +224,7 @@ func (e *Engine) Namespaces() []string { // Documents returns all of the documents loaded into the engine. // The result is a map where the key is the filepath of the document // and its value is the raw contents of the loaded document. -func (e *Engine) Documents() map[string]string { +func (e *Engine) Documents() map[string]interface{} { return e.docs } From 6c807d124cdc9e5bdaa93590863771fd06cabf07 Mon Sep 17 00:00:00 2001 From: Punith C K Date: Thu, 17 Oct 2024 20:33:11 +0530 Subject: [PATCH 3/4] fix: using any instead of interface{} as per comment https://github.com/open-policy-agent/conftest/pull/999#discussion_r1804024799 Signed-off-by: Punith C K --- policy/engine.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/policy/engine.go b/policy/engine.go index 997db16d23..f79e8d5762 100644 --- a/policy/engine.go +++ b/policy/engine.go @@ -224,7 +224,7 @@ func (e *Engine) Namespaces() []string { // Documents returns all of the documents loaded into the engine. // The result is a map where the key is the filepath of the document // and its value is the raw contents of the loaded document. -func (e *Engine) Documents() map[string]interface{} { +func (e *Engine) Documents() map[string]any { return e.docs } From e743c08ae2c65b99c453ff8d566a79a710997369 Mon Sep 17 00:00:00 2001 From: Punith C K Date: Mon, 28 Oct 2024 18:54:58 +0530 Subject: [PATCH 4/4] fix: using interface{} instead of any. Because we are getting an error in testcase after using type any instead of interface{} https://github.com/open-policy-agent/conftest/actions/runs/11387696959/job/31682907280?pr=1008 Signed-off-by: Punith C K --- policy/engine.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/policy/engine.go b/policy/engine.go index f79e8d5762..62e40c608f 100644 --- a/policy/engine.go +++ b/policy/engine.go @@ -31,7 +31,7 @@ type Engine struct { compiler *ast.Compiler store storage.Store policies map[string]string - docs map[string]any + docs map[string]interface{} } type compilerOptions struct { @@ -224,7 +224,7 @@ func (e *Engine) Namespaces() []string { // Documents returns all of the documents loaded into the engine. // The result is a map where the key is the filepath of the document // and its value is the raw contents of the loaded document. -func (e *Engine) Documents() map[string]any { +func (e *Engine) Documents() map[string]interface{} { return e.docs }