Initial version

.gitignore

@@ -1,3 +1,43 @@
+# Created by .ignore support plugin (hsz.mobi)
+### Linux template
+*~
+
+# temporary files which can be created if a process still has a handle open of a deleted file
+.fuse_hidden*
+
+# KDE directory preferences
+.directory
+
+# Linux trash folder which might appear on any partition or disk
+.Trash-*
+
+# .nfs files are created when an open file is removed but is still being accessed
+.nfs*
+### Windows template
+# Windows thumbnail cache files
+Thumbs.db
+ehthumbs.db
+ehthumbs_vista.db
+
+# Dump file
+*.stackdump
+
+# Folder config file
+[Dd]esktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk
+### Go template
 # Binaries for programs and plugins
 *.exe
 *.exe~
@@ -7,6 +47,67 @@
 
 # Test binary, build with `go test -c`
 *.test
+/vendor
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+### Java template
+# Compiled class file
+*.class
+
+# Log file
+*.log
+
+# BlueJ files
+*.ctxt
+
+# Mobile Tools for Java (J2ME)
+.mtj.tmp/
+
+# Package Files #
+*.jar
+*.war
+*.nar
+*.ear
+*.zip
+*.tar.gz
+*.rar
+
+# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*
+### JetBrains template
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests

.travis.yml

@@ -0,0 +1,9 @@
+language: go
+
+go:
+- 1.10.x
+- 1.x
+
+script:
+- go build
+- go test -race -v ./...

README.md

@@ -1,2 +1,17 @@
 # zipred
-Golang library to filter and download files from within an online zip file on the fly
+
+[![Build Status](https://travis-ci.org/gofunky/zipred.svg)](https://travis-ci.org/gofunky/zipred)
+[![GoDoc](https://godoc.org/github.com/gofunky/zipred?status.svg)](https://godoc.org/github.com/gofunky/zipred)
+[![Go Report Card](https://goreportcard.com/badge/github.com/gofunky/zipred)](https://goreportcard.com/report/github.com/gofunky/zipred)
+[![Codacy Badge](https://api.codacy.com/project/badge/Grade/7664447e93c742219959e310a1d3f2d9)](https://www.codacy.com/app/gofunky/zipred?utm_source=github.com&utm_medium=referral&utm_content=gofunky/zipred&utm_campaign=Badge_Grade)
+
+ZIP file operations can get costly, especially for large files. This library allows you to filter and extract an online zip file on the fly.
+
+In contrast to a conventional zip parser, it has the following benefits: 
+* There is less latency since data is processed directly from the buffer on the fly.
+* The download can be stopped once the metadata or target file has been found. Hence, less data is transferred.
+* Irrelevant data is directly discarded without memory allocation.
+
+This library gives you an efficient and idiomatic way for indexing zip files on the web.
+
+For examples, check the corresponding folder.

examples/gitignore/get.go

@@ -0,0 +1,70 @@
+package gitignore
+
+import (
+	"errors"
+	"github.com/gofunky/zipred"
+	"os"
+	"strings"
+)
+
+// gitIgnoreContent is the context that implements zipred.Zipred.
+type gitIgnoreContent struct {
+	// patterns to filter
+	patterns []string
+	// number of patterns found
+	count int
+}
+
+// URL to download the archive from
+func (c *gitIgnoreContent) URL() string {
+	return archiveURL
+}
+
+// Predicate indicates if the given file should be read or not.
+// It is to return a zero string to discard the file, otherwise the key name.
+// If error is nonempty, the download is aborted and the error is passed on.
+func (c *gitIgnoreContent) Predicate(fileInfo os.FileInfo) (key string, err error) {
+	fileName := fileInfo.Name()
+	if strings.HasSuffix(fileName, gitignoreSuffix) {
+		alias := strings.ToLower(strings.TrimSuffix(fileName, gitignoreSuffix))
+		for _, pat := range c.patterns {
+			if strings.ToLower(pat) == alias {
+				c.count++
+				return alias, nil
+			}
+		}
+	}
+	return
+}
+
+// Done indicates if enough data has been read and the download can be aborted ahead of the EOF.
+// isEOF is true if the end of the zip file has been reached.
+// If error is nonempty, the download is aborted and the error is passed on.
+func (c *gitIgnoreContent) Done(isEOF bool) (finish bool, err error) {
+	if c.count == len(c.patterns) {
+		return true, nil
+	} else if isEOF {
+		return true, errors.New("not all given gitignore patterns could be found")
+	}
+	return
+}
+
+// Get the given gitignore patterns.
+func Get(patterns []string) (files map[string][]byte, err error) {
+	context := &gitIgnoreContent{
+		patterns: patterns,
+	}
+	return zipred.FilterZipContent(context)
+}
+
+// GetAll available gitignore patterns.
+func GetAll() (files map[string][]byte, err error) {
+	allPatterns, err := List()
+	if err != nil {
+		return nil, err
+	}
+	context := &gitIgnoreContent{
+		patterns: allPatterns,
+	}
+	return zipred.FilterZipContent(context)
+}

examples/gitignore/get_test.go

@@ -0,0 +1,73 @@
+package gitignore
+
+import (
+	"testing"
+)
+
+func TestGet(t *testing.T) {
+	type args struct {
+		patterns []string
+	}
+	tests := []struct {
+		name      string
+		args      args
+		wantFiles []string
+		wantErr   bool
+	}{
+		{
+			name:      "Check some patterns",
+			args:      args{[]string{"go", "java"}},
+			wantFiles: []string{"go", "java"},
+		},
+		{
+			name:    "Invalid patterns",
+			args:    args{[]string{"invalid"}},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotFiles, err := Get(tt.args.patterns)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Get() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !tt.wantErr {
+				for _, pattern := range tt.wantFiles {
+					if val, ok := gotFiles[pattern]; !ok || len(val) == 0 {
+						t.Errorf("Get() is missing key %s or it is empty", pattern)
+					}
+				}
+			}
+		})
+	}
+}
+
+func TestGetAll(t *testing.T) {
+	tests := []struct {
+		name      string
+		wantFiles []string
+		wantErr   bool
+	}{
+		{
+			name:      "Check some patterns",
+			wantFiles: []string{"go", "java"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotFiles, err := GetAll()
+			if (err != nil) != tt.wantErr {
+				t.Errorf("GetAll() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !tt.wantErr {
+				for _, pattern := range tt.wantFiles {
+					if val, ok := gotFiles[pattern]; !ok || len(val) == 0 {
+						t.Errorf("GetAll() is missing key %s or it is empty", pattern)
+					}
+				}
+			}
+		})
+	}
+}

examples/gitignore/gitignore.go

@@ -0,0 +1,6 @@
+package gitignore
+
+const (
+	gitignoreSuffix = ".gitignore"
+	archiveURL      = "https://github.com/dvcs/gitignore/archive/master.zip"
+)

examples/gitignore/list.go

@@ -0,0 +1,38 @@
+package gitignore
+
+import (
+	"github.com/gofunky/zipred"
+	"os"
+	"strings"
+)
+
+// gitIgnoreList is the context that implements zipred.Zipred.
+type gitIgnoreList struct{}
+
+// URL to download the archive from
+func (c *gitIgnoreList) URL() string {
+	return archiveURL
+}
+
+// Predicate indicates if the given file should be read or not.
+// It is to return a zero string to discard the file, otherwise the key name.
+// If error is nonempty, the download is aborted and the error is passed on.
+func (c *gitIgnoreList) Predicate(fileInfo os.FileInfo) (key string, err error) {
+	fileName := fileInfo.Name()
+	if strings.HasSuffix(fileName, gitignoreSuffix) {
+		return strings.ToLower(strings.TrimSuffix(fileName, gitignoreSuffix)), nil
+	}
+	return
+}
+
+// Done indicates if enough data has been read and the download can be aborted ahead of the EOF.
+// isEOF is true if the end of the zip file has been reached.
+// If error is nonempty, the download is aborted and the error is passed on.
+func (c *gitIgnoreList) Done(isEOF bool) (finish bool, err error) {
+	return
+}
+
+// List all available gitignore patterns.
+func List() (patterns []string, err error) {
+	return zipred.FilterFileInfo(&gitIgnoreList{})
+}

examples/gitignore/list_test.go

@@ -0,0 +1,41 @@
+package gitignore
+
+import (
+	"testing"
+)
+
+func TestList(t *testing.T) {
+	tests := []struct {
+		name         string
+		wantPatterns []string
+		wantErr      bool
+	}{
+		{
+			name:         "Check some patterns",
+			wantPatterns: []string{"go", "java"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotPatterns, err := List()
+			if (err != nil) != tt.wantErr {
+				t.Errorf("List() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for _, want := range tt.wantPatterns {
+				if !contains(gotPatterns, want) {
+					t.Errorf("List() = %v, want %v", gotPatterns, tt.wantPatterns)
+				}
+			}
+		})
+	}
+}
+
+func contains(s []string, e string) bool {
+	for _, a := range s {
+		if a == e {
+			return true
+		}
+	}
+	return false
+}