close #24

README.md

@@ -24,17 +24,46 @@ program displays all the titles and descriptions of the available sites.
 ### [golang/cmd/go](https://golang.org/cmd/go/)
 
 ```bash
-go get github.com/digital-technology-agency/web-scan/pkg/models
-go get github.com/digital-technology-agency/web-scan/pkg/utils
+go get github.com/digital-technology-agency/web-scan
 ```
 
 ## Data store
 
 * sqlite - SQLite
 * jsoneachrow - Json Each row
 
-## Run on Linux
+## Generator type
+
+- simple - Simple generator type
+
+## Init configuration file
+
+```bash
+$ ./wscan init
+
+```
+
+---
+
+#### Configuration file `config.json`
+
+```json
+{
+  "process_count": 1,
+  "alphabet": "abcdefgefghijklmnop",
+  "url_len": 5,
+  "concurrency_count": 5,
+  "data_store_type": "sqlite",
+  "generator_type": "simple",
+  "protocol_types": [
+    "http",
+    "https"
+  ]
+}
+```
+
+## Run
 
 ```bash
-$ ./wscan -alphabet "abcdef" -len 2 -data_store "sqlite" -process_count 1 -concurrency 5
+$ ./wscan -configuration_file config.json
 ```

cmd/distr/wscan.go

@@ -3,69 +3,66 @@ package main
 import (
 	"flag"
 	"fmt"
-	"github.com/digital-technology-agency/web-scan/pkg/env"
+	"github.com/digital-technology-agency/web-scan/pkg/config"
+	"github.com/digital-technology-agency/web-scan/pkg/database"
 	"github.com/digital-technology-agency/web-scan/pkg/models"
-	"github.com/digital-technology-agency/web-scan/pkg/services/generators"
-	"github.com/digital-technology-agency/web-scan/pkg/services/json"
 	"github.com/digital-technology-agency/web-scan/pkg/services/page"
-	"github.com/digital-technology-agency/web-scan/pkg/utils"
 	"github.com/zenthangplus/goccm"
 	"os"
 	"runtime"
 )
 
 var (
-	processCount     = flag.String(`process_count`, "1", `Example 1`)
-	alphabet         = flag.String(`alphabet`, "", `Example abcdefg`)
-	urlLen           = flag.String(`len`, "", `Example 2`)
-	concurrencyCount = flag.String(`concurrency`, "5", `Example 5`)
-	dataStore        = flag.String(`data_store`, env.SQLITE_STORE, "Example:\n"+
-		fmt.Sprintf("%s\n", env.SQLITE_STORE)+
-		fmt.Sprintf("%s\n", env.JSON_EACH_ROW_STORE)+
-		"",
-	)
-	protocols = []string{env.HTTP_PROTOCOL, env.HTTPS_PROTOCOL}
-	/*services*/
-	protocolWriters = map[string]*json.EachRowWriter{}
-	dbStore         = env.InitDbStore()
+	configurationFile = flag.String(`configuration_file`, "", `Example: config.json`)
+	configuration     = config.Default()
 )
 
 func main() {
-	flag.Parse()
-	/*check flags*/
-	if *alphabet == "" && *urlLen == "" {
-		flag.PrintDefaults()
-		return
+	if len(os.Args) > 1 {
+		if os.Args[1] == "init" {
+			err := configuration.Save("config.json")
+			if err != nil {
+				fmt.Printf("Configuration file not save! err:[%s]\n", err.Error())
+				os.Exit(-1)
+			}
+			os.Exit(0)
+		}
 	}
-	if !env.CheckStore(*dataStore) {
-		fmt.Printf("Store [%s] - not found!\n", *dataStore)
-		flag.PrintDefaults()
-		return
+	flag.Parse()
+	if *configurationFile != "" {
+		loadConfig, err := config.Load(*configurationFile)
+		if err != nil {
+			fmt.Printf("Configuration file not correct! err:[%s]\n", err.Error())
+			os.Exit(-1)
+		}
+		/*validate config*/
+		err = loadConfig.Validate()
+		if err != nil {
+			fmt.Printf("Configuration file not correct! err:[%s]\n", err.Error())
+			os.Exit(-1)
+		}
+		configuration = *loadConfig
 	}
-	runtime.GOMAXPROCS(utils.Int(*processCount))
-	cuncurency := goccm.New(utils.Int(*concurrencyCount))
+	configuration.InitGenerator()
+	configuration.InitDataStore()
+	runtime.GOMAXPROCS(configuration.ProcessCount)
+	cuncurency := goccm.New(configuration.ConcurrencyCount)
 	total := 0
 	domenNames := 0
-	gen := generators.SimpleGenerator{
-		Alphabet: *alphabet,
-		Len:      utils.Int(*urlLen),
-	}
+	gen := configuration.Generator
+	protocols := configuration.ProtocolTypes
 	model := models.Page{}
 	/*check db service*/
-	if env.CheckDataBaseStore(*dataStore) {
-		err := model.CreateTable(dbStore[*dataStore])
-		if err != nil {
-			fmt.Printf("Db service, create table! err:[%s]\n", err.Error())
-			os.Exit(-1)
-		}
-	} else {
-		protocolWriters = json.NewEachRowWriters(protocols)
+	err := model.CreateTable(configuration.DataStore)
+	if err != nil {
+		fmt.Printf("Db service, create table! err:[%s]\n", err.Error())
+		os.Exit(-1)
 	}
 	for domenName := range gen.Gen() {
 		cuncurency.Wait()
 		total += 1
 		for _, protokol := range protocols {
-			go func(protokol, domen string) {
+			go func(protokol, domen string, dataStore database.DbService) {
 				defer cuncurency.Done()
 				url := fmt.Sprintf("%s://%s.ru", protokol, domen)
 				pageService := page.PageService{
@@ -79,20 +76,13 @@ func main() {
 					fmt.Printf("Page is nil\n")
 					return
 				}
-				switch *dataStore {
-				default:
-					fmt.Printf("Store [%s] - not found!\n", *dataStore)
-				case env.JSON_EACH_ROW_STORE:
-					err = protocolWriters[protokol].WriteLine(item)
-				case env.SQLITE_STORE:
-					err = item.AddOrUpdate(dbStore[env.SQLITE_STORE])
-				}
+				err = item.AddOrUpdate(dataStore)
 				if err != nil {
 					fmt.Printf("Write line to service! Err:[%s]\n", err.Error())
 					return
 				}
 				domenNames += 1
-			}(protokol, domenName)
+			}(protokol, domenName, configuration.DataStore)
 		}
 	}
 	cuncurency.WaitAllDone()

config.json

@@ -0,0 +1,12 @@
+{
+  "process_count": 1,
+  "alphabet": "abcdefgefghijklmnop",
+  "url_len": 5,
+  "concurrency_count": 5,
+  "data_store_type": "sqlite",
+  "generator_type": "simple",
+  "protocol_types": [
+    "http",
+    "https"
+  ]
+}

pkg/config/configuration.go

@@ -2,21 +2,26 @@ package config
 
 import (
 	"encoding/json"
+	"errors"
 	"github.com/digital-technology-agency/web-scan/pkg/database"
 	"github.com/digital-technology-agency/web-scan/pkg/database/sqlite"
 	"github.com/digital-technology-agency/web-scan/pkg/env"
+	generators "github.com/digital-technology-agency/web-scan/pkg/services/generators"
 	"io/ioutil"
+	"strings"
 )
 
 // Configuration configuration type
 type Configuration struct {
-	ProcessCount     int                `json:"process_count"`
-	Alphabet         string             `json:"alphabet"`
-	UrlLen           int                `json:"url_len"`
-	ConcurrencyCount int                `json:"concurrency_count"`
-	DataStoreType    string             `json:"data_store_type"`
-	DataStore        database.DbService `json:"-"`
-	ProtocolTypes    []string           `json:"protocol_types"`
+	ProcessCount     int                  `json:"process_count"`
+	Alphabet         string               `json:"alphabet"`
+	UrlLen           int                  `json:"url_len"`
+	ConcurrencyCount int                  `json:"concurrency_count"`
+	DataStoreType    string               `json:"data_store_type"`
+	DataStore        database.DbService   `json:"-"`
+	GeneratorType    string               `json:"generator_type"`
+	Generator        generators.Generator `json:"-"`
+	ProtocolTypes    []string             `json:"protocol_types"`
 }
 
 // Default default configuration
@@ -28,7 +33,12 @@ func Default() Configuration {
 		ConcurrencyCount: 5,
 		DataStoreType:    env.SQLITE_STORE,
 		DataStore:        sqlite.SqLite{},
-		ProtocolTypes:    []string{env.HTTP_PROTOCOL, env.HTTPS_PROTOCOL},
+		GeneratorType:    env.SIMPLE_GENERATOR,
+		Generator: &generators.SimpleGenerator{
+			Alphabet: "abcdefg",
+			Len:      2,
+		},
+		ProtocolTypes: []string{env.HTTP_PROTOCOL, env.HTTPS_PROTOCOL},
 	}
 }
 
@@ -46,6 +56,41 @@ func Load(path string) (*Configuration, error) {
 	return &result, nil
 }
 
+// Validate validate property params
+func (cfg Configuration) Validate() error {
+	if cfg.ProcessCount <= 0 {
+		return errors.New("Поле [process_count] - должно быть больше 0!")
+	}
+	if strings.TrimSpace(strings.ToLower(cfg.Alphabet)) == "" {
+		return errors.New("Поле [alphabet] - должно содержать набор символов!")
+	}
+	if cfg.UrlLen <= 0 {
+		return errors.New("Поле [url_len] - должно быть больше 0!")
+	}
+	if cfg.ConcurrencyCount <= 0 {
+		return errors.New("Поле [concurrency_count] - должно быть больше 0!")
+	}
+	if !env.CheckStore(cfg.DataStoreType) {
+		return errors.New("Поле [data_store_type] - должно содержать значение из предложенных вариантов!")
+	}
+	if !env.CheckGenerator(cfg.GeneratorType) {
+		return errors.New("Поле [generator_type] - должно содержать значение из предложенных вариантов!")
+	}
+	if len(cfg.ProtocolTypes) == 0 {
+		return errors.New("Поле [protocol_types] - должно содержать одно или несколько занчение из предложенных вариантов!")
+	}
+	for _, protocolType := range cfg.ProtocolTypes {
+		switch protocolType {
+		default:
+			return errors.New("Поле [protocol_types] - должно содержать одно или несколько занчение из предложенных вариантов!")
+		case env.HTTP_PROTOCOL:
+		case env.HTTPS_PROTOCOL:
+			continue
+		}
+	}
+	return nil
+}
+
 // Save configuration save to file
 func (cfg Configuration) Save(path string) error {
 	bytes, err := json.Marshal(cfg)
@@ -56,6 +101,22 @@ func (cfg Configuration) Save(path string) error {
 	return err
 }
 
+// InitGenerator init generator from type
+func (cfg *Configuration) InitGenerator() {
+	switch cfg.GeneratorType {
+	default:
+		cfg.Generator = &generators.SimpleGenerator{
+			Alphabet: cfg.Alphabet,
+			Len:      cfg.UrlLen,
+		}
+	case env.SIMPLE_GENERATOR:
+		cfg.Generator = &generators.SimpleGenerator{
+			Alphabet: cfg.Alphabet,
+			Len:      cfg.UrlLen,
+		}
+	}
+}
+
 // InitDataStore init data store from type
 func (cfg *Configuration) InitDataStore() {
 	switch cfg.DataStoreType {

pkg/database/base-service.go

@@ -2,11 +2,13 @@ package database
 
 import "github.com/jmoiron/sqlx"
 
+// DbService database interface
 type DbService interface {
 	Connect() (*sqlx.DB, error)
 	Execute(query string) error
 }
 
+// DbEntity entity interface
 type DbEntity interface {
 	GetTableName() string
 	CreateTable(service DbService) error

pkg/database/sqlite/client.go

@@ -5,6 +5,7 @@ import (
 	_ "github.com/mattn/go-sqlite3"
 )
 
+// SqLite sqlite client
 type SqLite struct {
 	db *sqlx.DB
 }

pkg/env/generator-env.go

@@ -0,0 +1,16 @@
+package env
+
+var (
+	SIMPLE_GENERATOR = `simple`
+)
+
+// CheckGenerator validate generator type
+func CheckGenerator(input string) bool {
+	switch input {
+	default:
+		return false
+	case
+		SIMPLE_GENERATOR:
+		return true
+	}
+}

pkg/env/store-env.go

@@ -1,22 +1,11 @@
 package env
 
-import (
-	"github.com/digital-technology-agency/web-scan/pkg/database"
-	"github.com/digital-technology-agency/web-scan/pkg/database/sqlite"
-)
-
 const (
 	SQLITE_STORE        = `sqlite`
 	JSON_EACH_ROW_STORE = `jsoneachrow`
 )
 
-func InitDbStore() map[string]database.DbService {
-	result := map[string]database.DbService{
-		SQLITE_STORE: sqlite.SqLite{},
-	}
-	return result
-}
-
+// CheckStore validate store type
 func CheckStore(input string) bool {
 	switch input {
 	default:
@@ -27,12 +16,3 @@ func CheckStore(input string) bool {
 		return true
 	}
 }
-
-func CheckDataBaseStore(input string) bool {
-	switch input {
-	default:
-		return false
-	case SQLITE_STORE:
-		return true
-	}
-}