diff --git a/.idea/dataSources.xml b/.idea/dataSources.xml index 148dd2f..c5ccaf5 100644 --- a/.idea/dataSources.xml +++ b/.idea/dataSources.xml @@ -1,7 +1,7 @@ - + sqlite.xerial true org.sqlite.JDBC diff --git a/README.md b/README.md index 1c9bf47..e37e379 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,7 @@ program displays all the titles and descriptions of the available sites. ### [golang/cmd/go](https://golang.org/cmd/go/) ```bash -go get github.com/digital-technology-agency/web-scan/pkg/models -go get github.com/digital-technology-agency/web-scan/pkg/utils +go get github.com/digital-technology-agency/web-scan ``` ## Data store @@ -33,8 +32,38 @@ go get github.com/digital-technology-agency/web-scan/pkg/utils * sqlite - SQLite * jsoneachrow - Json Each row -## Run on Linux +## Generator type + +- simple - Simple generator type + +## Init configuration file + +```bash +$ ./wscan init + +``` + +--- + +#### Configuration file `config.json` + +```json +{ + "process_count": 1, + "alphabet": "abcdefgefghijklmnop", + "url_len": 5, + "concurrency_count": 5, + "data_store_type": "sqlite", + "generator_type": "simple", + "protocol_types": [ + "http", + "https" + ] +} +``` + +## Run ```bash -$ ./wscan -alphabet "abcdef" -len 2 -data_store "sqlite" -process_count 1 -concurrency 5 +$ ./wscan -configuration_file config.json ``` diff --git a/cmd/distr/wscan.go b/cmd/distr/wscan.go index f1c2947..22bbc1d 100644 --- a/cmd/distr/wscan.go +++ b/cmd/distr/wscan.go @@ -3,69 +3,66 @@ package main import ( "flag" "fmt" - "github.com/digital-technology-agency/web-scan/pkg/env" + "github.com/digital-technology-agency/web-scan/pkg/config" + "github.com/digital-technology-agency/web-scan/pkg/database" "github.com/digital-technology-agency/web-scan/pkg/models" - "github.com/digital-technology-agency/web-scan/pkg/services/generators" - "github.com/digital-technology-agency/web-scan/pkg/services/json" "github.com/digital-technology-agency/web-scan/pkg/services/page" - "github.com/digital-technology-agency/web-scan/pkg/utils" "github.com/zenthangplus/goccm" "os" "runtime" ) var ( - processCount = flag.String(`process_count`, "1", `Example 1`) - alphabet = flag.String(`alphabet`, "", `Example abcdefg`) - urlLen = flag.String(`len`, "", `Example 2`) - concurrencyCount = flag.String(`concurrency`, "5", `Example 5`) - dataStore = flag.String(`data_store`, env.SQLITE_STORE, "Example:\n"+ - fmt.Sprintf("%s\n", env.SQLITE_STORE)+ - fmt.Sprintf("%s\n", env.JSON_EACH_ROW_STORE)+ - "", - ) - protocols = []string{env.HTTP_PROTOCOL, env.HTTPS_PROTOCOL} - /*services*/ - protocolWriters = map[string]*json.EachRowWriter{} - dbStore = env.InitDbStore() + configurationFile = flag.String(`configuration_file`, "", `Example: config.json`) + configuration = config.Default() ) func main() { - flag.Parse() - /*check flags*/ - if *alphabet == "" && *urlLen == "" { - flag.PrintDefaults() - return + if len(os.Args) > 1 { + if os.Args[1] == "init" { + err := configuration.Save("config.json") + if err != nil { + fmt.Printf("Configuration file not save! err:[%s]\n", err.Error()) + os.Exit(-1) + } + os.Exit(0) + } } - if !env.CheckStore(*dataStore) { - fmt.Printf("Store [%s] - not found!\n", *dataStore) - flag.PrintDefaults() - return + flag.Parse() + if *configurationFile != "" { + loadConfig, err := config.Load(*configurationFile) + if err != nil { + fmt.Printf("Configuration file not correct! err:[%s]\n", err.Error()) + os.Exit(-1) + } + /*validate config*/ + err = loadConfig.Validate() + if err != nil { + fmt.Printf("Configuration file not correct! err:[%s]\n", err.Error()) + os.Exit(-1) + } + configuration = *loadConfig } - runtime.GOMAXPROCS(utils.Int(*processCount)) - cuncurency := goccm.New(utils.Int(*concurrencyCount)) + configuration.InitGenerator() + configuration.InitDataStore() + runtime.GOMAXPROCS(configuration.ProcessCount) + cuncurency := goccm.New(configuration.ConcurrencyCount) total := 0 domenNames := 0 - gen := generators.SimpleGenerator{ - Alphabet: *alphabet, - Len: utils.Int(*urlLen), - } + gen := configuration.Generator + protocols := configuration.ProtocolTypes model := models.Page{} /*check db service*/ - if env.CheckDataBaseStore(*dataStore) { - err := model.CreateTable(dbStore[*dataStore]) - if err != nil { - fmt.Printf("Db service, create table! err:[%s]\n", err.Error()) - os.Exit(-1) - } - } else { - protocolWriters = json.NewEachRowWriters(protocols) + err := model.CreateTable(configuration.DataStore) + if err != nil { + fmt.Printf("Db service, create table! err:[%s]\n", err.Error()) + os.Exit(-1) } for domenName := range gen.Gen() { cuncurency.Wait() total += 1 for _, protokol := range protocols { - go func(protokol, domen string) { + go func(protokol, domen string, dataStore database.DbService) { defer cuncurency.Done() url := fmt.Sprintf("%s://%s.ru", protokol, domen) pageService := page.PageService{ @@ -79,20 +76,13 @@ func main() { fmt.Printf("Page is nil\n") return } - switch *dataStore { - default: - fmt.Printf("Store [%s] - not found!\n", *dataStore) - case env.JSON_EACH_ROW_STORE: - err = protocolWriters[protokol].WriteLine(item) - case env.SQLITE_STORE: - err = item.AddOrUpdate(dbStore[env.SQLITE_STORE]) - } + err = item.AddOrUpdate(dataStore) if err != nil { fmt.Printf("Write line to service! Err:[%s]\n", err.Error()) return } domenNames += 1 - }(protokol, domenName) + }(protokol, domenName, configuration.DataStore) } } cuncurency.WaitAllDone() diff --git a/config.json b/config.json new file mode 100644 index 0000000..6b14397 --- /dev/null +++ b/config.json @@ -0,0 +1,12 @@ +{ + "process_count": 1, + "alphabet": "abcdefgefghijklmnop", + "url_len": 5, + "concurrency_count": 5, + "data_store_type": "sqlite", + "generator_type": "simple", + "protocol_types": [ + "http", + "https" + ] +} diff --git a/pkg/config/configuration.go b/pkg/config/configuration.go index b1cdf19..d3ec55b 100644 --- a/pkg/config/configuration.go +++ b/pkg/config/configuration.go @@ -2,21 +2,26 @@ package config import ( "encoding/json" + "errors" "github.com/digital-technology-agency/web-scan/pkg/database" "github.com/digital-technology-agency/web-scan/pkg/database/sqlite" "github.com/digital-technology-agency/web-scan/pkg/env" + generators "github.com/digital-technology-agency/web-scan/pkg/services/generators" "io/ioutil" + "strings" ) // Configuration configuration type type Configuration struct { - ProcessCount int `json:"process_count"` - Alphabet string `json:"alphabet"` - UrlLen int `json:"url_len"` - ConcurrencyCount int `json:"concurrency_count"` - DataStoreType string `json:"data_store_type"` - DataStore database.DbService `json:"-"` - ProtocolTypes []string `json:"protocol_types"` + ProcessCount int `json:"process_count"` + Alphabet string `json:"alphabet"` + UrlLen int `json:"url_len"` + ConcurrencyCount int `json:"concurrency_count"` + DataStoreType string `json:"data_store_type"` + DataStore database.DbService `json:"-"` + GeneratorType string `json:"generator_type"` + Generator generators.Generator `json:"-"` + ProtocolTypes []string `json:"protocol_types"` } // Default default configuration @@ -28,7 +33,12 @@ func Default() Configuration { ConcurrencyCount: 5, DataStoreType: env.SQLITE_STORE, DataStore: sqlite.SqLite{}, - ProtocolTypes: []string{env.HTTP_PROTOCOL, env.HTTPS_PROTOCOL}, + GeneratorType: env.SIMPLE_GENERATOR, + Generator: &generators.SimpleGenerator{ + Alphabet: "abcdefg", + Len: 2, + }, + ProtocolTypes: []string{env.HTTP_PROTOCOL, env.HTTPS_PROTOCOL}, } } @@ -46,6 +56,41 @@ func Load(path string) (*Configuration, error) { return &result, nil } +// Validate validate property params +func (cfg Configuration) Validate() error { + if cfg.ProcessCount <= 0 { + return errors.New("Поле [process_count] - должно быть больше 0!") + } + if strings.TrimSpace(strings.ToLower(cfg.Alphabet)) == "" { + return errors.New("Поле [alphabet] - должно содержать набор символов!") + } + if cfg.UrlLen <= 0 { + return errors.New("Поле [url_len] - должно быть больше 0!") + } + if cfg.ConcurrencyCount <= 0 { + return errors.New("Поле [concurrency_count] - должно быть больше 0!") + } + if !env.CheckStore(cfg.DataStoreType) { + return errors.New("Поле [data_store_type] - должно содержать значение из предложенных вариантов!") + } + if !env.CheckGenerator(cfg.GeneratorType) { + return errors.New("Поле [generator_type] - должно содержать значение из предложенных вариантов!") + } + if len(cfg.ProtocolTypes) == 0 { + return errors.New("Поле [protocol_types] - должно содержать одно или несколько занчение из предложенных вариантов!") + } + for _, protocolType := range cfg.ProtocolTypes { + switch protocolType { + default: + return errors.New("Поле [protocol_types] - должно содержать одно или несколько занчение из предложенных вариантов!") + case env.HTTP_PROTOCOL: + case env.HTTPS_PROTOCOL: + continue + } + } + return nil +} + // Save configuration save to file func (cfg Configuration) Save(path string) error { bytes, err := json.Marshal(cfg) @@ -56,6 +101,22 @@ func (cfg Configuration) Save(path string) error { return err } +// InitGenerator init generator from type +func (cfg *Configuration) InitGenerator() { + switch cfg.GeneratorType { + default: + cfg.Generator = &generators.SimpleGenerator{ + Alphabet: cfg.Alphabet, + Len: cfg.UrlLen, + } + case env.SIMPLE_GENERATOR: + cfg.Generator = &generators.SimpleGenerator{ + Alphabet: cfg.Alphabet, + Len: cfg.UrlLen, + } + } +} + // InitDataStore init data store from type func (cfg *Configuration) InitDataStore() { switch cfg.DataStoreType { diff --git a/pkg/database/base-service.go b/pkg/database/base-service.go index fa39670..43575da 100644 --- a/pkg/database/base-service.go +++ b/pkg/database/base-service.go @@ -2,11 +2,13 @@ package database import "github.com/jmoiron/sqlx" +// DbService database interface type DbService interface { Connect() (*sqlx.DB, error) Execute(query string) error } +// DbEntity entity interface type DbEntity interface { GetTableName() string CreateTable(service DbService) error diff --git a/pkg/database/sqlite/client.go b/pkg/database/sqlite/client.go index 62a8208..1ab6c89 100644 --- a/pkg/database/sqlite/client.go +++ b/pkg/database/sqlite/client.go @@ -5,6 +5,7 @@ import ( _ "github.com/mattn/go-sqlite3" ) +// SqLite sqlite client type SqLite struct { db *sqlx.DB } diff --git a/pkg/env/generator-env.go b/pkg/env/generator-env.go new file mode 100644 index 0000000..bd4e5e4 --- /dev/null +++ b/pkg/env/generator-env.go @@ -0,0 +1,16 @@ +package env + +var ( + SIMPLE_GENERATOR = `simple` +) + +// CheckGenerator validate generator type +func CheckGenerator(input string) bool { + switch input { + default: + return false + case + SIMPLE_GENERATOR: + return true + } +} diff --git a/pkg/env/store-env.go b/pkg/env/store-env.go index d43e098..7a6d815 100644 --- a/pkg/env/store-env.go +++ b/pkg/env/store-env.go @@ -1,22 +1,11 @@ package env -import ( - "github.com/digital-technology-agency/web-scan/pkg/database" - "github.com/digital-technology-agency/web-scan/pkg/database/sqlite" -) - const ( SQLITE_STORE = `sqlite` JSON_EACH_ROW_STORE = `jsoneachrow` ) -func InitDbStore() map[string]database.DbService { - result := map[string]database.DbService{ - SQLITE_STORE: sqlite.SqLite{}, - } - return result -} - +// CheckStore validate store type func CheckStore(input string) bool { switch input { default: @@ -27,12 +16,3 @@ func CheckStore(input string) bool { return true } } - -func CheckDataBaseStore(input string) bool { - switch input { - default: - return false - case SQLITE_STORE: - return true - } -} diff --git a/pkg/models/page.go b/pkg/models/page.go index c874b17..b2d6363 100644 --- a/pkg/models/page.go +++ b/pkg/models/page.go @@ -17,10 +17,12 @@ type Page struct { Sitemap string `json:"sitemap" db:"sitemap"` } +// GetTableName get table name func (p Page) GetTableName() string { return PAGE_TABLE_NAME } +// CreateTable create table func (p Page) CreateTable(dbService database.DbService) error { query := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s ("+ "url TEXT PRIMARY KEY NOT NULL,"+ @@ -32,11 +34,13 @@ func (p Page) CreateTable(dbService database.DbService) error { return dbService.Execute(query) } +// DropTable drop table func (p Page) DropTable(dbService database.DbService) error { query := fmt.Sprintf("DROP TABLE IF EXISTS %s", PAGE_TABLE_NAME) return dbService.Execute(query) } +// SelectAll select all rows func (p Page) SelectAll(dbService database.DbService) ([]Page, error) { query := fmt.Sprintf("SELECT * FROM %s", PAGE_TABLE_NAME) connect, err := dbService.Connect() @@ -49,6 +53,7 @@ func (p Page) SelectAll(dbService database.DbService) ([]Page, error) { return result, err } +// AddOrUpdate add or update row func (p Page) AddOrUpdate(dbService database.DbService) error { connect, err := dbService.Connect() if err != nil { @@ -64,6 +69,7 @@ func (p Page) AddOrUpdate(dbService database.DbService) error { } } +// Insert insert data to table func (p Page) Insert(dbService database.DbService) error { connect, err := dbService.Connect() if err != nil { @@ -75,6 +81,7 @@ func (p Page) Insert(dbService database.DbService) error { return err } +// Update update data in table func (p Page) Update(dbService database.DbService) error { connect, err := dbService.Connect() if err != nil { @@ -86,6 +93,7 @@ func (p Page) Update(dbService database.DbService) error { return err } +// Delete delete data from table func (p Page) Delete(dbService database.DbService) error { connect, err := dbService.Connect() if err != nil { diff --git a/wScan.db b/wScan.db deleted file mode 100644 index 48aba16..0000000 Binary files a/wScan.db and /dev/null differ