breaking-change(redis): change Redis Architecture (#36)

* feat(redis): change redis architecture and fix unimplemented parts

* chore: change log

* feat(cmd): add version command

* feat(cmd): add batch size option

* chore: change err message

* chore: fix test

* chore: check batch-size

* feat(redis): update redis architecture

* chore(integration): update test

* chore: increase the sample_rate

GNUmakefile

@@ -83,6 +83,7 @@ clean-integration:
 	-pkill go-cpe.old
 	-pkill go-cpe.new
 	-rm integration/go-cpe.old integration/go-cpe.new integration/go-cpe.old.sqlite3 integration/go-cpe.new.sqlite3
+	-rm -rf integration/diff
 	-docker kill redis-old redis-new
 	-docker rm redis-old redis-new
 
@@ -102,21 +103,21 @@ fetch-redis:
 	integration/go-cpe.new fetch jvn --dbtype redis --dbpath "redis://127.0.0.1:6380/0"
 
 diff-server-rdb:
-	integration/go-cpe.old server --dbpath=$(PWD)/integration/go-cpe.old.sqlite3 --port 1325 > /dev/null & 
-	integration/go-cpe.new server --dbpath=$(PWD)/integration/go-cpe.new.sqlite3 --port 1326 > /dev/null &
-	@ python integration/diff_server_mode.py cpes --sample_rate 0.001
+	integration/go-cpe.old server --dbpath=$(PWD)/integration/go-cpe.old.sqlite3 --port 1325 > /dev/null 2>&1 & 
+	integration/go-cpe.new server --dbpath=$(PWD)/integration/go-cpe.new.sqlite3 --port 1326 > /dev/null 2>&1 &
+	@ python integration/diff_server_mode.py cpes --sample_rate 0.01
 	pkill go-cpe.old 
 	pkill go-cpe.new
 
 diff-server-redis:
-	integration/go-cpe.old server --dbtype redis --dbpath "redis://127.0.0.1:6379/0" --port 1325 > /dev/null & 
-	integration/go-cpe.new server --dbtype redis --dbpath "redis://127.0.0.1:6380/0" --port 1326 > /dev/null &
-	@ python integration/diff_server_mode.py cpes --sample_rate 0.001
+	integration/go-cpe.old server --dbtype redis --dbpath "redis://127.0.0.1:6379/0" --port 1325 > /dev/null 2>&1 &
+	integration/go-cpe.new server --dbtype redis --dbpath "redis://127.0.0.1:6380/0" --port 1326 > /dev/null 2>&1 &
+	@ python integration/diff_server_mode.py cpes --sample_rate 0.01
 	pkill go-cpe.old 
 	pkill go-cpe.new
 
 diff-server-rdb-redis:
-	integration/go-cpe.new server --dbpath=$(PWD)/integration/go-cpe.new.sqlite3 --port 1325 > /dev/null &
-	integration/go-cpe.new server --dbtype redis --dbpath "redis://127.0.0.1:6380/0" --port 1326 > /dev/null &
-	@ python integration/diff_server_mode.py cpes --sample_rate 0.001
+	integration/go-cpe.new server --dbpath=$(PWD)/integration/go-cpe.new.sqlite3 --port 1325 > /dev/null 2>&1 &
+	integration/go-cpe.new server --dbtype redis --dbpath "redis://127.0.0.1:6380/0" --port 1326 > /dev/null 2>&1 &
+	@ python integration/diff_server_mode.py cpes --sample_rate 0.01
 	pkill go-cpe.new

README.md

@@ -105,6 +105,7 @@ Global Flags:
       --http-proxy string   http://proxy-url:port (default: empty)
       --log-dir string      /path/to/log (default "/var/log/go-cpe-dictionary")
       --log-json            output log as JSON
+      --log-to-file         output log to file
 
 Use "go-cpe-dictionary fetch [command] --help" for more information about a command.
 
@@ -128,6 +129,7 @@ Global Flags:
       --http-proxy string   http://proxy-url:port (default: empty)
       --log-dir string      /path/to/log (default "/var/log/go-cpe-dictionary")
       --log-json            output log as JSON
+      --log-to-file         output log to file
 ```
 
 ----

commands/fetch.go

@@ -26,6 +26,9 @@ func init() {
 	fetchCmd.PersistentFlags().Int("threads", runtime.NumCPU(), "The number of threads to be used")
 	_ = viper.BindPFlag("threads", fetchCmd.PersistentFlags().Lookup("threads"))
 
+	fetchCmd.PersistentFlags().Int("batch-size", 100, "The number of batch size to insert.")
+	_ = viper.BindPFlag("batch-size", fetchCmd.PersistentFlags().Lookup("batch-size"))
+
 	fetchCmd.PersistentFlags().Uint("expire", 0, "timeout to set for Redis keys in seconds. If set to 0, the key is persistent.")
 	_ = viper.BindPFlag("expire", fetchCmd.PersistentFlags().Lookup("expire"))
 }

commands/fetchjvn.go

@@ -9,6 +9,7 @@ import (
 	"github.com/vulsio/go-cpe-dictionary/db"
 	"github.com/vulsio/go-cpe-dictionary/fetcher"
 	"github.com/vulsio/go-cpe-dictionary/models"
+	"github.com/vulsio/go-cpe-dictionary/util"
 	"golang.org/x/xerrors"
 )
 
@@ -24,6 +25,10 @@ func init() {
 }
 
 func fetchJvn(cmd *cobra.Command, args []string) (err error) {
+	if err := util.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil {
+		return xerrors.Errorf("Failed to SetLogger. err: %w", err)
+	}
+
 	log15.Info("Initialize Database")
 	driver, locked, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"))
 	if err != nil {

commands/fetchnvd.go

@@ -9,6 +9,7 @@ import (
 	"github.com/vulsio/go-cpe-dictionary/db"
 	"github.com/vulsio/go-cpe-dictionary/fetcher"
 	"github.com/vulsio/go-cpe-dictionary/models"
+	"github.com/vulsio/go-cpe-dictionary/util"
 	"golang.org/x/xerrors"
 )
 
@@ -24,6 +25,10 @@ func init() {
 }
 
 func fetchNvd(cmd *cobra.Command, args []string) (err error) {
+	if err := util.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil {
+		return xerrors.Errorf("Failed to SetLogger. err: %w", err)
+	}
+
 	log15.Info("Initialize Database")
 	driver, locked, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"))
 	if err != nil {

commands/root.go

@@ -28,6 +28,9 @@ func init() {
 
 	RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.go-cpe-dictionary.yaml)")
 
+	RootCmd.PersistentFlags().Bool("log-to-file", false, "output log to file")
+	_ = viper.BindPFlag("log-to-file", RootCmd.PersistentFlags().Lookup("log-to-file"))
+
 	RootCmd.PersistentFlags().String("log-dir", util.GetDefaultLogDir(), "/path/to/log")
 	_ = viper.BindPFlag("log-dir", RootCmd.PersistentFlags().Lookup("log-dir"))
 
@@ -74,8 +77,4 @@ func initConfig() {
 	if err := viper.ReadInConfig(); err == nil {
 		fmt.Println("Using config file:", viper.ConfigFileUsed())
 	}
-	logDir := viper.GetString("log-dir")
-	debug := viper.GetBool("debug")
-	logJSON := viper.GetBool("log-json")
-	util.SetLogger(logDir, debug, logJSON)
 }

commands/server.go

@@ -6,6 +6,8 @@ import (
 	"github.com/spf13/viper"
 	"github.com/vulsio/go-cpe-dictionary/db"
 	"github.com/vulsio/go-cpe-dictionary/server"
+	"github.com/vulsio/go-cpe-dictionary/util"
+	"golang.org/x/xerrors"
 )
 
 var serverCmd = &cobra.Command{
@@ -26,7 +28,10 @@ func init() {
 }
 
 func executeServer(cmd *cobra.Command, args []string) (err error) {
-	logDir := viper.GetString("log-dir")
+	if err := util.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil {
+		return xerrors.Errorf("Failed to SetLogger. err: %w", err)
+	}
+
 	driver, locked, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"))
 	if err != nil {
 		if locked {
@@ -36,7 +41,7 @@ func executeServer(cmd *cobra.Command, args []string) (err error) {
 	}
 
 	log15.Info("Starting HTTP Server...")
-	if err = server.Start(logDir, driver); err != nil {
+	if err = server.Start(viper.GetBool("log-to-file"), viper.GetString("log-dir"), driver); err != nil {
 		log15.Error("Failed to start server.", "err", err)
 		return err
 	}

commands/version.go

@@ -0,0 +1,21 @@
+package commands
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+	"github.com/vulsio/go-cpe-dictionary/config"
+)
+
+func init() {
+	RootCmd.AddCommand(versionCmd)
+}
+
+var versionCmd = &cobra.Command{
+	Use:   "version",
+	Short: "Show version",
+	Long:  `Show version`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Printf("go-cpe-dictionary %s %s\n", config.Version, config.Revision)
+	},
+}

db/common_test.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/knqyf263/go-cpe/common"
 	"github.com/knqyf263/go-cpe/naming"
+	"github.com/spf13/viper"
 	"github.com/vulsio/go-cpe-dictionary/models"
 )
 
@@ -53,6 +54,7 @@ func prepareTestData(driver DB) error {
 		})
 	}
 
+	viper.Set("batch-size", 1)
 	return driver.InsertCpes(models.NVD, testCpes)
 }
 
@@ -74,15 +76,15 @@ func testGetVendorProducts(t *testing.T, driver DB) {
 		"OK": {
 			Expected: Expected{
 				VendorProduct: []string{
-					"ntp::ntp",
-					"responsive_coming_soon_page_project::responsive_coming_soon_page",
-					"vendorName1::productName1\\-1", // TODO: what's with these slashes? Is it a bug?
-					"vendorName1::productName1\\-2", // TODO: what's with these slashes? Is it a bug?
-					"vendorName2::productName2",
-					"vendorName3::productName3",
-					"vendorName4::productName4",
-					"vendorName5::productName5",
-					"vendorName6::productName6",
+					"ntp#ntp",
+					"responsive_coming_soon_page_project#responsive_coming_soon_page",
+					"vendorName1#productName1\\-1", // TODO: what's with these slashes? Is it a bug?
+					"vendorName1#productName1\\-2", // TODO: what's with these slashes? Is it a bug?
+					"vendorName2#productName2",
+					"vendorName3#productName3",
+					"vendorName4#productName4",
+					"vendorName5#productName5",
+					"vendorName6#productName6",
 				},
 			},
 		},

db/rdb.go

@@ -11,6 +11,7 @@ import (
 	"github.com/cheggaaa/pb/v3"
 	"github.com/inconshreveable/log15"
 	sqlite3 "github.com/mattn/go-sqlite3"
+	"github.com/spf13/viper"
 	"github.com/vulsio/go-cpe-dictionary/config"
 	"github.com/vulsio/go-cpe-dictionary/models"
 	"golang.org/x/xerrors"
@@ -43,12 +44,17 @@ func (r *RDBDriver) Name() string {
 func (r *RDBDriver) OpenDB(dbType, dbPath string, debugSQL bool) (locked bool, err error) {
 	gormConfig := gorm.Config{
 		DisableForeignKeyConstraintWhenMigrating: true,
-		Logger:                                   logger.Default.LogMode(logger.Silent),
+		Logger: logger.New(
+			log.New(os.Stderr, "\r\n", log.LstdFlags),
+			logger.Config{
+				LogLevel: logger.Silent,
+			},
+		),
 	}
 
 	if debugSQL {
 		gormConfig.Logger = logger.New(
-			log.New(os.Stdout, "\r\n", log.LstdFlags),
+			log.New(os.Stderr, "\r\n", log.LstdFlags),
 			logger.Config{
 				SlowThreshold: time.Second,
 				LogLevel:      logger.Info,
@@ -171,7 +177,7 @@ func (r *RDBDriver) GetVendorProducts() (vendorProducts []string, err error) {
 	}
 
 	for _, vp := range results {
-		vendorProducts = append(vendorProducts, fmt.Sprintf("%s::%s", vp.Vendor, vp.Product))
+		vendorProducts = append(vendorProducts, fmt.Sprintf("%s#%s", vp.Vendor, vp.Product))
 	}
 	return
 }
@@ -209,6 +215,11 @@ func (r *RDBDriver) deleteAndInsertCpes(conn *gorm.DB, fetchType models.FetchTyp
 		tx.Commit()
 	}()
 
+	batchSize := viper.GetInt("batch-size")
+	if batchSize < 1 {
+		return xerrors.New("Failed to set batch-size. err: batch-size option is not set properly")
+	}
+
 	// Delete all old records
 	oldIDs := []int64{}
 	result := tx.Model(models.CategorizedCpe{}).Select("id").Where("fetch_type = ?", fetchType).Find(&oldIDs)
@@ -218,15 +229,15 @@ func (r *RDBDriver) deleteAndInsertCpes(conn *gorm.DB, fetchType models.FetchTyp
 
 	if result.RowsAffected > 0 {
 		log15.Info(fmt.Sprintf("Deleting records that match fetch_type = %s from your DB. This will take some time.", fetchType))
-		for idx := range chunkSlice(len(oldIDs), 10000) {
+		for idx := range chunkSlice(len(oldIDs), batchSize) {
 			if err := tx.Where("id IN ?", oldIDs[idx.From:idx.To]).Delete(&models.CategorizedCpe{}).Error; err != nil {
 				return xerrors.Errorf("Failed to delete: %w", err)
 			}
 		}
 	}
 
 	bar := pb.StartNew(len(cpes))
-	for idx := range chunkSlice(len(cpes), 2000) {
+	for idx := range chunkSlice(len(cpes), batchSize) {
 		if err := tx.Create(cpes[idx.From:idx.To]).Error; err != nil {
 			return xerrors.Errorf("Failed to insert. err: %w", err)
 		}