Merge pull request #55 from everFinance/dev/support-sqlite

Dev/support sqlite

arseeding.go

@@ -43,7 +43,7 @@ type Arseeding struct {
 }
 
 func New(
-	boltDirPath, dsn string,
+	boltDirPath, mySqlDsn string, sqliteDir string, useSqlite bool,
 	arWalletKeyPath string, arNode, payUrl string, noFee bool, enableManifest bool,
 	useS3 bool, s3AccKey, s3SecretKey, s3BucketPrefix, s3Region, s3Endpoint string,
 	use4EVER bool, port string,
@@ -52,7 +52,7 @@ func New(
 	KVDb := &Store{}
 	if useS3 {
 		if use4EVER {
-			s3Endpoint = rawdb.ForeverLandEndpoint //inject 4everland endpoint
+			s3Endpoint = rawdb.ForeverLandEndpoint // inject 4everland endpoint
 		}
 		KVDb, err = NewS3Store(s3AccKey, s3SecretKey, s3Region, s3BucketPrefix, s3Endpoint)
 	} else {
@@ -66,8 +66,12 @@ func New(
 	if err := jobmg.InitTaskMg(KVDb); err != nil {
 		panic(err)
 	}
-
-	wdb := NewWdb(dsn)
+	wdb := &Wdb{}
+	if useSqlite {
+		wdb = NewSqliteDb(sqliteDir)
+	} else {
+		wdb = NewMysqlDb(mySqlDsn)
+	}
 	if err = wdb.Migrate(noFee, enableManifest); err != nil {
 		panic(err)
 	}
@@ -87,7 +91,7 @@ func New(
 
 	localArseedUrl := "http://127.0.0.1" + port
 	a := &Arseeding{
-		config:              config.New(dsn),
+		config:              config.New(mySqlDsn, sqliteDir, useSqlite),
 		store:               KVDb,
 		engine:              gin.Default(),
 		submitLocker:        sync.Mutex{},

cmd/main.go

@@ -17,6 +17,9 @@ func main() {
 		Flags: []cli.Flag{
 			&cli.StringFlag{Name: "db_dir", Value: "./data/bolt", Usage: "bolt db dir path", EnvVars: []string{"DB_DIR"}},
 			&cli.StringFlag{Name: "mysql", Value: "root@tcp(127.0.0.1:3306)/arseed?charset=utf8mb4&parseTime=True&loc=Local", Usage: "mysql dsn", EnvVars: []string{"MYSQL"}},
+			&cli.BoolFlag{Name: "use_sqlite", Value: false, EnvVars: []string{"USE_SQLITE"}},
+			&cli.StringFlag{Name: "sqlite_dir", Value: "./data/sqlite", Usage: "sqlite db dir path", EnvVars: []string{"SQLITE_DIR"}},
+
 			&cli.StringFlag{Name: "key_path", Value: "./data/bundler-keyfile.json", Usage: "ar keyfile path", EnvVars: []string{"KEY_PATH"}},
 			&cli.StringFlag{Name: "ar_node", Value: "https://arweave.net", EnvVars: []string{"AR_NODE"}},
 			&cli.StringFlag{Name: "pay", Value: "https://api-dev.everpay.io", Usage: "pay url", EnvVars: []string{"PAY"}},
@@ -48,7 +51,8 @@ func run(c *cli.Context) error {
 	signal.Notify(signals, os.Interrupt, syscall.SIGTERM)
 
 	s := arseeding.New(
-		c.String("db_dir"), c.String("mysql"), c.String("key_path"), c.String("ar_node"), c.String("pay"), c.Bool("no_fee"), c.Bool("manifest"),
+		c.String("db_dir"), c.String("mysql"), c.String("sqlite_dir"), c.Bool("use_sqlite"),
+		c.String("key_path"), c.String("ar_node"), c.String("pay"), c.Bool("no_fee"), c.Bool("manifest"),
 		c.Bool("use_s3"), c.String("s3_acc_key"), c.String("s3_secret_key"), c.String("s3_prefix"), c.String("s3_region"), c.String("s3_endpoint"),
 		c.Bool("use_4ever"), c.String("port"))
 	s.Run(c.String("port"), c.Int("bundle_interval"))

config/config.go

@@ -14,8 +14,13 @@ type Config struct {
 	scheduler      *gocron.Scheduler
 }
 
-func New(configDSN string) *Config {
-	wdb := NewWdb(configDSN)
+func New(configDSN, sqliteDir string, useSqlite bool) *Config {
+	wdb := &Wdb{}
+	if useSqlite {
+		wdb = NewSqliteDb(sqliteDir)
+	} else {
+		wdb = NewMysqlDb(configDSN)
+	}
 	err := wdb.Migrate()
 	if err != nil {
 		panic(err)

config/schema/db.go

@@ -7,7 +7,7 @@ type FeeConfig struct {
 
 type IpRateWhitelist struct {
 	OriginOrIP  string // e.g "188.0.2.2"
-	Available   bool   `gorm:"index:idx1"` // true means effective
+	Available   bool   `gorm:"index:idx3"` // true means effective
 	Description string
 }
 

config/wdb.go

@@ -4,8 +4,15 @@ import (
 	"github.com/everFinance/arseeding/config/schema"
 	"github.com/everFinance/everpay-go/common"
 	"gorm.io/driver/mysql"
+	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
 	"gorm.io/gorm/logger"
+	"os"
+	"path"
+)
+
+const (
+	sqliteName = "seed.db"
 )
 
 var log = common.NewLog("config")
@@ -14,16 +21,32 @@ type Wdb struct {
 	Db *gorm.DB
 }
 
-func NewWdb(dsn string) *Wdb {
+func NewMysqlDb(dsn string) *Wdb {
 	db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{
-		Logger:          logger.Default.LogMode(logger.Error),
-		CreateBatchSize: 10,
+		Logger:          logger.Default.LogMode(logger.Silent),
+		CreateBatchSize: 200,
+	})
+	if err != nil {
+		panic(err)
+	}
+	log.Info("connect mysql db success")
+	return &Wdb{Db: db}
+}
+
+func NewSqliteDb(dbDir string) *Wdb {
+	if err := os.MkdirAll(dbDir, os.ModePerm); err != nil {
+		panic(err)
+	}
+	db, err := gorm.Open(sqlite.Open(path.Join(dbDir, sqliteName)), &gorm.Config{
+		Logger:          logger.Default.LogMode(logger.Silent),
+		CreateBatchSize: 200,
 	})
 	if err != nil {
 		panic(err)
 	}
-	log.Info("connect config db success")
+	log.Info("connect sqlite db success")
 	return &Wdb{Db: db}
+
 }
 
 func (w *Wdb) Migrate() error {

go.mod

@@ -20,6 +20,7 @@ require (
 	gopkg.in/h2non/gentleman.v2 v2.0.5
 	gorm.io/datatypes v1.0.1
 	gorm.io/driver/mysql v1.2.1
+	gorm.io/driver/sqlite v1.1.4
 	gorm.io/gorm v1.22.4
 )
 
@@ -54,6 +55,7 @@ require (
 	github.com/leodido/go-urn v1.2.0 // indirect
 	github.com/mattn/go-colorable v0.1.11 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mattn/go-sqlite3 v1.14.5 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect

schema/manifest.go

@@ -49,6 +49,6 @@ type Resource struct {
 
 type Manifest struct {
 	ID          uint   `gorm:"primarykey" json:"-"`
-	ManifestUrl string `gorm:"index:idx1,unique" json:"manifestUrl"`
+	ManifestUrl string `gorm:"index:idxMani0,unique" json:"manifestUrl"`
 	ManifestId  string `json:"manifestId"` // arId
 }

wdb.go

@@ -3,30 +3,55 @@ package arseeding
 import (
 	"github.com/everFinance/arseeding/schema"
 	"gorm.io/driver/mysql"
+	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
 	"gorm.io/gorm/clause"
 	"gorm.io/gorm/logger"
 	"math"
+	"os"
+	"path"
 	"strings"
 	"time"
 )
 
+const (
+	sqliteName = "seed.db"
+)
+
 type Wdb struct {
 	Db *gorm.DB
 }
 
-func NewWdb(dsn string) *Wdb {
+func NewMysqlDb(dsn string) *Wdb {
 	db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{
 		Logger:          logger.Default.LogMode(logger.Silent),
 		CreateBatchSize: 200,
 	})
 	if err != nil {
 		panic(err)
 	}
-	log.Info("connect db success")
+	log.Info("connect mysql db success")
 	return &Wdb{Db: db}
 }
 
+func NewSqliteDb(dbDir string) *Wdb {
+	if err := os.MkdirAll(dbDir, os.ModePerm); err != nil {
+		panic(err)
+	}
+	db, err := gorm.Open(sqlite.Open(path.Join(dbDir, sqliteName)), &gorm.Config{
+		Logger:          logger.Default.LogMode(logger.Silent),
+		CreateBatchSize: 200,
+	})
+	if err != nil {
+		panic(err)
+	}
+	log.Info("connect sqlite db success")
+	return &Wdb{Db: db}
+
+}
+
+// when use sqlite,same index name in different table will lead to migrate failed,
+
 func (w *Wdb) Migrate(noFee, enableManifest bool) error {
 	err := w.Db.AutoMigrate(&schema.Order{}, &schema.OnChainTx{})
 	if err != nil {

wdb_test.go

@@ -1,6 +1,10 @@
 package arseeding
 
-import "testing"
+import (
+	"github.com/everFinance/arseeding/schema"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
 
 func TestNewWdb(t *testing.T) {
 	// dsn := "root@tcp(127.0.0.1:3306)/arseed?charset=utf8mb4&parseTime=True&loc=Local"
@@ -14,3 +18,15 @@ func TestNewWdb(t *testing.T) {
 	// latest := wdb.IsLatestUnpaidOrd(itemId,expiredTime)
 	// t.Log(latest)
 }
+
+func TestSqlite(t *testing.T) {
+	db := NewSqliteDb("testSqlite")
+	err := db.Migrate(true, true)
+	assert.NoError(t, err)
+	err = db.InsertOrder(schema.Order{ID: 2, Fee: "123"})
+	assert.NoError(t, err)
+	ord := &schema.Order{}
+	err = db.Db.First(ord).Error
+	assert.NoError(t, err)
+	t.Log(ord)
+}