diff --git a/api.go b/api.go index c75126d..c3cb680 100644 --- a/api.go +++ b/api.go @@ -114,9 +114,9 @@ func (s *Arseeding) runAPI(port string) { } go func() { - gLog.Fatal(http.ListenAndServe(":8081", handlers.CompressHandler(http.DefaultServeMux))) + gLog.Fatal(http.ListenAndServe(":8061", handlers.CompressHandler(http.DefaultServeMux))) }() - gLog.Printf("you can now open http://localhost:8080/debug/charts/ in your browser") + gLog.Printf("you can now open http://localhost:8061/debug/charts/ in your browser") if err := r.Run(port); err != nil { panic(err) @@ -420,7 +420,7 @@ func (s *Arseeding) postTask(c *gin.Context) { return } tkType := c.Param("taskType") - if !strings.Contains(schema.TaskTypeSync+schema.TaskTypeBroadcast+schema.TaskTypeBroadcastMeta, tkType) { + if !strings.Contains(schema.TaskTypeSync+schema.TaskTypeBroadcast+schema.TaskTypeBroadcastMeta+schema.TaskTypeSyncManifest, tkType) { errorResponse(c, "tktype not exist") return } @@ -455,7 +455,7 @@ func (s *Arseeding) killTask(c *gin.Context) { func (s *Arseeding) getTask(c *gin.Context) { arid := c.Param("arid") tktype := c.Param("taskType") - if !strings.Contains(schema.TaskTypeSync+schema.TaskTypeBroadcast+schema.TaskTypeBroadcastMeta, tktype) { + if !strings.Contains(schema.TaskTypeSync+schema.TaskTypeBroadcast+schema.TaskTypeBroadcastMeta+schema.TaskTypeSyncManifest, tktype) { errorResponse(c, "tktype not exist") return } diff --git a/arseeding.go b/arseeding.go index 2283b82..8b48723 100644 --- a/arseeding.go +++ b/arseeding.go @@ -171,6 +171,7 @@ func (s *Arseeding) Close() { for _, k := range s.KWriters { k.Close() } + log.Warn("arseeding closed") } func (s *Arseeding) GetPerFee(tokenSymbol string) *schema.Fee { diff --git a/cli/arseeding/cmd/root.go b/cli/arseeding/cmd/root.go new file mode 100644 index 0000000..a817f80 --- /dev/null +++ b/cli/arseeding/cmd/root.go @@ -0,0 +1,77 @@ +/* +Copyright © 2023 NAME HERE +*/ +package cmd + +import ( + "fmt" + "github.com/everFinance/arseeding/schema" + "os" + + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var cfgFile string +var cfg schema.Config + +// rootCmd represents the base command when called without any subcommands +var rootCmd = &cobra.Command{ + Use: "arseeding", + Short: "arseeding", + Long: `arseeding`, + Version: "v1.2.3", + // Uncomment the following line if your bare application + // has an action associated with it: + // Run: func(cmd *cobra.Command, args []string) { }, +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute() { + err := rootCmd.Execute() + if err != nil { + os.Exit(1) + } +} + +func init() { + cobra.OnInitialize(initConfig) + + // Here you will define your flags and configuration settings. + // Cobra supports persistent flags, which, if defined here, + // will be global for your application. + + rootCmd.PersistentFlags().StringVar(&cfgFile, "cfg", "", "cfg file (default is $./arseeding.yaml)") + + // Cobra also supports local flags, which will only run + // when this action is called directly. + rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} + +// initConfig reads in cfg file and ENV variables if set. +func initConfig() { + if cfgFile != "" { + // Use cfg file from the flag. + viper.SetConfigFile(cfgFile) + } else { + // Search cfg in home directory with name ".arseeding" (without extension). + viper.AddConfigPath(".") + viper.SetConfigType("yaml") + viper.SetConfigName("arseeding") + } + + viper.AutomaticEnv() // read in environment variables that match + + // If a cfg file is found, read it in. + if err := viper.ReadInConfig(); err == nil { + fmt.Fprintln(os.Stderr, "Using config file:", viper.ConfigFileUsed()) + } else { + fmt.Println("can not find config file") + panic(err) + } + + if err := viper.Unmarshal(&cfg); err != nil { + panic(err) + } +} diff --git a/cli/arseeding/cmd/start.go b/cli/arseeding/cmd/start.go new file mode 100644 index 0000000..874d104 --- /dev/null +++ b/cli/arseeding/cmd/start.go @@ -0,0 +1,121 @@ +/* +Copyright © 2023 NAME HERE +*/ +package cmd + +import ( + "encoding/json" + "fmt" + "github.com/everFinance/arseeding" + "github.com/everFinance/goar/types" + "os" + "os/exec" + "os/signal" + "syscall" + "time" + + "github.com/spf13/cobra" +) + +const pidFile string = ".arseeding_pid.lock" + +var daemon bool + +// startCmd represents the start command +var startCmd = &cobra.Command{ + Use: "start", + Short: "start arseeding", + Long: `start arseeding`, + RunE: func(cmd *cobra.Command, args []string) error { + if daemon { + if _, err := os.Stat(pidFile); err == nil { + fmt.Println("Failed start, PID file exist.running...") + return nil + } + + path, err := os.Executable() + if err != nil { + return err + } + + command := exec.Command(path, "start") + + // add log + logFileName := fmt.Sprintf("arseeding_%d.log", time.Now().Unix()) + logFile, err := os.OpenFile(logFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + return err + } + + command.Stdout = logFile + command.Stderr = logFile + + if err := command.Start(); err != nil { + return err + } + err = os.WriteFile(pidFile, []byte(fmt.Sprintf("%d", command.Process.Pid)), 0666) + if err != nil { + return err + } + + daemon = false + os.Exit(0) + } else { + runServer() + } + return nil + }, +} + +func init() { + rootCmd.AddCommand(startCmd) + + startCmd.Flags().BoolVarP(&daemon, "deamon", "d", false, "is daemon?") + // Here you will define your flags and configuration settings. + + // Cobra supports Persistent Flags which will work for this command + // and all subcommands, e.g.: + // startCmd.PersistentFlags().String("foo", "", "A help for foo") + + // Cobra supports local flags which will only run when this command + // is called directly, e.g.: + // startCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} + +func runServer() { + signals := make(chan os.Signal, 1) + signal.Notify(signals, os.Interrupt, syscall.SIGTERM, os.Kill) + + useSqlite := false + sqliteDir := "" + + tagJs := cfg.Tags + if len(tagJs) == 0 { + tagJs = `{"Community":"PermaDAO","Website":"permadao.com"}` + } + tagsMap := make(map[string]string) + if err := json.Unmarshal([]byte(tagJs), &tagsMap); err != nil { + panic(err) + } + + customTags := make([]types.Tag, 0) + for k, v := range tagsMap { + customTags = append(customTags, types.Tag{ + Name: k, + Value: v, + }) + } + + m := arseeding.New(cfg.BoltDir, cfg.Mysql, sqliteDir, useSqlite, + cfg.RollupKeyPath, cfg.ArNode, cfg.Pay, cfg.NoFee, cfg.Manifest, + cfg.S3KV.UseS3, cfg.S3KV.AccKey, cfg.S3KV.SecretKey, cfg.S3KV.Prefix, cfg.S3KV.Region, cfg.S3KV.Endpoint, cfg.S3KV.User4Ever, + cfg.AliyunKV.UseAliyun, cfg.AliyunKV.Endpoint, cfg.AliyunKV.AccKey, cfg.AliyunKV.SecretKey, cfg.AliyunKV.Prefix, + cfg.MongoDBKV.UseMongoDB, cfg.MongoDBKV.Uri, + cfg.Port, customTags, + cfg.Kafka.Start, cfg.Kafka.Uri) + + m.Run(cfg.Port, cfg.BundleInterval) + + <-signals + m.Close() +} diff --git a/cli/arseeding/cmd/stop.go b/cli/arseeding/cmd/stop.go new file mode 100644 index 0000000..9a5d33d --- /dev/null +++ b/cli/arseeding/cmd/stop.go @@ -0,0 +1,49 @@ +/* +Copyright © 2023 NAME HERE +*/ +package cmd + +import ( + "fmt" + "os" + "os/exec" + + "github.com/spf13/cobra" +) + +// stopCmd represents the stop command +var stopCmd = &cobra.Command{ + Use: "stop", + Short: "stop arseeding", + Long: `stop arseeding`, + RunE: func(cmd *cobra.Command, args []string) error { + strb, err := os.ReadFile(pidFile) + if err != nil { + fmt.Println("Stop server failed, err: %v", err) + return nil + } + command := exec.Command("kill", string(strb)) + if err := command.Start(); err != nil { + return err + } + if err := os.Remove(pidFile); err != nil { + return err + } + println("arseeding stopped") + return nil + }, +} + +func init() { + rootCmd.AddCommand(stopCmd) + + // Here you will define your flags and configuration settings. + + // Cobra supports Persistent Flags which will work for this command + // and all subcommands, e.g.: + // stopCmd.PersistentFlags().String("foo", "", "A help for foo") + + // Cobra supports local flags which will only run when this command + // is called directly, e.g.: + // stopCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} diff --git a/cli/arseeding/example_arseeding.yaml b/cli/arseeding/example_arseeding.yaml new file mode 100644 index 0000000..23c6784 --- /dev/null +++ b/cli/arseeding/example_arseeding.yaml @@ -0,0 +1,16 @@ +rollupKeyPath: ./data/arweave-keyfile-permaweb.json +pay: https://api.everpay.io +arNode: https://arweave.net +mysql: arseeding:7YXMHxKeQmbeCpnD@(127.0.0.1:13306)/arseeding?charset=utf8mb4&parseTime=True&loc=Local +port: :6080 +manifest: true +noFee: false +bundleInterval: 120 +boltDir: ./data +s3KV: + useS3: true + user4Ever: false + accKey: AKIAR63DMM2P4DGMWZU5 + secretKey: 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 + prefix: public-arseed + region: ap-southeast-1 diff --git a/cli/arseeding/main.go b/cli/arseeding/main.go new file mode 100644 index 0000000..14d4745 --- /dev/null +++ b/cli/arseeding/main.go @@ -0,0 +1,10 @@ +/* +Copyright © 2023 NAME HERE +*/ +package main + +import "github.com/everFinance/arseeding/cli/arseeding/cmd" + +func main() { + cmd.Execute() +} diff --git a/go.mod b/go.mod index 36e81cc..972952a 100644 --- a/go.mod +++ b/go.mod @@ -2,13 +2,11 @@ module github.com/everFinance/arseeding go 1.21.1 -toolchain go1.21.5 - require ( github.com/aliyun/aliyun-oss-go-sdk v2.2.6+incompatible github.com/aws/aws-sdk-go v1.27.0 github.com/ethereum/go-ethereum v1.13.5 - github.com/everFinance/goar v1.5.8 + github.com/everFinance/goar v1.5.9 github.com/everFinance/goether v1.1.9 github.com/gin-gonic/gin v1.8.1 github.com/go-co-op/gocron v1.11.0 @@ -37,6 +35,8 @@ require ( github.com/everFinance/goarns v0.0.3 github.com/everVision/everpay-kits v0.0.6-0.20240201142725-21cc7715d94d github.com/segmentio/kafka-go v0.4.40 + github.com/spf13/cobra v1.5.0 + github.com/spf13/viper v1.18.2 go.mongodb.org/mongo-driver v1.11.4 ) @@ -51,12 +51,13 @@ require ( github.com/consensys/gnark-crypto v0.12.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect github.com/ethereum/c-kzg-4844 v0.4.0 // indirect github.com/everFinance/ethrpc v1.0.4 // indirect github.com/everFinance/gojwk v1.0.0 // indirect github.com/everFinance/ttcrsa v1.1.3 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.5.0 // indirect github.com/getsentry/sentry-go v0.25.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect @@ -75,14 +76,17 @@ require ( github.com/google/go-tpm v0.9.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/hamba/avro v1.5.6 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/uint256 v1.2.3 // indirect github.com/inconshreveable/log15 v2.16.0+incompatible // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.4 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.16.0 // indirect + github.com/klauspost/compress v1.17.0 // indirect github.com/leodido/go-urn v1.2.1 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-sqlite3 v1.14.5 // indirect @@ -92,16 +96,23 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect - github.com/pelletier/go-toml/v2 v2.0.5 // indirect + github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a // indirect github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.6.0 // indirect github.com/supranational/blst v0.3.11 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect @@ -116,14 +127,18 @@ require ( github.com/xdg-go/stringprep v1.0.4 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.17.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/net v0.19.0 // indirect - golang.org/x/sync v0.3.0 // indirect + golang.org/x/sync v0.5.0 // indirect golang.org/x/sys v0.15.0 // indirect golang.org/x/term v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/time v0.3.0 // indirect - google.golang.org/protobuf v1.29.1 // indirect + golang.org/x/time v0.5.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect diff --git a/manifest.go b/manifest.go index 12d1dac..aaed828 100644 --- a/manifest.go +++ b/manifest.go @@ -77,13 +77,14 @@ func getArTxOrItemDataForManifest(id string, db *Store, s *Arseeding) (decodeTag // find bundle item form local decodeTags, binaryReader, data, err = getArTxOrItemData(id, db) + log.Error("getArTxOrItemDataForManifest getArTxOrItemData", "err", err) // if err equal ErrLocalNotExist, put task to queue to sync data if err == schema.ErrLocalNotExist { - // registerTask if err = s.registerTask(id, schema.TaskTypeSyncManifest); err != nil { log.Error("registerTask sync_manifest error", "err", err) } + log.Debug("registerTask sync_manifest...", "id", id) return nil, nil, nil, schema.ErrLocalNotExist } @@ -137,14 +138,12 @@ func parseBundleItem(binaryReader *os.File, itemBinary []byte) (item *types.Bund } func syncManifestData(id string, s *Arseeding) (err error) { - // get manifest data data, contentType, err := getRawById(id) if err != nil { return err } - log.Debug("syncManifestData get raw end ", "id", id, "contentType", contentType, "data", string(data)) var bundleInItemsMap = make(map[string][]string) var L1Artxs []string var itemIds []string @@ -153,8 +152,7 @@ func syncManifestData(id string, s *Arseeding) (err error) { // if contentType == "application/x.arweave-manifest+json" parse it if contentType == "application/x.arweave-manifest+json" { - - //is manifest data parse it + // is manifest data parse it mani := schema.ManifestData{} if err := json.Unmarshal(data, &mani); err != nil { return err @@ -167,8 +165,6 @@ func syncManifestData(id string, s *Arseeding) (err error) { } } - log.Debug("total txId in manifest", "len", len(itemIds)) - // query itemIds from graphql gq := argraphql.NewARGraphQL("https://arweave.net/graphql", http.Client{}) @@ -198,12 +194,11 @@ func syncManifestData(id string, s *Arseeding) (err error) { } } - log.Debug("syncManifestData bundleInItemsMap", "bundleInItemsMap", bundleInItemsMap, "L1Artxs", L1Artxs) + log.Debug("syncManifestData bundleInItemsMap", "bundleInItemsMap", len(bundleInItemsMap), "L1Artxs", L1Artxs) // get bundle item form goar c := goar.NewClient("https://arweave.net") for bundleId, itemIds := range bundleInItemsMap { - - log.Debug("syncManifestData GetBundleItems ", "bundleId", bundleId, "itemIds", itemIds) + log.Debug("syncManifestData GetBundleItems ", "bundleId", bundleId, "itemIds", len(itemIds)) // GetBundleItems items, err := c.GetBundleItems(bundleId, itemIds) diff --git a/manifets_test.go b/manifets_test.go index acc6d98..2d5542a 100644 --- a/manifets_test.go +++ b/manifets_test.go @@ -1,9 +1,13 @@ package arseeding -import "testing" +import ( + "encoding/json" + "github.com/everFinance/arseeding/schema" + "testing" +) func Test_getRawById(t *testing.T) { - data, contentType, err := getRawById("arDRw5qt51v4pOV9TrQXKJM2iLK-c39dvs2K-7b3oDk") + data, contentType, err := getRawById("U1FqvR_xTuL2qxrJDw20oIghpGt1eTumJ9ZfCczc5_M") if err != nil { t.Error(err) @@ -12,4 +16,14 @@ func Test_getRawById(t *testing.T) { t.Log(err) t.Log(contentType) t.Log(string(data)) + + mani := schema.ManifestData{} + err = json.Unmarshal(data, &mani) + t.Log(err) +} + +func TestNewS3Store(t *testing.T) { + err := syncManifestData("U1FqvR_xTuL2qxrJDw20oIghpGt1eTumJ9ZfCczc5_M", nil) + // err := syncManifestData("WUg9McRBT1i_F6utVYjFYS_pP6dzKcrC1FRccjH6inE", nil) + t.Log(err) } diff --git a/middleware.go b/middleware.go index b2858aa..302902c 100644 --- a/middleware.go +++ b/middleware.go @@ -149,12 +149,14 @@ func ManifestMiddleware(s *Arseeding) gin.HandlerFunc { "web3infura.io", "arweave.world", "arweave.asia", + "localhost", + "127.0.0.1", } log.Debug("middleware", "currentHost", currentHost) for _, b := range apiHostList { - if currentHost == b { + if strings.Split(currentHost, ":")[0] == b { // if current host in apiHostList, notApiHost is false notApiHost = false break diff --git a/middleware_test.go b/middleware_test.go index 43d4884..62f4006 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -1,6 +1,11 @@ package arseeding -import "testing" +import ( + "github.com/everFinance/goarns" + "github.com/stretchr/testify/assert" + "testing" + "time" +) func TestSandboxMiddleware(t *testing.T) { host := "p6qmubetdqoqlsoktncg3hiec2nbyjmgqgmhboopftn67xfk.arseed.web3infura.io" @@ -15,3 +20,16 @@ func TestGetSubDomain(t *testing.T) { res := getSubDomain(host) t.Log(res) } + +func TestLimiterMiddleware(t *testing.T) { + dreUrl := "https://dre-1.warp.cc" + arNSAddress := "bLAgYxAdX2Ry-nt6aH2ixgvJXbpsEYm28NgJgyqfs-U" + timeout := 10 * time.Second + + a := goarns.NewArNS(dreUrl, arNSAddress, timeout) + + domain := "cookbook" + txId, err := a.QueryLatestRecord(domain) + assert.NoError(t, err) + t.Log(txId) +} diff --git a/schema/cli.go b/schema/cli.go new file mode 100644 index 0000000..b02510a --- /dev/null +++ b/schema/cli.go @@ -0,0 +1,48 @@ +package schema + +type Config struct { + RollupKeyPath string `yaml:"rollupKeyPath"` + Pay string `yaml:"pay"` + ArNode string `yaml:"arNode"` + Mysql string `yaml:"mysql"` + Port string `yaml:"port"` + Manifest bool `yaml:"manifest"` + NoFee bool `yaml:"noFee"` + BundleInterval int `yaml:"bundleInterval"` + Tags string `yaml:"tags"` + + BoltDir string `yaml:"boltDir"` + S3KV S3KV `yaml:"s3KV"` + AliyunKV AliyunKV `yaml:"aliyunKV"` + MongoDBKV MongoDBKV `yaml:"mongoDBKV"` + + Kafka Kafka `yaml:"kafka"` +} + +type S3KV struct { + UseS3 bool `yaml:"useS3"` + User4Ever bool `yaml:"user4Ever"` + AccKey string `yaml:"accKey"` + SecretKey string `yaml:"secretKey"` + Prefix string `yaml:"prefix"` + Region string `yaml:"region"` + Endpoint string `yaml:"endpoint"` +} + +type AliyunKV struct { + UseAliyun bool `yaml:"useAliyun"` + Endpoint string `yaml:"endpoint"` + AccKey string `yaml:"accKey"` + SecretKey string `yaml:"secretKey"` + Prefix string `yaml:"prefix"` +} + +type MongoDBKV struct { + UseMongoDB bool `yaml:"useMongoDB"` + Uri string `yaml:"uri"` +} + +type Kafka struct { + Start bool `yaml:"start"` + Uri string `yaml:"uri"` +}