Merge pull request #44 from everFinance/feature/support-absolute-path

feat(): support manifest absolute path

api.go

@@ -3,12 +3,12 @@ package arseeding
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/everFinance/arseeding/common"
 	"github.com/everFinance/arseeding/schema"
 	"github.com/everFinance/everpay-go/account"
 	"github.com/everFinance/goar/types"
 	"github.com/everFinance/goar/utils"
 	"github.com/gin-gonic/gin"
+	"gorm.io/gorm"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -19,13 +19,13 @@ import (
 
 func (s *Arseeding) runAPI(port string) {
 	r := s.engine
-	r.Use(common.CORSMiddleware())
+	r.Use(CORSMiddleware())
 	if s.EnableManifest {
-		r.Use(common.SandboxMiddleware())
+		r.Use(ManifestMiddleware(s.wdb, s.store))
 	}
 
 	if !s.NoFee {
-		r.Use(common.LimiterMiddleware(3000, "M", s.config.GetIPWhiteList()))
+		r.Use(LimiterMiddleware(3000, "M", s.config.GetIPWhiteList()))
 	}
 	v1 := r.Group("/")
 	{
@@ -77,8 +77,7 @@ func (s *Arseeding) runAPI(port string) {
 		v1.GET("/bundle/fees", s.bundleFees)
 		v1.GET("/bundle/fee/:size/:currency", s.bundleFee)
 		v1.GET("/bundle/orders/:signer", s.getOrders)
-		v1.GET("/:id", s.dataResponse)       // get arTx data or bundleItem data
-		v1.GET("/:id/*path", s.dataResponse) // get pathData from manifest data
+		v1.GET("/:id", s.dataRoute) // get arTx data or bundleItem data
 
 		// submit native data with X-API-KEY
 		v1.POST("/bundle/data", s.submitNativeData)
@@ -93,8 +92,8 @@ func (s *Arseeding) runAPI(port string) {
 func (s *Arseeding) arseedInfo(c *gin.Context) {
 	c.JSON(http.StatusOK, gin.H{
 		"Name":          "Arseeding",
-		"Version":       "v1.0.12",
-		"Documentation": "https://web3infura.io",
+		"Version":       "v1.0.13",
+		"Documentation": "https://web3infra.dev",
 	})
 }
 
@@ -739,26 +738,35 @@ func (s *Arseeding) bundleFees(c *gin.Context) {
 	c.JSON(http.StatusOK, s.bundlePerFeeMap)
 }
 
-func (s *Arseeding) dataResponse(c *gin.Context) {
-	tags, data, err := getArTxOrItemData(c.Param("id"), s.store)
+func (s *Arseeding) dataRoute(c *gin.Context) {
+	txId := c.Param("id")
+	tags, data, err := getArTxOrItemData(txId, s.store)
 	switch err {
 	case nil:
 		// process manifest
 		if s.EnableManifest && getTagValue(tags, schema.ContentType) == schema.ManifestType {
-			tags, data, err = handleManifest(data, c.Param("path"), s.store)
-			if err != nil {
-				if err == schema.ErrLocalNotExist {
-					proxyArweaveGateway(c)
-				} else if err == schema.ErrPageNotFound {
-					notFoundResponse(c, err.Error())
-				} else {
+			mfUrl := expectedTxSandbox(txId)
+			if _, err = s.wdb.GetManifestId(mfUrl); err == gorm.ErrRecordNotFound {
+				// insert new record
+				if err = s.wdb.InsertManifest(schema.Manifest{
+					ManifestUrl: mfUrl,
+					ManifestId:  txId,
+				}); err != nil {
 					internalErrorResponse(c, err.Error())
+					return
 				}
-				return
 			}
-		}
 
-		c.Data(200, fmt.Sprintf("%s; charset=utf-8", getTagValue(tags, schema.ContentType)), data)
+			protocol := "https"
+			if c.Request.TLS == nil {
+				protocol = "http"
+			}
+			redirectUrl := fmt.Sprintf("%s://%s.%s", protocol, mfUrl, c.Request.Host)
+
+			c.Redirect(302, redirectUrl)
+		} else {
+			c.Data(200, fmt.Sprintf("%s; charset=utf-8", getTagValue(tags, schema.ContentType)), data)
+		}
 
 	case schema.ErrLocalNotExist:
 		proxyArweaveGateway(c)

arseeding.go

@@ -64,7 +64,7 @@ func New(
 	}
 
 	wdb := NewWdb(dsn)
-	if err = wdb.Migrate(noFee); err != nil {
+	if err = wdb.Migrate(noFee, enableManifest); err != nil {
 		panic(err)
 	}
 	bundler, err := goar.NewWalletFromPath(arWalletKeyPath, arNode)

go.mod

@@ -9,10 +9,10 @@ require (
 	github.com/everFinance/goether v1.1.7
 	github.com/gin-gonic/gin v1.7.7
 	github.com/go-co-op/gocron v1.11.0
-	github.com/panjf2000/ants/v2 v2.4.7
+	github.com/panjf2000/ants/v2 v2.5.0
 	github.com/prometheus/client_golang v1.12.2
 	github.com/shopspring/decimal v1.2.0
-	github.com/stretchr/testify v1.7.0
+	github.com/stretchr/testify v1.7.1
 	github.com/tidwall/gjson v1.14.1
 	github.com/ulule/limiter/v3 v3.10.0
 	github.com/urfave/cli/v2 v2.3.0

jobs.go

@@ -207,47 +207,56 @@ func (s *Arseeding) mergeReceiptAndOrder() {
 	}
 
 	for _, urtx := range unspentRpts {
-		paymentItemId := gjson.Parse(urtx.Data).Get("itemId").String()
-		ord, err := s.wdb.GetUnPaidOrder(paymentItemId)
+		itemIds, err := parseItemIds(urtx.Data)
 		if err != nil {
-			log.Error("s.wdb.GetUnPaidOrder", "err", err, "itemId", paymentItemId)
+			log.Error("parseItemIds(urtx.Data)", "err", err, "urtx", urtx.EverHash)
+			if err = s.wdb.UpdateReceiptStatus(urtx.RawId, schema.UnRefund, nil); err != nil {
+				log.Error("s.wdb.UpdateReceiptStatus1", "err", err, "id", urtx.RawId)
+			}
+			continue
+		}
+		// get orders by itemIds
+		ordArr, err := s.getUnPaidOrdersByItemIds(itemIds)
+		if err != nil {
+			log.Error("s.wdb.GetUnPaidOrder", "err", err, "id", urtx.RawId)
 			if err == gorm.ErrRecordNotFound {
-				log.Warn("need refund about not find order", "itemId", paymentItemId)
+				log.Warn("need refund about not find order", "id", urtx.RawId)
 				// update receipt status is unrefund and waiting refund
 				if err = s.wdb.UpdateReceiptStatus(urtx.RawId, schema.UnRefund, nil); err != nil {
-					log.Error("s.wdb.UpdateReceiptStatus", "err", err, "id", urtx.RawId)
+					log.Error("s.wdb.UpdateReceiptStatus2", "err", err, "id", urtx.RawId)
 				}
 			}
 			continue
 		}
 
-		// verify payment token and amount
-		amt, ok := new(big.Int).SetString(urtx.Amount, 10)
-		if !ok {
-			log.Error("SetString(urtx.Amount,10)", "amount", urtx.Amount)
-			continue
-		}
-		fee, ok := new(big.Int).SetString(ord.Fee, 10)
-		if !ok {
-			log.Error("SetString(ord.Fee,10)", "fee", ord.Fee)
+		// check currency, orders currency must == paymentTxSymbol
+		if err = checkOrdersCurrency(ordArr, urtx.Symbol); err != nil {
+			log.Error("checkOrdersCurrency(ordArr, urtx.Symbol)", "err", err, "urtx", urtx.EverHash)
+			if err = s.wdb.UpdateReceiptStatus(urtx.RawId, schema.UnRefund, nil); err != nil {
+				log.Error("s.wdb.UpdateReceiptStatus3", "err", err, "id", urtx.RawId)
+			}
 			continue
 		}
-		if strings.ToLower(ord.Currency) != strings.ToLower(urtx.Symbol) || amt.Cmp(fee) < 0 {
-			log.Warn("need refund about currency or amount", "itemId", paymentItemId, "paySymble", urtx.Symbol, "payAmount", amt.String())
-			// update receipt status is unrefund and waiting refund
+
+		// check amount
+		if err = checkOrdersAmount(ordArr, urtx.Amount); err != nil {
+			log.Error("checkOrdersAmount(ordArr, urtx.Amount)", "err", err, "urtx", urtx.EverHash)
 			if err = s.wdb.UpdateReceiptStatus(urtx.RawId, schema.UnRefund, nil); err != nil {
-				log.Error("s.wdb.UpdateReceiptStatus", "err", err, "id", urtx.RawId)
+				log.Error("s.wdb.UpdateReceiptStatus4", "err", err, "id", urtx.RawId)
 			}
 			continue
 		}
 
 		// update order payment status
 		dbTx := s.wdb.Db.Begin()
-		if err = s.wdb.UpdateOrderPay(ord.ID, urtx.EverHash, schema.SuccPayment, dbTx); err != nil {
-			log.Error("s.wdb.UpdateOrderPay(ord.ID,schema.SuccPayment,dbTx)", "err", err)
-			dbTx.Rollback()
-			continue
+		for _, ord := range ordArr {
+			if err = s.wdb.UpdateOrderPay(ord.ID, urtx.EverHash, schema.SuccPayment, dbTx); err != nil {
+				log.Error("s.wdb.UpdateOrderPay(ord.ID,schema.SuccPayment,dbTx)", "err", err)
+				dbTx.Rollback()
+				break
+			}
 		}
+
 		if err = s.wdb.UpdateReceiptStatus(urtx.RawId, schema.Spent, dbTx); err != nil {
 			log.Error("s.wdb.UpdateReceiptStatus(urtx.ID,schema.Spent,dbTx)", "err", err)
 			dbTx.Rollback()
@@ -257,6 +266,63 @@ func (s *Arseeding) mergeReceiptAndOrder() {
 	}
 }
 
+func checkOrdersAmount(ordArr []schema.Order, txAmount string) error {
+	txAmountInt, ok := new(big.Int).SetString(txAmount, 10)
+	if !ok {
+		return errors.New("txAmount incorrect")
+	}
+	totalFee := big.NewInt(0)
+	for _, ord := range ordArr {
+		fee, ok := new(big.Int).SetString(ord.Fee, 10)
+		if !ok {
+			return errors.New("order fee incorrect")
+		}
+		totalFee = new(big.Int).Add(totalFee, fee)
+	}
+	if txAmountInt.Cmp(totalFee) < 0 {
+		return errors.New("payAmount fee not enough")
+	}
+	return nil
+}
+
+func checkOrdersCurrency(ordArr []schema.Order, txSymbol string) error {
+	for _, ord := range ordArr {
+		if strings.ToUpper(ord.Currency) != strings.ToUpper(txSymbol) {
+			return errors.New("currency incorrect")
+		}
+	}
+	return nil
+}
+
+func parseItemIds(txData string) ([]string, error) {
+	itemIds := make([]string, 0)
+	res := gjson.Parse(txData)
+	// appName must be arseeding
+	if res.Get("appName").String() != "arseeding" {
+		return nil, errors.New("txData.appName not be arseeding")
+	}
+	for _, it := range res.Get("itemIds").Array() {
+		itemIds = append(itemIds, it.String())
+	}
+	if len(itemIds) == 0 {
+		return nil, errors.New("itemIds is empty")
+	}
+	return itemIds, nil
+}
+
+func (s *Arseeding) getUnPaidOrdersByItemIds(itemIds []string) ([]schema.Order, error) {
+	ordArr := make([]schema.Order, 0, len(itemIds))
+	for _, itemId := range itemIds {
+		ord, err := s.wdb.GetUnPaidOrder(itemId)
+		if err != nil {
+			log.Error("s.wdb.GetUnPaidOrder(itemId)", "err", err, "itemId", itemId)
+			return nil, err
+		}
+		ordArr = append(ordArr, ord)
+	}
+	return ordArr, nil
+}
+
 func (s *Arseeding) refundReceipt() {
 	recpts, err := s.wdb.GetReceiptsByStatus(schema.UnRefund)
 	if err != nil {
@@ -278,8 +344,9 @@ func (s *Arseeding) refundReceipt() {
 		}
 		// everTx data
 		mmap := map[string]string{
-			"App-Name":        "arseeding",
-			"Refund-EverHash": rpt.EverHash,
+			"appName":        "arseeding",
+			"action":         "refund",
+			"refundEverHash": rpt.EverHash,
 		}
 		data, _ := json.Marshal(mmap)
 		everTx, err := s.everpaySdk.Transfer(rpt.Symbol, amount, rpt.From, string(data))

manifest.go

@@ -15,8 +15,6 @@ func handleManifest(maniData []byte, path string, db *Store) ([]types.Tag, []byt
 		return nil, nil, err
 	}
 
-	originalPath := path
-
 	path = strings.TrimPrefix(path, "/")
 	path = strings.TrimSuffix(path, "/")
 	if path == "" {
@@ -27,14 +25,11 @@ func handleManifest(maniData []byte, path string, db *Store) ([]types.Tag, []byt
 	}
 	txId, ok := mani.Paths[path]
 	if !ok {
-		if originalPath[len(originalPath)-1] == '/' {
-			txId, ok = mani.Paths[path+"/"+"index.html"]
-			if !ok {
-				return nil, nil, schema.ErrPageNotFound
-			}
-		} else {
-			return nil, nil, schema.ErrPageNotFound
-		}
+		// could ignore index.html, so add index.html and try again
+		txId, ok = mani.Paths[path+"/"+"index.html"]
+	}
+	if !ok {
+		return nil, nil, schema.ErrPageNotFound
 	}
 
 	tags, data, err := getArTxOrItemData(txId.TxId, db)

common/middleware.go → middleware.go

@@ -1,9 +1,10 @@
-package common
+package arseeding
 
 import (
 	"encoding/base32"
 	"errors"
 	"fmt"
+	"github.com/everFinance/arseeding/schema"
 	"github.com/everFinance/goar/utils"
 	"github.com/gin-gonic/gin"
 	"github.com/ulule/limiter/v3"
@@ -67,31 +68,46 @@ func CORSMiddleware() gin.HandlerFunc {
 	}
 }
 
-func SandboxMiddleware() gin.HandlerFunc {
+func ManifestMiddleware(wdb *Wdb, store *Store) gin.HandlerFunc {
 	return func(c *gin.Context) {
-		txId := getTxIdFromPath(c.Request.RequestURI)
-		isBrowser := false
-		if strings.Contains(c.GetHeader("User-Agent"), "Mozilla") {
-			isBrowser = true
-		}
-		if isBrowser && txId != "" {
-			currentSandbox := getRequestSandbox(c.Request)
-			expectedSandbox := expectedTxSandbox(txId)
-			if currentSandbox != expectedSandbox {
+		prefixUri := getRequestSandbox(c.Request.Host)
+		if len(prefixUri) > 0 && c.Request.Method == "GET" {
+			// compatible url https://xxxxxxx.arseed.web3infra.dev/{{arId}}
+			txId := getTxIdFromPath(c.Request.RequestURI)
+			if txId != "" && prefixUri == expectedTxSandbox(txId) {
 				protocol := "https"
 				if c.Request.TLS == nil {
 					protocol = "http"
 				}
-				redirectUrl := fmt.Sprintf("%s://%s.%s%s", protocol, expectedSandbox, c.Request.Host, c.Request.RequestURI)
-				// add "/" fix double slash
-				redirectUrl = strings.TrimSuffix(redirectUrl, "/")
-				if c.Param("path") == "" {
-					redirectUrl = redirectUrl + "/"
-				}
-
+				// redirect url: https://arseed.web3infra.dev/{{arId}}
+				rootHost := strings.SplitN(c.Request.Host, ".", 2)[1]
+				redirectUrl := fmt.Sprintf("%s://%s/%s", protocol, rootHost, txId)
 				c.Redirect(302, redirectUrl)
+
+				c.Abort()
+				return
+			}
+
+			mfId, err := wdb.GetManifestId(prefixUri)
+			if err != nil {
+				c.Next()
+				return
+			}
+			_, mfData, err := getArTxOrItemData(mfId, store)
+			if err != nil {
+				c.Abort()
+				internalErrorResponse(c, err.Error())
 				return
 			}
+			tags, data, err := handleManifest(mfData, c.Request.URL.Path, store)
+			if err != nil {
+				c.Abort()
+				internalErrorResponse(c, err.Error())
+				return
+			}
+			c.Abort()
+			c.Data(http.StatusOK, fmt.Sprintf("%s; charset=utf-8", getTagValue(tags, schema.ContentType)), data)
+			return
 		}
 		c.Next()
 	}
@@ -106,10 +122,10 @@ func getTxIdFromPath(path string) string {
 	return ""
 }
 
-func getRequestSandbox(req *http.Request) string {
-	hostStr := strings.Split(req.Host, ".")
-	if len(hostStr) > 0 {
-		return strings.ToLower(hostStr[0])
+func getRequestSandbox(host string) string {
+	prefix := strings.Split(host, ".")[0]
+	if len(prefix) > 40 { // todo 40
+		return prefix
 	}
 	return ""
 }