diff --git a/.gitignore b/.gitignore index 3369f05..3cf0919 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ build data .idea vendor -tmpFile \ No newline at end of file +tmpFile +deploy.sh diff --git a/Makefile b/Makefile index 7f27ed0..93d2a1a 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,10 @@ all: go mod tidy - go build -o ./build/arseeding ./cmd \ No newline at end of file + go build -o ./build/arseeding ./cmd + +gen-graphql: + go get github.com/Khan/genqlient + genqlient ./argraphql/genqlient.yaml + +build-linux-bin: + GOOS=linux GOARCH=amd64 go build -o ./build/arseeding ./cmd \ No newline at end of file diff --git a/api.go b/api.go index d2a3658..089fea9 100644 --- a/api.go +++ b/api.go @@ -37,7 +37,7 @@ func (s *Arseeding) runAPI(port string) { r := s.engine r.Use(CORSMiddleware()) if s.EnableManifest { - r.Use(ManifestMiddleware(s.wdb, s.store)) + r.Use(ManifestMiddleware(s)) } if !s.NoFee { @@ -1251,7 +1251,7 @@ func (s *Arseeding) getApiKey(c *gin.Context) { return } - addr, err := goether.Ecrecover(accounts.TextHash([]byte(timestamp)), common.FromHex(signature)) + _, addr, err := goether.Ecrecover(accounts.TextHash([]byte(timestamp)), common.FromHex(signature)) if err != nil { internalErrorResponse(c, err.Error()) return diff --git a/argraphql/generated.go b/argraphql/generated.go new file mode 100644 index 0000000..fcb86ce --- /dev/null +++ b/argraphql/generated.go @@ -0,0 +1,377 @@ +// Code generated by github.com/Khan/genqlient, DO NOT EDIT. + +package argraphql + +import ( + "context" + + "github.com/Khan/genqlient/graphql" +) + +// BatchGetItemsBundleInResponse is returned by BatchGetItemsBundleIn on success. +type BatchGetItemsBundleInResponse struct { + Transactions BatchGetItemsBundleInTransactionsTransactionConnection `json:"transactions"` +} + +// GetTransactions returns BatchGetItemsBundleInResponse.Transactions, and is useful for accessing the field via an interface. +func (v *BatchGetItemsBundleInResponse) GetTransactions() BatchGetItemsBundleInTransactionsTransactionConnection { + return v.Transactions +} + +// BatchGetItemsBundleInTransactionsTransactionConnection includes the requested fields of the GraphQL type TransactionConnection. +type BatchGetItemsBundleInTransactionsTransactionConnection struct { + PageInfo BatchGetItemsBundleInTransactionsTransactionConnectionPageInfo `json:"pageInfo"` + Edges []BatchGetItemsBundleInTransactionsTransactionConnectionEdgesTransactionEdge `json:"edges"` +} + +// GetPageInfo returns BatchGetItemsBundleInTransactionsTransactionConnection.PageInfo, and is useful for accessing the field via an interface. +func (v *BatchGetItemsBundleInTransactionsTransactionConnection) GetPageInfo() BatchGetItemsBundleInTransactionsTransactionConnectionPageInfo { + return v.PageInfo +} + +// GetEdges returns BatchGetItemsBundleInTransactionsTransactionConnection.Edges, and is useful for accessing the field via an interface. +func (v *BatchGetItemsBundleInTransactionsTransactionConnection) GetEdges() []BatchGetItemsBundleInTransactionsTransactionConnectionEdgesTransactionEdge { + return v.Edges +} + +// BatchGetItemsBundleInTransactionsTransactionConnectionEdgesTransactionEdge includes the requested fields of the GraphQL type TransactionEdge. +type BatchGetItemsBundleInTransactionsTransactionConnectionEdgesTransactionEdge struct { + Cursor string `json:"cursor"` + Node BatchGetItemsBundleInTransactionsTransactionConnectionEdgesTransactionEdgeNodeTransaction `json:"node"` +} + +// GetCursor returns BatchGetItemsBundleInTransactionsTransactionConnectionEdgesTransactionEdge.Cursor, and is useful for accessing the field via an interface. +func (v *BatchGetItemsBundleInTransactionsTransactionConnectionEdgesTransactionEdge) GetCursor() string { + return v.Cursor +} + +// GetNode returns BatchGetItemsBundleInTransactionsTransactionConnectionEdgesTransactionEdge.Node, and is useful for accessing the field via an interface. +func (v *BatchGetItemsBundleInTransactionsTransactionConnectionEdgesTransactionEdge) GetNode() BatchGetItemsBundleInTransactionsTransactionConnectionEdgesTransactionEdgeNodeTransaction { + return v.Node +} + +// BatchGetItemsBundleInTransactionsTransactionConnectionEdgesTransactionEdgeNodeTransaction includes the requested fields of the GraphQL type Transaction. +type BatchGetItemsBundleInTransactionsTransactionConnectionEdgesTransactionEdgeNodeTransaction struct { + Id string `json:"id"` + BundledIn BatchGetItemsBundleInTransactionsTransactionConnectionEdgesTransactionEdgeNodeTransactionBundledInBundle `json:"bundledIn"` +} + +// GetId returns BatchGetItemsBundleInTransactionsTransactionConnectionEdgesTransactionEdgeNodeTransaction.Id, and is useful for accessing the field via an interface. +func (v *BatchGetItemsBundleInTransactionsTransactionConnectionEdgesTransactionEdgeNodeTransaction) GetId() string { + return v.Id +} + +// GetBundledIn returns BatchGetItemsBundleInTransactionsTransactionConnectionEdgesTransactionEdgeNodeTransaction.BundledIn, and is useful for accessing the field via an interface. +func (v *BatchGetItemsBundleInTransactionsTransactionConnectionEdgesTransactionEdgeNodeTransaction) GetBundledIn() BatchGetItemsBundleInTransactionsTransactionConnectionEdgesTransactionEdgeNodeTransactionBundledInBundle { + return v.BundledIn +} + +// BatchGetItemsBundleInTransactionsTransactionConnectionEdgesTransactionEdgeNodeTransactionBundledInBundle includes the requested fields of the GraphQL type Bundle. +type BatchGetItemsBundleInTransactionsTransactionConnectionEdgesTransactionEdgeNodeTransactionBundledInBundle struct { + Id string `json:"id"` +} + +// GetId returns BatchGetItemsBundleInTransactionsTransactionConnectionEdgesTransactionEdgeNodeTransactionBundledInBundle.Id, and is useful for accessing the field via an interface. +func (v *BatchGetItemsBundleInTransactionsTransactionConnectionEdgesTransactionEdgeNodeTransactionBundledInBundle) GetId() string { + return v.Id +} + +// BatchGetItemsBundleInTransactionsTransactionConnectionPageInfo includes the requested fields of the GraphQL type PageInfo. +type BatchGetItemsBundleInTransactionsTransactionConnectionPageInfo struct { + HasNextPage bool `json:"hasNextPage"` +} + +// GetHasNextPage returns BatchGetItemsBundleInTransactionsTransactionConnectionPageInfo.HasNextPage, and is useful for accessing the field via an interface. +func (v *BatchGetItemsBundleInTransactionsTransactionConnectionPageInfo) GetHasNextPage() bool { + return v.HasNextPage +} + +// GetTransactionResponse is returned by GetTransaction on success. +type GetTransactionResponse struct { + Transaction GetTransactionTransaction `json:"transaction"` +} + +// GetTransaction returns GetTransactionResponse.Transaction, and is useful for accessing the field via an interface. +func (v *GetTransactionResponse) GetTransaction() GetTransactionTransaction { return v.Transaction } + +// GetTransactionTransaction includes the requested fields of the GraphQL type Transaction. +type GetTransactionTransaction struct { + Id string `json:"id"` + Anchor string `json:"anchor"` + Signature string `json:"signature"` + Recipient string `json:"recipient"` + Owner GetTransactionTransactionOwner `json:"owner"` + Fee GetTransactionTransactionFeeAmount `json:"fee"` + Quantity GetTransactionTransactionQuantityAmount `json:"quantity"` + Data GetTransactionTransactionDataMetaData `json:"data"` + Tags []GetTransactionTransactionTagsTag `json:"tags"` + Block GetTransactionTransactionBlock `json:"block"` + BundledIn GetTransactionTransactionBundledInBundle `json:"bundledIn"` +} + +// GetId returns GetTransactionTransaction.Id, and is useful for accessing the field via an interface. +func (v *GetTransactionTransaction) GetId() string { return v.Id } + +// GetAnchor returns GetTransactionTransaction.Anchor, and is useful for accessing the field via an interface. +func (v *GetTransactionTransaction) GetAnchor() string { return v.Anchor } + +// GetSignature returns GetTransactionTransaction.Signature, and is useful for accessing the field via an interface. +func (v *GetTransactionTransaction) GetSignature() string { return v.Signature } + +// GetRecipient returns GetTransactionTransaction.Recipient, and is useful for accessing the field via an interface. +func (v *GetTransactionTransaction) GetRecipient() string { return v.Recipient } + +// GetOwner returns GetTransactionTransaction.Owner, and is useful for accessing the field via an interface. +func (v *GetTransactionTransaction) GetOwner() GetTransactionTransactionOwner { return v.Owner } + +// GetFee returns GetTransactionTransaction.Fee, and is useful for accessing the field via an interface. +func (v *GetTransactionTransaction) GetFee() GetTransactionTransactionFeeAmount { return v.Fee } + +// GetQuantity returns GetTransactionTransaction.Quantity, and is useful for accessing the field via an interface. +func (v *GetTransactionTransaction) GetQuantity() GetTransactionTransactionQuantityAmount { + return v.Quantity +} + +// GetData returns GetTransactionTransaction.Data, and is useful for accessing the field via an interface. +func (v *GetTransactionTransaction) GetData() GetTransactionTransactionDataMetaData { return v.Data } + +// GetTags returns GetTransactionTransaction.Tags, and is useful for accessing the field via an interface. +func (v *GetTransactionTransaction) GetTags() []GetTransactionTransactionTagsTag { return v.Tags } + +// GetBlock returns GetTransactionTransaction.Block, and is useful for accessing the field via an interface. +func (v *GetTransactionTransaction) GetBlock() GetTransactionTransactionBlock { return v.Block } + +// GetBundledIn returns GetTransactionTransaction.BundledIn, and is useful for accessing the field via an interface. +func (v *GetTransactionTransaction) GetBundledIn() GetTransactionTransactionBundledInBundle { + return v.BundledIn +} + +// GetTransactionTransactionBlock includes the requested fields of the GraphQL type Block. +type GetTransactionTransactionBlock struct { + Id string `json:"id"` + Timestamp int `json:"timestamp"` + Height int `json:"height"` + Previous string `json:"previous"` +} + +// GetId returns GetTransactionTransactionBlock.Id, and is useful for accessing the field via an interface. +func (v *GetTransactionTransactionBlock) GetId() string { return v.Id } + +// GetTimestamp returns GetTransactionTransactionBlock.Timestamp, and is useful for accessing the field via an interface. +func (v *GetTransactionTransactionBlock) GetTimestamp() int { return v.Timestamp } + +// GetHeight returns GetTransactionTransactionBlock.Height, and is useful for accessing the field via an interface. +func (v *GetTransactionTransactionBlock) GetHeight() int { return v.Height } + +// GetPrevious returns GetTransactionTransactionBlock.Previous, and is useful for accessing the field via an interface. +func (v *GetTransactionTransactionBlock) GetPrevious() string { return v.Previous } + +// GetTransactionTransactionBundledInBundle includes the requested fields of the GraphQL type Bundle. +type GetTransactionTransactionBundledInBundle struct { + Id string `json:"id"` +} + +// GetId returns GetTransactionTransactionBundledInBundle.Id, and is useful for accessing the field via an interface. +func (v *GetTransactionTransactionBundledInBundle) GetId() string { return v.Id } + +// GetTransactionTransactionDataMetaData includes the requested fields of the GraphQL type MetaData. +type GetTransactionTransactionDataMetaData struct { + Type string `json:"type"` + Size string `json:"size"` +} + +// GetType returns GetTransactionTransactionDataMetaData.Type, and is useful for accessing the field via an interface. +func (v *GetTransactionTransactionDataMetaData) GetType() string { return v.Type } + +// GetSize returns GetTransactionTransactionDataMetaData.Size, and is useful for accessing the field via an interface. +func (v *GetTransactionTransactionDataMetaData) GetSize() string { return v.Size } + +// GetTransactionTransactionFeeAmount includes the requested fields of the GraphQL type Amount. +type GetTransactionTransactionFeeAmount struct { + Winston string `json:"winston"` + Ar string `json:"ar"` +} + +// GetWinston returns GetTransactionTransactionFeeAmount.Winston, and is useful for accessing the field via an interface. +func (v *GetTransactionTransactionFeeAmount) GetWinston() string { return v.Winston } + +// GetAr returns GetTransactionTransactionFeeAmount.Ar, and is useful for accessing the field via an interface. +func (v *GetTransactionTransactionFeeAmount) GetAr() string { return v.Ar } + +// GetTransactionTransactionOwner includes the requested fields of the GraphQL type Owner. +type GetTransactionTransactionOwner struct { + Address string `json:"address"` + Key string `json:"key"` +} + +// GetAddress returns GetTransactionTransactionOwner.Address, and is useful for accessing the field via an interface. +func (v *GetTransactionTransactionOwner) GetAddress() string { return v.Address } + +// GetKey returns GetTransactionTransactionOwner.Key, and is useful for accessing the field via an interface. +func (v *GetTransactionTransactionOwner) GetKey() string { return v.Key } + +// GetTransactionTransactionQuantityAmount includes the requested fields of the GraphQL type Amount. +type GetTransactionTransactionQuantityAmount struct { + Winston string `json:"winston"` + Ar string `json:"ar"` +} + +// GetWinston returns GetTransactionTransactionQuantityAmount.Winston, and is useful for accessing the field via an interface. +func (v *GetTransactionTransactionQuantityAmount) GetWinston() string { return v.Winston } + +// GetAr returns GetTransactionTransactionQuantityAmount.Ar, and is useful for accessing the field via an interface. +func (v *GetTransactionTransactionQuantityAmount) GetAr() string { return v.Ar } + +// GetTransactionTransactionTagsTag includes the requested fields of the GraphQL type Tag. +type GetTransactionTransactionTagsTag struct { + Name string `json:"name"` + Value string `json:"value"` +} + +// GetName returns GetTransactionTransactionTagsTag.Name, and is useful for accessing the field via an interface. +func (v *GetTransactionTransactionTagsTag) GetName() string { return v.Name } + +// GetValue returns GetTransactionTransactionTagsTag.Value, and is useful for accessing the field via an interface. +func (v *GetTransactionTransactionTagsTag) GetValue() string { return v.Value } + +// __BatchGetItemsBundleInInput is used internally by genqlient +type __BatchGetItemsBundleInInput struct { + Ids []string `json:"ids"` + First int `json:"first"` + After string `json:"after"` +} + +// GetIds returns __BatchGetItemsBundleInInput.Ids, and is useful for accessing the field via an interface. +func (v *__BatchGetItemsBundleInInput) GetIds() []string { return v.Ids } + +// GetFirst returns __BatchGetItemsBundleInInput.First, and is useful for accessing the field via an interface. +func (v *__BatchGetItemsBundleInInput) GetFirst() int { return v.First } + +// GetAfter returns __BatchGetItemsBundleInInput.After, and is useful for accessing the field via an interface. +func (v *__BatchGetItemsBundleInInput) GetAfter() string { return v.After } + +// __GetTransactionInput is used internally by genqlient +type __GetTransactionInput struct { + Id string `json:"id"` +} + +// GetId returns __GetTransactionInput.Id, and is useful for accessing the field via an interface. +func (v *__GetTransactionInput) GetId() string { return v.Id } + +// The query or mutation executed by BatchGetItemsBundleIn. +const BatchGetItemsBundleIn_Operation = ` +query BatchGetItemsBundleIn ($ids: [ID!]!, $first: Int, $after: String) { + transactions(ids: $ids, first: $first, after: $after) { + pageInfo { + hasNextPage + } + edges { + cursor + node { + id + bundledIn { + id + } + } + } + } +} +` + +func BatchGetItemsBundleIn( + ctx context.Context, + client graphql.Client, + ids []string, + first int, + after string, +) (*BatchGetItemsBundleInResponse, error) { + req := &graphql.Request{ + OpName: "BatchGetItemsBundleIn", + Query: BatchGetItemsBundleIn_Operation, + Variables: &__BatchGetItemsBundleInInput{ + Ids: ids, + First: first, + After: after, + }, + } + var err error + + var data BatchGetItemsBundleInResponse + resp := &graphql.Response{Data: &data} + + err = client.MakeRequest( + ctx, + req, + resp, + ) + + return &data, err +} + +// The query or mutation executed by GetTransaction. +const GetTransaction_Operation = ` +query GetTransaction ($id: ID!) { + transaction(id: $id) { + id + anchor + signature + recipient + owner { + address + key + } + fee { + winston + ar + } + quantity { + winston + ar + } + data { + type + size + } + tags { + name + value + } + block { + id + timestamp + height + previous + } + bundledIn { + id + } + } +} +` + +// Write your query or mutation here +func GetTransaction( + ctx context.Context, + client graphql.Client, + id string, +) (*GetTransactionResponse, error) { + req := &graphql.Request{ + OpName: "GetTransaction", + Query: GetTransaction_Operation, + Variables: &__GetTransactionInput{ + Id: id, + }, + } + var err error + + var data GetTransactionResponse + resp := &graphql.Response{Data: &data} + + err = client.MakeRequest( + ctx, + req, + resp, + ) + + return &data, err +} diff --git a/argraphql/genqlient.graphql b/argraphql/genqlient.graphql new file mode 100644 index 0000000..8aa72a9 --- /dev/null +++ b/argraphql/genqlient.graphql @@ -0,0 +1,57 @@ +# Write your query or mutation here +query GetTransaction ($id: ID!){ + transaction(id: $id) { + id + anchor + signature + recipient + owner { + address + key + } + fee { + winston + ar + } + quantity { + winston + ar + } + data { + type + size + } + tags { + name + value + } + block { + id + timestamp + height + previous + } + + bundledIn { + id + } + } +} + +query BatchGetItemsBundleIn($ids : [ID!]!, $first: Int, $after: String){ + transactions(ids: $ids, first: $first, after: $after){ + pageInfo { + hasNextPage, + } + edges { + cursor + node { + id + bundledIn { + id + } + } + } + } +} + diff --git a/argraphql/genqlient.yaml b/argraphql/genqlient.yaml new file mode 100644 index 0000000..d8a3eb9 --- /dev/null +++ b/argraphql/genqlient.yaml @@ -0,0 +1,11 @@ +schema: schema.graphql +operations: + - genqlient.graphql +generated: generated.go +# needed since it doesn't match the directory name: +package: argraphql + + +bindings: + DateTime: + type: time.Time \ No newline at end of file diff --git a/argraphql/query.go b/argraphql/query.go new file mode 100644 index 0000000..6687743 --- /dev/null +++ b/argraphql/query.go @@ -0,0 +1,42 @@ +package argraphql + +import ( + "context" + "github.com/Khan/genqlient/graphql" + "github.com/everFinance/go-everpay/common" + "net/http" +) + +var logger = common.NewLog("arseeding") + +type ARGraphQL struct { + Client graphql.Client +} + +func NewARGraphQL(endpoint string, httpClient http.Client) *ARGraphQL { + return &ARGraphQL{ + Client: graphql.NewClient(endpoint, &httpClient), + } +} + +func (g *ARGraphQL) QueryTransaction(ctx context.Context, id string) (res *GetTransactionResponse, err error) { + + txResp, err := GetTransaction(ctx, g.Client, id) + + if err != nil { + logger.Error("ARGraphQL get transaction error", "err", err) + } + + return txResp, err +} + +func (g *ARGraphQL) BatchGetItemsBundleIn(ctx context.Context, ids []string, first int, after string) (res *BatchGetItemsBundleInResponse, err error) { + + batchResp, err := BatchGetItemsBundleIn(ctx, g.Client, ids, first, after) + + if err != nil { + logger.Error("ARGraphQL batch get items bundle in error", "err", err) + } + + return batchResp, err +} diff --git a/argraphql/query_test.go b/argraphql/query_test.go new file mode 100644 index 0000000..8103f5c --- /dev/null +++ b/argraphql/query_test.go @@ -0,0 +1,214 @@ +package argraphql + +import ( + "context" + "github.com/Khan/genqlient/graphql" + "github.com/stretchr/testify/assert" + "net/http" + "reflect" + "testing" +) + +func TestARGraphQL_QueryTransaction(t *testing.T) { + type fields struct { + Client graphql.Client + } + type args struct { + ctx context.Context + id string + } + tests := []struct { + name string + fields fields + args args + wantRes *GetTransactionResponse + wantErr bool + }{ + { + name: "test", + fields: fields{ + Client: graphql.NewClient("https://arweave.net/graphql", &http.Client{}), + }, + // oLQYpPmfv35ZGn5Vv0_H2GKLoyY5bnFAChngzjLiCEs + args: args{ + ctx: context.Background(), + id: "oLQYpPmfv35ZGn5Vv0_H2GKLoyY5bnFAChngzjLiCEs", + }, + /** + { + "transaction": { + "id": "oLQYpPmfv35ZGn5Vv0_H2GKLoyY5bnFAChngzjLiCEs", + "anchor": "MDh5WGFnK1lieTdKekl4K25LaFZkRjRjQ0s2YU9BdmM", + "signature": "hiHfDFJ0e6NFTooQwUg5hpKqtjBQAO50ApSILtQw_arT1PZ0oT8IapqxPg_iKUmAjyfS6rFHusbZkckahfP0_2sodoEEwv22XFcoRVJR3IlEgDBPAkrx2p7jZXN25EalqdUby6CUBusU54bUU4COhu3NTQQ5E7QyWo-hadCQm1mPQXxWV0sceS1qnhhsnDLO0vb9AOXt5P6zd5GCgbFDgxMZXGFA5-RfnyfIQwEe2kttwfxuLgfPqJKV7hUFB55rDDZXfw4Ps-PTUKJMSrwpRm95aWobFe9uKmf-LGFBZ-CnvxW0Jr6qf8QyAg7LzmboxIKClXV1QBbQzjlacGP_pZQFQLMna6k8VFZsja_L4jkftubgvdxQJygzWFVuXD0RiY0IwOVMDyrQePgRJXFi2kKoDGoF71ulbkL7dMVejQ6wb-7xc7wZp0cJU-Gk50mY70ASO-3tcCfs6_F1BRv7Mpt0P8Jq2GZKNRA7NVQVt6hImd-S8hh2_uVa-Q3R1YExnp5duP4LZGgiryXg_e0M1B895aPXdAktSbvJMButnWy_rH6TBiMA14q-Hqa8TDMiV_PWGJSM1cKj2qKZlqJj_UO_BFT09uNzQzMZKG3iwZI1qXqTddY_NjjsKkrezCAzCQG-A517tGhSA8KG7v6WjMD2gYVJxNkF3tQaPQ_joIc", + "recipient": "", + "owner": { + "address": "wQH9ojMXplWYiQvkSGVjIdWclk3XDM59poFoaOXKR8E", + "key": "l_V57UmpROcyH6bYlIjeflxe6YRLDkeC0bhF4yUX7140wwbTGXnF-WJQNoG3XHTvMdeSf4eCMF9G5V8uxnXWYs2PvGjR_5l_Ox-IWR2JIA8118oBZ1FQh9gkkSShoDQK4dJ1EmIQVJZFAmyx370COnM5kCU1yFHMFM3XfBPG9PQDgewpXL2haFzwDEq2jNff_q-cmPr1tmZY0gTQW_l-3aeruteZg9VMg-JK5sRann4L53dvP4DJ2LLM585E_et4cSTf_3gCkOwoykp3UM61UaeNBi31xeDfH2sd73id1sP-vG2NO1BOqL2PGWudC3z9gbU9CzlF8juf927s9MszlJf6IhfjwgVcmu3BKBmJ9D9yhMrXlLi_v5TT1R7e-m7-6JSDAT9oNs3KRQ-B7gZ-kWhWuZZdgb7KW5N2q5EMAXu8a8WVipicGMXG5y6IIt8aPtHM0MzUOiCN-956sXTkF546pysMtQHYv5A1ctUgjwg5L5-rKLi_nrlH5t6aEmP4TJwDA122kYIa4qYz0Vfle7DP94et5lwPVP2ofzgyVVzpCmxLI53uqbHy5pPpzLVH_JB4bVgCYjK3ddtlNernufUV5kRhAVS48fPZKQ2Y-gpR4908i6A5EnGxYbqkY8Q2-fgqX2hc17xFXLdzsMiwPxO1z7rg14BGPEH9xY3aMY0" + }, + "fee": { + "winston": "0", + "ar": "0.000000000000" + }, + "quantity": { + "winston": "0", + "ar": "0.000000000000" + }, + "data": { + "type": "", + "size": "818" + }, + "tags": [{ + "name": "Content-Type", + "value": "application/javascript; charset=utf-8" + }], + "block": { + "id": "-pOVy3h66x40s0ITVN55FVUAzQU3b56PXCr9DjQasnKV_I855cheOPU4DWHPBzne", + "timestamp": 1689883479, + "height": 1224045, + "previous": "IeBQvT4sX5iLOrDLXmd3AaWIwI5p9sb9ktN1RUJUiJeFmnCM6ztn_EBMy2maSAVp" + }, + "bundledIn": { + "id": "D0i_pTTxNo_e1csmGSI-rJvKz6xi61ksQVjLEig568E" + } + } + } + + full json to strut + */ + wantRes: &GetTransactionResponse{ + Transaction: GetTransactionTransaction{ + + Id: "D0i_pTTxNo_e1csmGSI-rJvKz6xi61ksQVjLEig568E", + Anchor: "MDh5WGFnK1lieTdKekl4K25LaFZkRjRjQ0s2YU9BdmM", + Signature: "hiHfDFJ0e6NFTooQwUg5hpKqtjBQAO50ApSILtQw_arT1PZ0oT8IapqxPg_iKUmAjyfS6rFHusbZkckahfP0_2sodoEEwv22XFcoRVJR3IlEgDBPAkrx2p7jZXN25EalqdUby6CUBusU54bUU4COhu3NTQQ5E7QyWo-hadCQm1mPQXxWV0sceS1qnhhsnDLO0vb9AOXt5P6zd5GCgbFDgxMZXGFA5-RfnyfIQwEe2kttwfxuLgfPqJKV7hUFB55rDDZXfw4Ps-PTUKJMSrwpRm95aWobFe9uKmf-LGFBZ-CnvxW0Jr6qf8QyAg7LzmboxIKClXV1QBbQzjlacGP_pZQFQLMna6k8VFZsja_L4jkftubgvdxQJygzWFVuXD0RiY0IwOVMDyrQePgRJXFi2kKoDGoF71ulbkL7dMVejQ6wb-7xc7wZp0cJU-Gk50mY70ASO-3tcCfs6_F1BRv7Mpt0P8Jq2GZKNRA7NVQVt6hImd-S8hh2_uVa-Q3R1YExnp5duP4LZGgiryXg_e0M1B895aPXdAktSbvJMButnWy_rH6TBiMA14q-Hqa8TDMiV_PWGJSM1cKj2qKZlqJj_UO_BFT09uNzQzMZKG3iwZI1qXqTddY_NjjsKkrezCAzCQG-A517tGhSA8KG7v6WjMD2gYVJxNkF3tQaPQ_joIc", + Recipient: "", + Owner: GetTransactionTransactionOwner{ + Address: "wQH9ojMXplWYiQvkSGVjIdWclk3XDM59poFoaOXKR8E", + Key: "l_V57UmpROcyH6bYlIjeflxe6YRLDkeC0bhF4yUX7140wwbTGXnF-WJQNoG3XHTvMdeSf4eCMF9G5V8uxnXWYs2PvGjR_5l_Ox-IWR2JIA8118oBZ1FQh9gkkSShoDQK4dJ1EmIQVJZFAmyx370COnM5kCU1yFHMFM3XfBPG9PQDgewpXL2haFzwDEq2jNff_q-cmPr1tmZY0gTQW_l-3aeruteZg9VMg-JK5sRann4L53dvP4DJ2LLM585E_et4cSTf_3gCkOwoykp3UM61UaeNBi31xeDfH2sd73id1sP-vG2NO1BOqL2PGWudC3z9gbU9CzlF8juf927s9MszlJf6IhfjwgVcmu3BKBmJ9D9yhMrXlLi_v5TT1R7e-m7-6JSDAT9oNs3KRQ-B7gZ-kWhWuZZdgb7KW5N2q5EMAXu8a8WVipicGMXG5y6IIt8aPtHM0MzUOiCN-956sXTkF546pysMtQHYv5A1ctUgjwg5L5-rKLi_nrlH5t6aEmP4TJwDA122kYIa4qYz0Vfle7DP94et5lwPVP2ofzgyVVzpCmxLI53uqbHy5pPpzLVH_JB4bVgCYjK3ddtlNernufUV5kRhAVS48fPZKQ2Y-gpR4908i6A5EnGxYbqkY8Q2-fgqX2hc17xFXLdzsMiwPxO1z7rg14BGPEH9xY3aMY0", + }, + Fee: GetTransactionTransactionFeeAmount{ + Winston: "0", + Ar: "0.000000000000", + }, + Quantity: GetTransactionTransactionQuantityAmount{ + Winston: "0", + Ar: "0.000000000000", + }, + Data: GetTransactionTransactionDataMetaData{ + Type: "", + Size: "818", + }, + Tags: []GetTransactionTransactionTagsTag{ + { + Name: "Content-Type", + Value: "application/javascript; charset=utf-8", + }, + }, + Block: GetTransactionTransactionBlock{ + Id: "-pOVy3h66x40s0ITVN55FVUAzQU3b56PXCr9DjQasnKV_I855cheOPU4DWHPBzne", + Timestamp: 1689883479, + Height: 1224045, + Previous: "IeBQvT4sX5iLOrDLXmd3AaWIwI5p9sb9ktN1RUJUiJeFmnCM6ztn_EBMy2maSAVp", + }, + BundledIn: GetTransactionTransactionBundledInBundle{ + Id: "D0i_pTTxNo_e1csmGSI-rJvKz6xi61ksQVjLEig568E", + }, + }, + }, + + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := &ARGraphQL{ + Client: tt.fields.Client, + } + gotRes, err := g.QueryTransaction(tt.args.ctx, tt.args.id) + if (err != nil) != tt.wantErr { + t.Errorf("QueryTransaction() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if !reflect.DeepEqual(gotRes, tt.wantRes) { + t.Errorf("QueryTransaction() gotRes = %v, want %v", gotRes, tt.wantRes) + } + }) + } +} + +func TestNewARGraphQL(t *testing.T) { + type args struct { + endpoint string + httpClient http.Client + } + tests := []struct { + name string + args args + want *ARGraphQL + }{ + { + name: "test", + args: args{ + endpoint: "http://localhost:1984", + httpClient: http.Client{}, + }, + want: &ARGraphQL{ + Client: graphql.NewClient("http://localhost:1984", &http.Client{}), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := NewARGraphQL(tt.args.endpoint, tt.args.httpClient); !reflect.DeepEqual(got, tt.want) { + t.Errorf("NewARGraphQL() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestQueryTransaction(t *testing.T) { + + gq := NewARGraphQL("https://arweave.net/graphql", http.Client{}) + + res, err := gq.QueryTransaction(context.Background(), "oLQYpPmfv35ZGn5Vv0_H2GKLoyY5bnFAChngzjLiCEs") + + assert.NoError(t, err) + + assert.Equal(t, "oLQYpPmfv35ZGn5Vv0_H2GKLoyY5bnFAChngzjLiCEs", res.Transaction.Id) + + assert.Equal(t, "0.000000000000", res.Transaction.Fee.Ar) + + assert.LessOrEqualf(t, "application/javascript; charset=utf-8", res.Transaction.Tags[0].Value, "tag name should be less than or equal to application/javascript; charset=utf-8") + + assert.Equal(t, "D0i_pTTxNo_e1csmGSI-rJvKz6xi61ksQVjLEig568E", res.Transaction.BundledIn.Id) +} + +func TestBatchGetItemsBundleIn(t *testing.T) { + + gq := NewARGraphQL("https://arweave.net/graphql", http.Client{}) + ids := []string{"arDRw5qt51v4pOV9TrQXKJM2iLK-c39dvs2K-7b3oDk", "O0N7iKmdv7Tmc0fnvJSKSeKuibvDrHbpMlb4K8pHwXg", "oLQYpPmfv35ZGn5Vv0_H2GKLoyY5bnFAChngzjLiCEs"} + + res, err := gq.BatchGetItemsBundleIn(context.Background(), ids, 100, "") + + assert.NoError(t, err) + assert.False(t, res.Transactions.PageInfo.HasNextPage) + assert.Equal(t, 3, len(res.Transactions.Edges)) + + for _, edge := range res.Transactions.Edges { + if edge.Node.Id == "arDRw5qt51v4pOV9TrQXKJM2iLK-c39dvs2K-7b3oDk" { + assert.Equal(t, "FnzJJ_6TDcgapyvs_-8vL2ImIWwehvRp_aWdhwS57U0", edge.Node.BundledIn.Id) + } + + if edge.Node.Id == "O0N7iKmdv7Tmc0fnvJSKSeKuibvDrHbpMlb4K8pHwXg" { + assert.Equal(t, "FnzJJ_6TDcgapyvs_-8vL2ImIWwehvRp_aWdhwS57U0", edge.Node.BundledIn.Id) + } + + if edge.Node.Id == "oLQYpPmfv35ZGn5Vv0_H2GKLoyY5bnFAChngzjLiCEs" { + assert.Equal(t, "D0i_pTTxNo_e1csmGSI-rJvKz6xi61ksQVjLEig568E", edge.Node.BundledIn.Id) + } + } + + t.Log(res) + +} diff --git a/argraphql/schema.graphql b/argraphql/schema.graphql new file mode 100644 index 0000000..bae51b6 --- /dev/null +++ b/argraphql/schema.graphql @@ -0,0 +1,241 @@ +# Arweave Gateway +# Copyright (C) 2022 Permanent Data Solutions, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +type Query { + # Get a transaction by its id + transaction(id: ID!): Transaction + + # Get a paginated set of matching transactions using filters. + transactions( + # Find transactions from a list of ids. + ids: [ID!] + + # Find transactions from a list of owner wallet addresses, or wallet owner public keys. + owners: [String!] + + # Find transactions from a list of recipient wallet addresses. + recipients: [String!] + + # Find transactions using tags. + tags: [TagFilter!] + + # Find data items from the given data bundles. + # See: https://github.com/ArweaveTeam/arweave-standards/blob/master/ans/ANS-104.md + bundledIn: [ID!] + + # Find transactions within a given block height range. + block: BlockFilter + + # Result page size (max: 100) + first: Int = 10 + + # A pagination cursor value, for fetching subsequent pages from a result set. + after: String + + # Optionally specify the result sort order. + sort: SortOrder = HEIGHT_DESC + ): TransactionConnection! + block(id: String): Block + blocks( + # Find blocks from a list of ids. + ids: [ID!] + + # Find blocks within a given block height range. + height: BlockFilter + + # Result page size (max: 100) + first: Int = 10 + + # A pagination cursor value, for fetching subsequent pages from a result set. + after: String + + # Optionally specify the result sort order. + sort: SortOrder = HEIGHT_DESC + ): BlockConnection! +} + +# Optionally reverse the result sort order from `HEIGHT_DESC` (default) to `HEIGHT_ASC`. +enum SortOrder { + # Results are sorted by the transaction block height in ascending order, with the oldest transactions appearing first, and the most recent and pending/unconfirmed appearing last. + HEIGHT_ASC + + # Results are sorted by the transaction block height in descending order, with the most recent and unconfirmed/pending transactions appearing first. + HEIGHT_DESC +} + +# Find transactions with the folowing tag name and value +input TagFilter { + # The tag name + name: String! + + # An array of values to match against. If multiple values are passed then transactions with _any_ matching tag value from the set will be returned. + # + # e.g. + # + # \`{name: "app-name", values: ["app-1"]}\` + # + # Returns all transactions where the \`app-name\` tag has a value of \`app-1\`. + # + # \`{name: "app-name", values: ["app-1", "app-2", "app-3"]}\` + # + # Returns all transactions where the \`app-name\` tag has a value of either \`app-1\` _or_ \`app-2\` _or_ \`app-3\`. + values: [String!]! + + # The operator to apply to to the tag filter. Defaults to EQ (equal). + op: TagOperator = EQ +} + +# Find blocks within a given range +input BlockFilter { + # Minimum block height to filter from + min: Int + + # Maximum block height to filter to + max: Int +} + +# Paginated result set using the GraphQL cursor spec, +# see: https://relay.dev/graphql/connections.htm. +type BlockConnection { + pageInfo: PageInfo! + edges: [BlockEdge!]! +} + +# Paginated result set using the GraphQL cursor spec. +type BlockEdge { + # The cursor value for fetching the next page. + # + # Pass this to the \`after\` parameter in \`blocks(after: $cursor)\`, the next page will start from the next item after this. + cursor: String! + + # A block object. + node: Block! +} + +# Paginated result set using the GraphQL cursor spec, +# see: https://relay.dev/graphql/connections.htm. +type TransactionConnection { + pageInfo: PageInfo! + edges: [TransactionEdge!]! +} + +# Paginated result set using the GraphQL cursor spec. +type TransactionEdge { + # The cursor value for fetching the next page. + # + # Pass this to the \`after\` parameter in \`transactions(after: $cursor)\`, the next page will start from the next item after this. + cursor: String! + + # A transaction object. + node: Transaction! +} + +# Paginated page info using the GraphQL cursor spec. +type PageInfo { + hasNextPage: Boolean! +} + +type Transaction { + id: ID! + anchor: String! + signature: String! + recipient: String! + owner: Owner! + fee: Amount! + quantity: Amount! + data: MetaData! + tags: [Tag!]! + + # Transactions with a null block are recent and unconfirmed, if they aren't mined into a block within 60 minutes they will be removed from results. + block: Block + + # @deprecated Don't use, kept for backwards compatability only! + parent: Parent @deprecated(reason: "Use `bundledIn`") + + # For bundled data items this references the containing bundle ID. + # See: https://github.com/ArweaveTeam/arweave-standards/blob/master/ans/ANS-104.md + bundledIn: Bundle +} + +# The parent transaction for bundled transactions, +# see: https://github.com/ArweaveTeam/arweave-standards/blob/master/ans/ANS-102.md. +type Parent { + id: ID! +} + +# The data bundle containing the current data item. +# See: https://github.com/ArweaveTeam/arweave-standards/blob/master/ans/ANS-104.md. +type Bundle { + # ID of the containing data bundle. + id: ID! +} + +type Block { + # The block ID. + id: ID! + + # The block timestamp (UTC). + timestamp: Int! + + # The block height. + height: Int! + + # The previous block ID. + previous: ID! +} + +# Basic metadata about the transaction data payload. +type MetaData { + # Size of the associated data in bytes. + size: String! + + # Type is derrived from the \`content-type\` tag on a transaction. + type: String +} + +# Representation of a value transfer between wallets, in both winson and ar. +type Amount { + # Amount as a winston string e.g. \`"1000000000000"\`. + winston: String! + + # Amount as an AR string e.g. \`"0.000000000001"\`. + ar: String! +} + +# Representation of a transaction owner. +type Owner { + # The owner's wallet address. + address: String! + + # The owner's public key as a base64url encoded string. + key: String! +} + +type Tag { + # UTF-8 tag name + name: String! + + # UTF-8 tag value + value: String! +} + +# The operator to apply to a tag value. +enum TagOperator { + # Equal + EQ + + # Not equal + NEQ +} diff --git a/arseeding.go b/arseeding.go index c139022..a319034 100644 --- a/arseeding.go +++ b/arseeding.go @@ -2,6 +2,7 @@ package arseeding import ( "context" + "github.com/everFinance/arseeding/cache" "github.com/everFinance/arseeding/config" "github.com/everFinance/arseeding/rawdb" "github.com/everFinance/arseeding/schema" @@ -47,6 +48,7 @@ type Arseeding struct { expectedRange int64 // default 50 block customTags []types.Tag locker sync.RWMutex + localCache *cache.Cache } func New( @@ -149,6 +151,11 @@ func New( a.KWriters = kwriters } + localCache, err := cache.NewLocalCache(60 * time.Minute) + if err != nil { + log.Error("NewLocalCache", "err", err) + } + a.localCache = localCache return a } diff --git a/cache/bigcache.go b/cache/bigcache.go new file mode 100644 index 0000000..ac08d87 --- /dev/null +++ b/cache/bigcache.go @@ -0,0 +1,29 @@ +package cache + +import ( + "context" + "github.com/allegro/bigcache/v3" + "time" +) + +type BigCache struct { + Cache *bigcache.BigCache +} + +func NewBigCache(allKeysExpTime time.Duration) (*BigCache, error) { + + cache, err := bigcache.New(context.Background(), bigcache.DefaultConfig(allKeysExpTime)) + + if err != nil { + return nil, err + } + return &BigCache{Cache: cache}, nil +} + +func (s *BigCache) Set(key string, entry []byte) (err error) { + return s.Cache.Set(key, entry) +} + +func (s *BigCache) Get(key string) ([]byte, error) { + return s.Cache.Get(key) +} diff --git a/cache/cache.go b/cache/cache.go new file mode 100644 index 0000000..06d44b2 --- /dev/null +++ b/cache/cache.go @@ -0,0 +1,21 @@ +package cache + +import "time" + +type Cache struct { + Cache ICache +} + +type ICache interface { + Set(key string, entry []byte) error + + Get(key string) ([]byte, error) +} + +func NewLocalCache(allKeysExpTime time.Duration) (*Cache, error) { + cache, err := NewBigCache(allKeysExpTime) + if err != nil { + return nil, err + } + return &Cache{Cache: cache}, nil +} diff --git a/cache/cache_test.go b/cache/cache_test.go new file mode 100644 index 0000000..8857ac5 --- /dev/null +++ b/cache/cache_test.go @@ -0,0 +1,32 @@ +package cache + +import ( + "testing" + "time" +) + +func TestNewLocalCache(t *testing.T) { + + cache, err := NewLocalCache(time.Second * 1) + + if err != nil { + t.Error(err) + } + + err = cache.Cache.Set("test-key", []byte("test-data")) + + if err != nil { + t.Error(err) + } + + data, err := cache.Cache.Get("test-key") + + if err != nil { + t.Error(err) + } + + if string(data) != "test-data" { + t.Error("data not match") + } + +} diff --git a/cmd/main.go b/cmd/main.go index 4fd8d2f..0bb7986 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -27,7 +27,7 @@ func main() { &cli.StringFlag{Name: "ar_node", Value: "https://arweave.net", EnvVars: []string{"AR_NODE"}}, &cli.StringFlag{Name: "pay", Value: "https://api-dev.everpay.io", Usage: "pay url", EnvVars: []string{"PAY"}}, &cli.BoolFlag{Name: "no_fee", Value: false, EnvVars: []string{"NO_FEE"}}, - &cli.BoolFlag{Name: "manifest", Value: false, EnvVars: []string{"MANIFEST"}}, + &cli.BoolFlag{Name: "manifest", Value: true, EnvVars: []string{"MANIFEST"}}, &cli.IntFlag{Name: "bundle_interval", Value: 120, Usage: "bundle tx on chain time interval(seconds)", EnvVars: []string{"BUNDLE_INTERVAL"}}, &cli.BoolFlag{Name: "use_s3", Value: false, Usage: "run with s3 store", EnvVars: []string{"USE_S3"}}, diff --git a/go.mod b/go.mod index 1f07c9b..3c0640d 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( github.com/aliyun/aliyun-oss-go-sdk v2.2.6+incompatible github.com/aws/aws-sdk-go v1.27.0 github.com/ethereum/go-ethereum v1.10.20 - github.com/everFinance/goar v1.5.5 - github.com/everFinance/goether v1.1.8 + github.com/everFinance/goar v1.5.7 + github.com/everFinance/goether v1.1.9 github.com/gin-gonic/gin v1.7.7 github.com/go-co-op/gocron v1.11.0 github.com/google/uuid v1.3.0 @@ -16,10 +16,10 @@ require ( github.com/panjf2000/ants/v2 v2.6.0 github.com/prometheus/client_golang v1.12.2 github.com/shopspring/decimal v1.2.0 - github.com/stretchr/testify v1.8.0 - github.com/tidwall/gjson v1.14.1 + github.com/stretchr/testify v1.8.2 + github.com/tidwall/gjson v1.14.4 github.com/ulule/limiter/v3 v3.10.0 - github.com/urfave/cli/v2 v2.10.2 + github.com/urfave/cli/v2 v2.24.4 go.etcd.io/bbolt v1.3.6 gopkg.in/h2non/gentleman.v2 v2.0.5 gorm.io/datatypes v1.0.1 @@ -29,8 +29,10 @@ require ( ) require ( - github.com/everFinance/go-everpay v0.0.7 - github.com/gabriel-vasile/mimetype v1.4.2 + github.com/Khan/genqlient v0.6.0 + github.com/allegro/bigcache/v3 v3.1.0 + github.com/everFinance/go-everpay v0.0.8 + github.com/everFinance/goarns v0.0.3 github.com/segmentio/kafka-go v0.4.40 go.mongodb.org/mongo-driver v1.11.4 ) @@ -67,8 +69,8 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.15.9 // indirect github.com/leodido/go-urn v1.2.0 // indirect - github.com/mattn/go-colorable v0.1.11 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-sqlite3 v1.14.5 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -84,23 +86,24 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect github.com/tidwall/match v1.1.1 // indirect - github.com/tidwall/pretty v1.2.0 // indirect + github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.4 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.0 // indirect github.com/ugorji/go/codec v1.1.7 // indirect + github.com/vektah/gqlparser/v2 v2.5.1 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect golang.org/x/crypto v0.3.0 // indirect - golang.org/x/net v0.8.0 // indirect + golang.org/x/net v0.9.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect - google.golang.org/protobuf v1.26.0 // indirect + google.golang.org/protobuf v1.28.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/manifest.go b/manifest.go index e20eeb5..12d1dac 100644 --- a/manifest.go +++ b/manifest.go @@ -1,8 +1,15 @@ package arseeding import ( + "context" "encoding/json" + "errors" + "fmt" + "github.com/everFinance/arseeding/argraphql" + "github.com/everFinance/goar" + "gopkg.in/h2non/gentleman.v2" "io" + "net/http" "os" "strings" @@ -65,6 +72,24 @@ func getArTxOrItemData(id string, db *Store) (decodeTags []types.Tag, binaryRead return nil, nil, nil, schema.ErrLocalNotExist } +func getArTxOrItemDataForManifest(id string, db *Store, s *Arseeding) (decodeTags []types.Tag, binaryReader *os.File, data []byte, err error) { + + // find bundle item form local + decodeTags, binaryReader, data, err = getArTxOrItemData(id, db) + + // if err equal ErrLocalNotExist, put task to queue to sync data + if err == schema.ErrLocalNotExist { + + // registerTask + if err = s.registerTask(id, schema.TaskTypeSyncManifest); err != nil { + log.Error("registerTask sync_manifest error", "err", err) + } + return nil, nil, nil, schema.ErrLocalNotExist + } + + return decodeTags, binaryReader, data, err +} + func getArTxOrItemTags(id string, db *Store) (decodeTags []types.Tag, err error) { itemMeta, err := db.LoadItemMeta(id) if err == nil { @@ -110,3 +135,115 @@ func parseBundleItem(binaryReader *os.File, itemBinary []byte) (item *types.Bund } return } + +func syncManifestData(id string, s *Arseeding) (err error) { + + // get manifest data + data, contentType, err := getRawById(id) + if err != nil { + return err + } + + log.Debug("syncManifestData get raw end ", "id", id, "contentType", contentType, "data", string(data)) + var bundleInItemsMap = make(map[string][]string) + var L1Artxs []string + var itemIds []string + + itemIds = append(itemIds, id) + + // if contentType == "application/x.arweave-manifest+json" parse it + if contentType == "application/x.arweave-manifest+json" { + + //is manifest data parse it + mani := schema.ManifestData{} + if err := json.Unmarshal(data, &mani); err != nil { + return err + } + + log.Debug("total txId in manifest", "len", len(mani.Paths)) + // for each path in manifest + for _, txId := range mani.Paths { + itemIds = append(itemIds, txId.TxId) + } + } + + log.Debug("total txId in manifest", "len", len(itemIds)) + + // query itemIds from graphql + gq := argraphql.NewARGraphQL("https://arweave.net/graphql", http.Client{}) + + total := len(itemIds) + + var txs []argraphql.BatchGetItemsBundleInTransactionsTransactionConnectionEdgesTransactionEdge + // 90 items per query + for i := 0; i < total; i += 90 { + end := i + 90 + if end > total { + end = total + } + log.Debug("BatchGetItemsBundleIn", "start", i, "end", end) + resp, err := gq.BatchGetItemsBundleIn(context.Background(), itemIds[i:end], 90, "") + if err != nil { + return errors.New("BatchGetItemsBundleIn error:" + err.Error()) + } + txs = append(txs, resp.Transactions.Edges...) + } + + // for each txs to bundleInItemsMap + for _, tx := range txs { + if tx.Node.BundledIn.Id == "" { + L1Artxs = append(L1Artxs, tx.Node.Id) + } else { + bundleInItemsMap[tx.Node.BundledIn.Id] = append(bundleInItemsMap[tx.Node.BundledIn.Id], tx.Node.Id) + } + } + + log.Debug("syncManifestData bundleInItemsMap", "bundleInItemsMap", bundleInItemsMap, "L1Artxs", L1Artxs) + // get bundle item form goar + c := goar.NewClient("https://arweave.net") + for bundleId, itemIds := range bundleInItemsMap { + + log.Debug("syncManifestData GetBundleItems ", "bundleId", bundleId, "itemIds", itemIds) + // GetBundleItems + items, err := c.GetBundleItems(bundleId, itemIds) + + if err != nil { + return errors.New("GetBundleItems error:" + err.Error()) + } + + // for each item + for _, item := range items { + log.Debug("syncManifestData save ", "item id", item.Id) + // save item to store + err = s.saveItem(*item) + + if err != nil { + return errors.New("saveItem error:" + err.Error()) + } + } + + } + + return err +} + +func getRawById(id string) (data []byte, contentType string, err error) { + + var arGateway = "https://arweave.net" + client := gentleman.New().URL(arGateway) + + res, err := client.Get().AddPath(fmt.Sprintf("/raw/%s", id)).Send() + + if err != nil { + return nil, "", err + } + + if !res.Ok { + return nil, "", fmt.Errorf("get raw data statuscode: %d id: %s", res.StatusCode, id) + } + + contentType = res.Header.Get("Content-Type") + data = res.Bytes() + + return +} diff --git a/manifets_test.go b/manifets_test.go new file mode 100644 index 0000000..acc6d98 --- /dev/null +++ b/manifets_test.go @@ -0,0 +1,15 @@ +package arseeding + +import "testing" + +func Test_getRawById(t *testing.T) { + data, contentType, err := getRawById("arDRw5qt51v4pOV9TrQXKJM2iLK-c39dvs2K-7b3oDk") + + if err != nil { + t.Error(err) + } + + t.Log(err) + t.Log(contentType) + t.Log(string(data)) +} diff --git a/middleware.go b/middleware.go index 2aaa6c7..b2858aa 100644 --- a/middleware.go +++ b/middleware.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/everFinance/arseeding/schema" "github.com/everFinance/goar/utils" + "github.com/everFinance/goarns" "github.com/gin-gonic/gin" "github.com/ulule/limiter/v3" mgin "github.com/ulule/limiter/v3/drivers/middleware/gin" @@ -15,6 +16,7 @@ import ( "os" "regexp" "strings" + "time" ) var ( @@ -71,7 +73,12 @@ func CORSMiddleware() gin.HandlerFunc { } } -func ManifestMiddleware(wdb *Wdb, store *Store) gin.HandlerFunc { +func ManifestMiddleware(s *Arseeding) gin.HandlerFunc { + + wdb := s.wdb + store := s.store + localCache := s.localCache + return func(c *gin.Context) { prefixUri := getRequestSandbox(c.Request.Host) if len(prefixUri) > 0 && c.Request.Method == "GET" { @@ -99,6 +106,7 @@ func ManifestMiddleware(wdb *Wdb, store *Store) gin.HandlerFunc { c.Abort() return } + _, dataReader, mfData, err := getArTxOrItemData(mfId, store) defer func() { if dataReader != nil { @@ -129,6 +137,121 @@ func ManifestMiddleware(wdb *Wdb, store *Store) gin.HandlerFunc { c.Data(http.StatusOK, fmt.Sprintf("%s; charset=utf-8", getTagValue(tags, schema.ContentType)), data) return } + + // Gateway logic + currentHost := c.Request.Host + domain := getSubDomain(c.Request.Host) + // default true + notApiHost := true + apiHostList := []string{ + "seed-dev.everpay.io", + "arseed.web3infra.dev", + "web3infura.io", + "arweave.world", + "arweave.asia", + } + + log.Debug("middleware", "currentHost", currentHost) + for _, b := range apiHostList { + + if currentHost == b { + // if current host in apiHostList, notApiHost is false + notApiHost = false + break + } + } + + // if domain is not empty and method is get and not in apiHostList + if len(domain) > 0 && c.Request.Method == "GET" && notApiHost { + + txId := "" + keyPrefix := "txId_" + + // get txId by localCache + value, err := localCache.Cache.Get(keyPrefix + domain) + if err != nil { + log.Info("middleware get localCache ", "error:", err) + } + + if value != nil { + txId = string(value) + } + + // if txId is empty, get txId by sdk + if txId == "" { + + // todo config or variable + dreUrl := "https://dre-3.warp.cc" + arNSAddress := "bLAgYxAdX2Ry-nt6aH2ixgvJXbpsEYm28NgJgyqfs-U" + timeout := 10 * time.Second + + a := goarns.NewArNS(dreUrl, arNSAddress, timeout) + + txId, err = a.QueryLatestRecord(domain) + if err != nil { + c.Abort() + internalErrorResponse(c, err.Error()) + return + } + + // set cache + err = localCache.Cache.Set(keyPrefix+domain, []byte(txId)) + + if err != nil { + log.Error("middleware set localCache", "error", err) + } + } + + log.Debug(fmt.Sprintf("permaweb domian: %s txId: %s", domain, txId)) + decodeTags, dataReader, mfData, err := getArTxOrItemDataForManifest(txId, store, s) + defer func() { + if dataReader != nil { + dataReader.Close() + os.Remove(dataReader.Name()) + } + }() + + // if err equal to schema.ErrLocalNotExist, return 200 and "syncing data please wait some minutes and fresh the page" + if err == schema.ErrLocalNotExist { + c.Abort() + c.JSON(http.StatusOK, gin.H{ + "msg": "syncing data please wait some minutes and fresh the page", + }) + return + } + + if err != nil { + c.Abort() + internalErrorResponse(c, err.Error()) + return + } + if dataReader != nil { + mfData, err = io.ReadAll(dataReader) + if err != nil { + c.Abort() + internalErrorResponse(c, err.Error()) + return + } + } + + // if content type is text/html, return mfData + if getTagValue(decodeTags, schema.ContentType) == "text/html" { + c.Abort() + c.Data(http.StatusOK, fmt.Sprintf("%s; charset=utf-8", getTagValue(decodeTags, schema.ContentType)), mfData) + return + } + + tags, data, err := handleManifest(mfData, c.Request.URL.Path, store) + if err != nil { + c.Abort() + internalErrorResponse(c, err.Error()) + return + } + c.Abort() + c.Data(http.StatusOK, fmt.Sprintf("%s; charset=utf-8", getTagValue(tags, schema.ContentType)), data) + return + + } c.Next() } } @@ -166,3 +289,7 @@ func replaceId(txId string) string { } return string(byteArr) } + +func getSubDomain(domain string) string { + return strings.Split(domain, ".")[0] +} diff --git a/middleware_test.go b/middleware_test.go index cd01617..43d4884 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -8,3 +8,10 @@ func TestSandboxMiddleware(t *testing.T) { res := getRequestSandbox(host) t.Log(res) } + +func TestGetSubDomain(t *testing.T) { + host := "cookbook.arseed.web3infura.io" + + res := getSubDomain(host) + t.Log(res) +} diff --git a/schema/task.go b/schema/task.go index 080d980..4caab11 100644 --- a/schema/task.go +++ b/schema/task.go @@ -4,6 +4,7 @@ const ( TaskTypeBroadcast = "broadcast" // include tx and tx data TaskTypeBroadcastMeta = "broadcast_meta" // not include tx data TaskTypeSync = "sync" + TaskTypeSyncManifest = "sync_manifest" // sync manifest ) type Task struct { diff --git a/sdk/manifest_test.go b/sdk/manifest_test.go index a3fd99d..ae92205 100644 --- a/sdk/manifest_test.go +++ b/sdk/manifest_test.go @@ -10,7 +10,7 @@ func TestSDK_UploadFolderAndPay(t *testing.T) { priKey := "1d8bdd0d2f1e73dffe1111111118325b7e195669541f76559760ef615a588be3" eccSigner, err := goether.NewSigner(priKey) assert.NoError(t, err) - seedUrl := "https://arseed-dev.web3infra.dev" + seedUrl := "https://seed-dev.everpay.io" payUrl := "https://api.everpay.io" sdk, err := NewSDK(seedUrl, payUrl, eccSigner) assert.NoError(t, err) diff --git a/task.go b/task.go index 442c65f..ace90f2 100644 --- a/task.go +++ b/task.go @@ -27,7 +27,10 @@ func (s *Arseeding) processTask(taskId string) { err = s.broadcastTxTask(arId) case schema.TaskTypeBroadcastMeta: err = s.broadcastTxMetaTask(arId) + case schema.TaskTypeSyncManifest: + err = s.syncManifestTask(arId) } + if err != nil { log.Error("process task failed", "err", err, "taskId", taskId) } @@ -147,3 +150,23 @@ func (s *Arseeding) setProcessedTask(arId string, tktype string) error { // remove pending pool return s.store.DelPendingPoolTaskId(taskId) } + +func (s *Arseeding) syncManifestTask(arId string) (err error) { + + if s.taskMg.IsClosed(arId, schema.TaskTypeSyncManifest) { + return + } + + err = syncManifestData(arId, s) + + if err == nil { + s.taskMg.IncSuccessed(arId, schema.TaskTypeSyncManifest) + closeErr := s.taskMg.CloseTask(arId, schema.TaskTypeSyncManifest) + + if closeErr != nil { + log.Error("s.taskMg.CloseTask(arId, schema.TaskTypeSyncManifest)", "err", closeErr, "arId", arId) + } + } + return err + +}