feat: Use IFPS gateways instead of ipfs shell (#502)

* fix: No not use ipfs shell to download ipfs files. * fix: Upload context file to testbed
0xPolygonID · Oct 9, 2023 · 54c52c1 · 54c52c1
1 parent caa6e8a
commit 54c52c1
Show file tree

Hide file tree

Showing 15 changed files with 48 additions and 26 deletions.
diff --git a/cmd/notifications/main.go b/cmd/notifications/main.go
@@ -8,8 +8,6 @@ import (
 	"syscall"
 
 	vault "github.com/hashicorp/vault/api"
-	"github.com/iden3/go-schema-processor/v2/loaders"
-	shell "github.com/ipfs/go-ipfs-api"
 
 	"github.com/polygonid/sh-id-platform/internal/buildinfo"
 	"github.com/polygonid/sh-id-platform/internal/config"
@@ -19,6 +17,7 @@ import (
 	"github.com/polygonid/sh-id-platform/internal/db"
 	"github.com/polygonid/sh-id-platform/internal/gateways"
 	"github.com/polygonid/sh-id-platform/internal/kms"
+	"github.com/polygonid/sh-id-platform/internal/loader"
 	"github.com/polygonid/sh-id-platform/internal/log"
 	"github.com/polygonid/sh-id-platform/internal/providers"
 	"github.com/polygonid/sh-id-platform/internal/redis"
@@ -132,7 +131,7 @@ func newCredentialsService(cfg *config.Configuration, storage *db.Storage, cache
 	rhsp := reverse_hash.NewRhsPublisher(nil, false)
 
 	// TODO: Cache only if cfg.APIUI.SchemaCache == true
-	schemaLoader := loaders.NewDocumentLoader(shell.NewShell(cfg.IPFS.GatewayURL), cfg.IPFS.GatewayURL)
+	schemaLoader := loader.NewDocumentLoader(cfg.IPFS.GatewayURL)
 
 	mtService := services.NewIdentityMerkleTrees(mtRepository)
 	qrService := services.NewQrStoreService(cachex)

diff --git a/cmd/pending_publisher/main.go b/cmd/pending_publisher/main.go
@@ -11,8 +11,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/ethclient"
 	vault "github.com/hashicorp/vault/api"
-	"github.com/iden3/go-schema-processor/v2/loaders"
-	shell "github.com/ipfs/go-ipfs-api"
 
 	"github.com/polygonid/sh-id-platform/internal/buildinfo"
 	"github.com/polygonid/sh-id-platform/internal/config"
@@ -21,6 +19,7 @@ import (
 	"github.com/polygonid/sh-id-platform/internal/db"
 	"github.com/polygonid/sh-id-platform/internal/gateways"
 	"github.com/polygonid/sh-id-platform/internal/kms"
+	"github.com/polygonid/sh-id-platform/internal/loader"
 	"github.com/polygonid/sh-id-platform/internal/log"
 	"github.com/polygonid/sh-id-platform/internal/providers"
 	"github.com/polygonid/sh-id-platform/internal/redis"
@@ -75,7 +74,7 @@ func main() {
 	}(storage)
 
 	// TODO: Cache only if cfg.APIUI.SchemaCache == true
-	schemaLoader := loaders.NewDocumentLoader(shell.NewShell(cfg.IPFS.GatewayURL), cfg.IPFS.GatewayURL)
+	schemaLoader := loader.NewDocumentLoader(cfg.IPFS.GatewayURL)
 
 	var vaultCli *vault.Client
 	var vaultErr error

diff --git a/cmd/platform/main.go b/cmd/platform/main.go
@@ -14,9 +14,7 @@ import (
 	"github.com/go-chi/cors"
 	redis2 "github.com/go-redis/redis/v8"
 	vault "github.com/hashicorp/vault/api"
-	"github.com/iden3/go-schema-processor/v2/loaders"
 	proof "github.com/iden3/merkletree-proof"
-	shell "github.com/ipfs/go-ipfs-api"
 
 	"github.com/polygonid/sh-id-platform/internal/api"
 	"github.com/polygonid/sh-id-platform/internal/buildinfo"
@@ -27,6 +25,7 @@ import (
 	"github.com/polygonid/sh-id-platform/internal/gateways"
 	"github.com/polygonid/sh-id-platform/internal/health"
 	"github.com/polygonid/sh-id-platform/internal/kms"
+	"github.com/polygonid/sh-id-platform/internal/loader"
 	"github.com/polygonid/sh-id-platform/internal/log"
 	"github.com/polygonid/sh-id-platform/internal/providers"
 	"github.com/polygonid/sh-id-platform/internal/providers/blockchain"
@@ -75,7 +74,7 @@ func main() {
 	cachex := cache.NewRedisCache(rdb)
 
 	// TODO: Cache only if cfg.APIUI.SchemaCache == true
-	schemaLoader := loaders.NewDocumentLoader(shell.NewShell(cfg.IPFS.GatewayURL), cfg.IPFS.GatewayURL)
+	schemaLoader := loader.NewDocumentLoader(cfg.IPFS.GatewayURL)
 
 	var vaultCli *vault.Client
 	var vaultErr error

diff --git a/cmd/platform_ui/main.go b/cmd/platform_ui/main.go
@@ -20,9 +20,7 @@ import (
 	"github.com/iden3/go-iden3-auth/v2/pubsignals"
 	"github.com/iden3/go-iden3-auth/v2/state"
 	"github.com/iden3/go-iden3-core/v2/w3c"
-	"github.com/iden3/go-schema-processor/v2/loaders"
 	proof "github.com/iden3/merkletree-proof"
-	shell "github.com/ipfs/go-ipfs-api"
 
 	"github.com/polygonid/sh-id-platform/internal/api_ui"
 	"github.com/polygonid/sh-id-platform/internal/buildinfo"
@@ -34,6 +32,7 @@ import (
 	"github.com/polygonid/sh-id-platform/internal/gateways"
 	"github.com/polygonid/sh-id-platform/internal/health"
 	"github.com/polygonid/sh-id-platform/internal/kms"
+	"github.com/polygonid/sh-id-platform/internal/loader"
 	"github.com/polygonid/sh-id-platform/internal/log"
 	"github.com/polygonid/sh-id-platform/internal/providers"
 	"github.com/polygonid/sh-id-platform/internal/providers/blockchain"
@@ -82,7 +81,7 @@ func main() {
 	cachex := cache.NewRedisCache(rdb)
 
 	// TODO: Cache only if cfg.APIUI.SchemaCache == true
-	schemaLoader := loaders.NewDocumentLoader(shell.NewShell(cfg.IPFS.GatewayURL), cfg.IPFS.GatewayURL)
+	schemaLoader := loader.NewDocumentLoader(cfg.IPFS.GatewayURL)
 
 	var vaultCli *vault.Client
 	var vaultErr error

diff --git a/...b5dkz5ekDqBrYVVxBFQZoCbzamnmMUn9B8twCEgDL → ...b5dkz5ekDqBrYVVxBFQZoCbzamnmMUn9B8twCEgDL b/...b5dkz5ekDqBrYVVxBFQZoCbzamnmMUn9B8twCEgDL → ...b5dkz5ekDqBrYVVxBFQZoCbzamnmMUn9B8twCEgDL
diff --git a/...e3i3i5753DRyEnEw9jDSt6HhuFr43HwkKzQHRfsX9 → ...e3i3i5753DRyEnEw9jDSt6HhuFr43HwkKzQHRfsX9 b/...e3i3i5753DRyEnEw9jDSt6HhuFr43HwkKzQHRfsX9 → ...e3i3i5753DRyEnEw9jDSt6HhuFr43HwkKzQHRfsX9
diff --git a/...YA1K1xiFPHTUbDtVkV6HoKRCentZetfv4yLbrjwfY → ...YA1K1xiFPHTUbDtVkV6HoKRCentZetfv4yLbrjwfY b/...YA1K1xiFPHTUbDtVkV6HoKRCentZetfv4yLbrjwfY → ...YA1K1xiFPHTUbDtVkV6HoKRCentZetfv4yLbrjwfY
diff --git a/infrastructure/local/test_data/schema/ipfs/QmeMevwUeD7o6hjfmdaeFD1q4L84hSDiRjeXZLi1bZK1My b/infrastructure/local/test_data/schema/ipfs/QmeMevwUeD7o6hjfmdaeFD1q4L84hSDiRjeXZLi1bZK1My
@@ -0,0 +1,25 @@
+
+
+{
+  "@context": [
+    {
+      "@protected": true,
+      "@version": 1.1,
+      "id": "@id",
+      "type": "@type",
+      "testNewType": {
+        "@context": {
+          "@propagate": true,
+          "@protected": true,
+          "polygon-vocab": "urn:uuid:0a8092e3-7100-4068-ba67-fae502cc6e7b#",
+          "xsd": "http://www.w3.org/2001/XMLSchema#",
+          "testNewTypeInt": {
+            "@id": "polygon-vocab:testNewTypeInt",
+            "@type": "xsd:integer"
+          }
+        },
+        "@id": "urn:uuid:412d4b27-bae2-4c26-ac57-bacc141010d6"
+      }
+    }
+  ]
+}
diff --git a/internal/api/main_test.go b/internal/api/main_test.go
@@ -9,9 +9,7 @@ import (
 	"github.com/go-chi/chi/v5"
 	"github.com/hashicorp/vault/api"
 	"github.com/iden3/go-iden3-core/v2/w3c"
-	"github.com/iden3/go-schema-processor/v2/loaders"
 	"github.com/iden3/iden3comm/v2"
-	shell "github.com/ipfs/go-ipfs-api"
 	"github.com/piprate/json-gold/ld"
 
 	"github.com/polygonid/sh-id-platform/internal/config"
@@ -20,6 +18,7 @@ import (
 	"github.com/polygonid/sh-id-platform/internal/db/tests"
 	"github.com/polygonid/sh-id-platform/internal/errors"
 	"github.com/polygonid/sh-id-platform/internal/kms"
+	"github.com/polygonid/sh-id-platform/internal/loader"
 	"github.com/polygonid/sh-id-platform/internal/log"
 	"github.com/polygonid/sh-id-platform/internal/providers"
 	"github.com/polygonid/sh-id-platform/pkg/cache"
@@ -84,7 +83,7 @@ func TestMain(m *testing.M) {
 
 	cfg.ServerUrl = "https://testing.env/"
 
-	schemaLoader = loaders.NewDocumentLoader(shell.NewShell(ipfsGatewayURL), "")
+	schemaLoader = loader.NewDocumentLoader(ipfsGatewayURL)
 
 	m.Run()
 }

diff --git a/internal/api_ui/main_test.go b/internal/api_ui/main_test.go
@@ -11,9 +11,7 @@ import (
 	"github.com/go-chi/chi/v5"
 	vaultApi "github.com/hashicorp/vault/api"
 	"github.com/iden3/go-iden3-core/v2/w3c"
-	"github.com/iden3/go-schema-processor/v2/loaders"
 	"github.com/iden3/iden3comm/v2"
-	shell "github.com/ipfs/go-ipfs-api"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -88,7 +86,7 @@ func TestMain(m *testing.M) {
 
 	cfg.ServerUrl = "https://testing.env/"
 
-	schemaLoader = loaders.NewDocumentLoader(shell.NewShell(ipfsGatewayURL), "")
+	schemaLoader = loader.NewDocumentLoader(ipfsGatewayURL)
 
 	m.Run()
 }

diff --git a/internal/config/config.go b/internal/config/config.go
@@ -22,7 +22,7 @@ import (
 
 const (
 	CIConfigPath = "/home/runner/work/sh-id-platform/sh-id-platform/" // CIConfigPath variable contain the CI configuration path
-	ipfsGateway  = "https://ipfs.io"
+	ipfsGateway  = "https://cloudflare-ipfs.com"
 )
 
 // Configuration holds the project configuration

diff --git a/internal/core/services/tests/main_test.go b/internal/core/services/tests/main_test.go
@@ -6,8 +6,6 @@ import (
 	"testing"
 
 	"github.com/hashicorp/vault/api"
-	"github.com/iden3/go-schema-processor/v2/loaders"
-	shell "github.com/ipfs/go-ipfs-api"
 
 	"github.com/polygonid/sh-id-platform/internal/config"
 	"github.com/polygonid/sh-id-platform/internal/db"
@@ -76,7 +74,7 @@ func TestMain(m *testing.M) {
 	}
 	cachex = cache.NewMemoryCache()
 
-	docLoader = loaders.NewDocumentLoader(shell.NewShell(ipfsGatewayURL), "")
+	docLoader = loader.NewDocumentLoader(ipfsGatewayURL)
 
 	m.Run()
 }

diff --git a/internal/jsonschema/schema_test.go b/internal/jsonschema/schema_test.go
@@ -4,15 +4,15 @@ import (
 	"context"
 	"testing"
 
-	"github.com/iden3/go-schema-processor/v2/loaders"
-	shell "github.com/ipfs/go-ipfs-api"
 	"github.com/stretchr/testify/assert"
+
+	"github.com/polygonid/sh-id-platform/internal/loader"
 )
 
 func TestValidateCredentialSubject(t *testing.T) {
 	const ipfsGatewayURL = "http://127.0.0.1:8080"
 	ctx := context.Background()
-	ld := loaders.NewDocumentLoader(shell.NewShell(ipfsGatewayURL), "")
+	ld := loader.NewDocumentLoader(ipfsGatewayURL)
 
 	type config struct {
 		name              string

diff --git a/internal/loader/ipfs_test.go b/internal/loader/ipfs_test.go
@@ -40,7 +40,7 @@ func TestIpfsCID(t *testing.T) {
 			},
 		},
 		{
-			url: "https://gateway.ipfs.io/ipfs/QmUrDHtC3fGYg1CqWzrgbxU5tXeQa4y323h277m6hXX84k",
+			url: "https://cloudflare-ipfs.com/ipfs/QmUrDHtC3fGYg1CqWzrgbxU5tXeQa4y323h277m6hXX84k",
 			expected: expected{
 				cid: "",
 				err: errors.New("invalid ipfs url"),

diff --git a/internal/loader/loader.go b/internal/loader/loader.go
@@ -2,6 +2,7 @@ package loader
 
 import (
 	"github.com/iden3/go-schema-processor/processor"
+	"github.com/iden3/go-schema-processor/v2/loaders"
 	"github.com/piprate/json-gold/ld"
 )
 
@@ -15,3 +16,8 @@ type Loader interface {
 
 // Factory defines the interface that a loader constructor should satisfy
 type Factory func(url string) Loader
+
+// NewDocumentLoader returns a new ld.DocumentLoader that will use http ipfs gateway to download documents
+func NewDocumentLoader(ipfsGateway string) ld.DocumentLoader {
+	return loaders.NewDocumentLoader(nil, ipfsGateway)
+}