diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 230bfde..5026f39 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -20,17 +20,18 @@ jobs: steps: - uses: actions/checkout@v3 - uses: actions/setup-go@v3 - - name: Set up Go 1.19 + - name: Set up Go 1.20 uses: actions/setup-go@v3 with: - go-version: 1.19 + go-version: 1.20.5 id: go - run: go version - name: Lint run: | curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.53.3 - ./bin/golangci-lint run --timeout=3m + go mod tidy + ./bin/golangci-lint run -v --timeout=3m --go=1.20.5 # Optional: working directory, useful for monorepos # working-directory: somedir diff --git a/Dockerfile b/Dockerfile index dbfa47d..8280e2f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,13 +9,15 @@ RUN go mod download && CGO_ENABLED=0 GOOS=linux go build -a -ldflags="-s -w" -o FROM alpine:3.19 RUN mkdir /app WORKDIR /app -RUN apk update && apk add --no-cache bash=5.2.21-r0 +RUN apk update && apk add --no-cache bash nodejs npm git && npm install -g @web3-storage/w3cli COPY --from=builder --chmod=700 /build/ipfs-server /app +RUN apk del git && rm -rf /var/cache/apk/* /root/.npm /tmp/* + HEALTHCHECK --interval=10s --timeout=5s CMD wget --no-verbose --tries=1 --spider localhost:3001/health ENTRYPOINT [ "/bin/bash", "-l", "-c" ] -CMD [ "./ipfs-server -port 3001 -jwt $WEB3_JWT" ] +CMD [ "./ipfs-server -port 3001 -w3-agent-key $W3_AGENT_KEY -w3-delegation-file $W3_DELEGATION_FILE" ] # ipfs-pinner API server; EXPOSE 3001 @@ -26,4 +28,4 @@ EXPOSE 5001 # Web Gateway; can be exposed publicly with a proxy, e.g. as https://ipfs.example.org EXPOSE 8080 # Swarm Websockets; must be exposed publicly when the node is listening using the websocket transport (/ipX/.../tcp/8081/ws). -EXPOSE 8081 +EXPOSE 8081 \ No newline at end of file diff --git a/README.md b/README.md index 73a9132..c671395 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,17 @@ - [Upload a file](#upload-a-file) - [Download content (given cid)](#download-content-given-cid) - [Find the cid given some content](#find-the-cid-given-some-content) + - [migration to UCAN and capabilities setup](#migration-to-ucan-and-capabilities-setup) + - [setting up w3cli](#setting-up-w3cli) + - [installation](#installation) + - [login and check spaces](#login-and-check-spaces) + - [generate ucan key](#generate-ucan-key) + - [create delegation to store/add and upload/add](#create-delegation-to-storeadd-and-uploadadd) + - [communicate to the operator](#communicate-to-the-operator) + - [operator invocation](#operator-invocation) - [Running ipfs-pinner server with docker](#running-ipfs-pinner-server-with-docker) + - [Running the image](#running-the-image) + - [Building the docker image](#building-the-docker-image) - [Docker Volume setup](#docker-volume-setup) - [Port mapping setup](#port-mapping-setup) - [Development](#development) @@ -24,7 +34,7 @@ ## Introduction - A wrapper on top of ipfs node, utilising go-ipfs as a library. -- Extended support for custom file upload endpoints provided by pinata & web3.storage. +- Extended support for custom file upload endpoints provided by web3.storage. - Content archive file generation and lightweight deterministic CID generation on client side (using CARs). - It can be used as a go library (see `binary/main.go` for usage) or as a http server. @@ -40,18 +50,19 @@ To avoid this issue, the merkle DAG thus generated is exported into special file ## Running ipfs-pinner server -1. Set the environment variable `WEB3_JWT` +1. Get the agent key, did and delegation proof from Covalent -2. to start a server which listens for request on 3001 port, run: +2. build the server and run: ```bash -make clean server-dbg run +make clean server-dbg ``` NOTE: If you want more control over CLI params, you can run the server binary (after `make clean server-dbg`): ```bash ./build/bin/server -jwt -port 3001 +./build/bin/server -w3-agent-key -w3-delegation-file ``` NOTE: If you get some error when running this, check if the diagnostic is there in [known issues](#known-issues) @@ -107,6 +118,96 @@ There's a timeout (check code for value) for the download request, if it doesn't {"cid": "bafkreicszve3ewhhrgobm366mdctki2m2qwzide5e54zh5aifnesg3ofne"}% ``` + +## migration to UCAN and capabilities setup +- web3.storage is sunsetting its custom upload endpoint (on 9th January, 2024), and we need to migrate from using that to w3up. +- w3up uses UCAN which is a capabilities-based authorization system (learn more [here](https://web3.storage/docs/concepts/ucans-and-web3storage/)). +- In this setup, the "central account" (owned by Covalent) sets up a "space" (think namespace for data). The central account (controlled by the email) is delegated the capabilty to operate on this space. +- among other capabilties, the central account can delegate certain capabilities (like uploading to space) to other **agents**. This has to be done at our end, and scripts will be made available for it in this repo. +- once an agent is granted the capability, we share the credentials with the operators, who run ipfs-pinner with it, and can then upload or fetch. + + +### setting up w3cli + +- Create a web3.storage account in the [console](https://console.web3.storage/). +- Create a space which you want to use to upload artifacts. We want to use different spaces for different artifacts to keep a clear separation. + +We'll use w3cli to login and create a new space and register. + +#### installation +```bash +➜ npm install -g @web3-storage/w3cli + +➜ w3 --version +w3, 7.0.3 +``` + +#### login and check spaces +```bash +➜ w3 login sudeep@covalenthq.com + +➜ w3 space ls + did:key:z6MkgSK6VEu3bvrAFtYNyjsnzG7dVXzYi3yT5TasEgeaQrCe mock_artifacts + +➜ w3 space use did:key:z6MkgSK6VEu3bvrAFtYNyjsnzG7dVXzYi3yT5TasEgeaQrCe +did:key:z6MkgSK6VEu3bvrAFtYNyjsnzG7dVXzYi3yT5TasEgeaQrCe +``` + +The did key is the identifier for this space. Now let's generate some DIDs for an operator and delegate upload capabilities to it. + +#### generate ucan key +```bash +➜ npx ucan-key ed --json +{ + "did": "did:key:z6MkpzWw1fDZYMpESgVKFAT87SZAuHiCQZVBC3hmQjB18Nzj", + "key": "MgCbc48J8n+BMdzA4XxwYOaKmdu5Ov34jE71U8vV07IVIjO0BnJa05mNMcB8GSz1lib014QAhvAxorG6zACrstm6PBGA=" +} +``` + +#### create delegation to store/add and upload/add + +```bash +➜ w3 delegation create -c 'store/add' -c 'upload/add' did:key:z6MkpzWw1fDZYMpESgVKFAT87SZAuHiCQZVBC3hmQjB18Nzj -o proof.out +``` + + +Copy the output. This is the delegation string. + +#### communicate to the operator + +Provide the operator with the `did`, `key` string + `proof.out` file. These will be passed to operator's setup of the +ipfs-pinner, which can then make the delegations. + + +#### operator invocation + +the operator can pass the `key` for `-w3-agent-key` and proof file in `-w3-delegation-file` flag. + +```bash +go run server/main.go -w3-agent-key -w3-delegation-file ./proof.out +ipfs-pinner +ipfs-pinner Version: 0.1.16 +Architecture: arm64 +Go Version: go1.20.5 +Operating System: darwin +GOPATH=/Users/sudeep/go/ +GOROOT=/usr/local/go +2024/01/04 15:52:05 agent did: did:key:z6MkoLvhaiE9NRYs3vJcynCM8CeyP8hXduWhE5Ter2U2x93y +generating 2048-bit RSA keypair...done +peer identity: QmY49BMJdGneQjJAbTPrGSqaQcLjpCE1WFkRBP6XZEHd6i +2024/01/04 15:52:09 setting up w3up for uploads.... +2024/01/04 15:52:10 w3up agent did: did:key:z6MkoLvhaiE9NRYs3vJcynCM8CeyP8hXduWhE5Ter2U2x93y +2024/01/04 15:52:10 w3up space did: did:key:z6MkgSK6VEu3bvrAFtYNyjsnzG7dVXzYi3yT5TasEgeaQrCe +2024/01/04 15:52:10 w3up setup complete +2024/01/04 15:52:10 Listening... +2024/01/04 15:52:15 generated dag has root cid: bafybeigvijf76lcsjwcmkr6rmzovoiiqdog3muqs5vnplvf4jxh47shfiu +2024/01/04 15:52:15 car file location: /var/folders/w0/bf3y1c7d6ys15tq97ffk5qhw0000gn/T/3475885728.car +2024/01/04 15:53:06 w3 up output: {"root":{"/":"bafybeigvijf76lcsjwcmkr6rmzovoiiqdog3muqs5vnplvf4jxh47shfiu"}} +2024/01/04 15:53:28 uploaded file has root cid: bafybeigvijf76lcsjwcmkr6rmzovoiiqdog3muqs5vnplvf4jxh47shfiu +``` + + + ## Running ipfs-pinner server with docker We can also run the ipfs-pinner server via docker. @@ -117,34 +218,46 @@ for ipfs-pinner to function properly with docker, we need Docker run command should have: -- Volumes for data persistence +- Volumes for data persistence; - Port mappings -- JWT token passed in the env +- W3up agent key passed in the env + + +### Running the image + +Copy the delegation proof file into the ipfs directory which will be mapped onto the docker image. ```bash -docker buildx create --name builder --use --platform=linux/amd64,linux/arm64 && docker buildx build --platform=linux/amd64,linux/arm64 . -t gcr.io/covalent-project/ipfs-pinner:latest +mv proof.out /tmp/data/.ipfs/ ``` -Now, we can run the container: +Then one can run the docker container: ```bash docker container run --detach --name ipfs-pinner-instance \ --volume /tmp/data/.ipfs/:/root/.ipfs/ \ -p 3001:3001 \ - --env WEB3_JWT=$WEB3_JWT \ + --env W3_AGENT_KEY=$W3_AGENT_KEY \ + --env W3_DELEGATION_FILE=/root/.ipfs/proof.out ``` + +### Building the docker image +```bash +docker buildx create --name builder --use --platform=linux/amd64,linux/arm64 && docker buildx build --platform=linux/amd64,linux/arm64 . -t gcr.io/covalent-project/ipfs-pinner:latest +``` + ### Docker Volume setup -There's 1 docker volume that needs to be shared (and persisted) between the container and the host - this `~/.ipfs` directory needs to have its lifecycle unaffected by container lifecycle (since it contains the merklelized nodes, blockstore etc.), and so that is docker volume managed. +There's 1 docker volume that needs to be shared (and persisted) between the container and the host - the `~/.ipfs` directory, which needs to have its lifecycle unaffected by container lifecycle (since it contains the merklelized nodes, blockstore etc.), and so that is docker volume managed. ### Port mapping setup -:4001 : swarm port for p2p (currently disabled) -:8080 - http gateway (used by encapsulated ipfs-node) -:5001: local api (should be bound to 127.0.0.1 only, and must never be exposed publicly as it allows one to control the ipfs node; also used by encapsulated ipfs-node) -:3001: The ipfs-pinner itself exposes its REST API on this port +`:4001` - swarm port for p2p (currently disabled) +`:8080` - http gateway (used by encapsulated ipfs-node) +`:5001` - local api (should be bound to 127.0.0.1 only, and must never be exposed publicly as it allows one to control the ipfs node; also used by encapsulated ipfs-node) +`:3001` - The ipfs-pinner itself exposes its REST API on this port Out of the above, only the swarm port and the REST api port (3001) are essential. @@ -237,7 +350,7 @@ Users would sometimes want to maintain a different volume to fulfil large storag ipfs-pinner currently uses some known IPFS gateways to fetch content. These gateways are expected to be run and maintained for a long time, but if you need to update the gateways list due to one of the going down, or a more efficient gateway being introduced etc. you can change the list: ```bash -./build/bin/server -jwt -port 3001 -ipfs-gateway-urls "https://w3s.link/ipfs/%s,https://dweb.link/ipfs/%s,https://ipfs.io/ipfs/%s" +./build/bin/server -ipfs-gateway-urls "https://w3s.link/ipfs/%s,https://dweb.link/ipfs/%s,https://ipfs.io/ipfs/%s" ##OTHER PARAMS ``` The `-ipfs-gateways-urls` is a comma separated list of http urls with a `%s` present in it, which is formatted to replace the IPFS content identifier (CID) in it. diff --git a/binary/main.go b/binary/main.go index b4ef36b..fc93ae8 100644 --- a/binary/main.go +++ b/binary/main.go @@ -18,12 +18,7 @@ var WEB3_JWT = "WEB3_JWT" var UPLOAD_FILE = "./main.go" // uploading current file itself func main() { - token, present := os.LookupEnv(WEB3_JWT) - if !present { - log.Fatalf("token (%s) not found in env", WEB3_JWT) - } - - clientCreateReq := client.NewClientRequest(core.Web3Storage).BearerToken(token) + clientCreateReq := client.NewClientRequest(core.Web3Storage) // check if cid compute true works with car uploads nodeCreateReq := pinner.NewNodeRequest(clientCreateReq, []string{"https://w3s.link/ipfs/%s"}).CidVersion(1).CidComputeOnly(false) node := pinner.NewPinnerNode(*nodeCreateReq) diff --git a/core/support.go b/core/support.go index 0f22788..fae71a9 100644 --- a/core/support.go +++ b/core/support.go @@ -13,11 +13,11 @@ const ( Web3Storage PinningService = "web3.storage" Other PinningService = "other" // IpfsPinnerVersionMajor is Major version component of the current release - IpfsPinnerVersionMajor = 0 + IpfsPinnerVersionMajor = 1 // IpfsPinnerVersionMinor is Minor version component of the current release - IpfsPinnerVersionMinor = 1 + IpfsPinnerVersionMinor = 0 // IpfsPinnerVersionPatch is Patch version component of the current release - IpfsPinnerVersionPatch = 16 + IpfsPinnerVersionPatch = 0 clientIdentifier = "ipfs-pinner" // Client identifier to advertise over the network ) diff --git a/go.mod b/go.mod index 990952e..d841937 100644 --- a/go.mod +++ b/go.mod @@ -15,8 +15,9 @@ require ( github.com/multiformats/go-multibase v0.2.0 github.com/multiformats/go-multihash v0.2.3 github.com/pkg/errors v0.9.1 + github.com/web3-storage/go-ucanto v0.1.0 github.com/ybbus/httpretry v1.0.2 - golang.org/x/oauth2 v0.8.0 + golang.org/x/oauth2 v0.15.0 ) require ( @@ -185,13 +186,13 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.24.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/crypto v0.14.0 // indirect + golang.org/x/crypto v0.16.0 // indirect golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.17.0 // indirect + golang.org/x/net v0.19.0 // indirect golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.11.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gonum.org/v1/gonum v0.13.0 // indirect diff --git a/go.sum b/go.sum index bff7c86..3166709 100644 --- a/go.sum +++ b/go.sum @@ -745,6 +745,8 @@ github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsX github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/web3-storage/go-ucanto v0.1.0 h1:Hg6jO7OLLeDLtmseQZWQGBFJMp+5t5OWAFVL5Y3LsOs= +github.com/web3-storage/go-ucanto v0.1.0/go.mod h1:XD6zahQ8HLh8Z2CSK7xYcTR0A7oCVYXTZB7fFtYqGF8= github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc h1:BCPnHtcboadS0DvysUuJXZ4lWVv5Bh5i7+tbIyi+ck4= github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM= github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIfyDmc1Em5GqlNRzcdtlv4HTNmdpt7XH0= @@ -846,8 +848,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -927,8 +929,8 @@ golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -937,8 +939,8 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= -golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= +golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ= +golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1011,8 +1013,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1026,8 +1028,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/pinclient/client_create_request.go b/pinclient/client_create_request.go index 1f9e5ca..bb3d041 100644 --- a/pinclient/client_create_request.go +++ b/pinclient/client_create_request.go @@ -4,14 +4,20 @@ import ( "net/http" "github.com/covalenthq/ipfs-pinner/core" + "github.com/web3-storage/go-ucanto/did" ) type ClientCreateRequest struct { ps core.PinningService pinningServiceBaseUrl string filePinBaseUrl string - bearerToken string - httpClient *http.Client + //bearerToken string + + W3_AgentKey string + W3_AgentDid did.DID + W3_DelegationProofPath string + + httpClient *http.Client } func NewClientRequest(ps core.PinningService) ClientCreateRequest { @@ -21,8 +27,18 @@ func NewClientRequest(ps core.PinningService) ClientCreateRequest { return request } -func (r ClientCreateRequest) BearerToken(token string) ClientCreateRequest { - r.bearerToken = token +func (r ClientCreateRequest) W3AgentKey(key string) ClientCreateRequest { + r.W3_AgentKey = key + return r +} + +func (r ClientCreateRequest) W3AgentDid(did did.DID) ClientCreateRequest { + r.W3_AgentDid = did + return r +} + +func (r ClientCreateRequest) DelegationProofPath(proofPath string) ClientCreateRequest { + r.W3_DelegationProofPath = proofPath return r } diff --git a/pinclient/pinclient.go b/pinclient/pinclient.go index e48cf6c..d2351f2 100644 --- a/pinclient/pinclient.go +++ b/pinclient/pinclient.go @@ -12,32 +12,30 @@ import ( core "github.com/covalenthq/ipfs-pinner/core" ihttp "github.com/covalenthq/ipfs-pinner/http" "github.com/covalenthq/ipfs-pinner/openapi" + "github.com/covalenthq/ipfs-pinner/w3up" "github.com/ipfs/go-cid" "github.com/multiformats/go-multibase" "github.com/pkg/errors" - - logging "github.com/ipfs/go-log/v2" ) -var logger = logging.Logger("ipfs-pinner") - const UserAgent = "ipfs-pinner" type Client struct { client *openapi.APIClient ps core.PinningService cidVersion int + w3up *w3up.W3up } -func NewClient(request ClientCreateRequest, cidVersion int) PinServiceAPI { +func NewClient(request ClientCreateRequest, cidVersion int, w3up *w3up.W3up) PinServiceAPI { // assuming we are getting a supported pinning service request config := openapi.NewConfiguration() if request.httpClient == nil { request.httpClient = ihttp.NewHttpClient(nil) } config.UserAgent = UserAgent - bearer := fmt.Sprintf("Bearer %s", request.bearerToken) - config.AddDefaultHeader("Authorization", bearer) + //bearer := fmt.Sprintf("Bearer %s", request.bearerToken) + //config.AddDefaultHeader("Authorization", bearer) config.Servers = openapi.ServerConfigurations{ openapi.ServerConfiguration{ URL: request.pinningServiceBaseUrl, @@ -50,7 +48,7 @@ func NewClient(request ClientCreateRequest, cidVersion int) PinServiceAPI { } config.HTTPClient = request.httpClient - return &Client{client: openapi.NewAPIClient(config), ps: request.ps, cidVersion: cidVersion} + return &Client{client: openapi.NewAPIClient(config), ps: request.ps, cidVersion: cidVersion, w3up: w3up} } func (c *Client) IsIPFSSupportedFor(ps core.PinningService) bool { @@ -90,24 +88,12 @@ func (c *Client) Add(ctx context.Context, cid cid.Cid, opts ...AddOption) (core. } func (c *Client) UploadFile(ctx context.Context, file *os.File) (cid.Cid, error) { - var err error - var fcid core.CidGetter - switch c.ps { - case core.Pinata: - fcid, err = c.uploadFileViaPinata(ctx, file) - - case core.Web3Storage: - fcid, err = c.uploadFileViaWeb3Storage(ctx, file) - - default: - logger.Fatalf("only pinata supported for file upload") - } - + fcid, err := c.uploadFileViaWeb3Storage(ctx, file) if err != nil { return cid.Undef, fmt.Errorf("%w", err) } - return fcid.GetCid(), nil + return fcid, nil } func (c *Client) ServiceType() core.PinningService { @@ -138,29 +124,6 @@ func httperr(resp *http.Response, e error) error { return errors.Wrapf(e, "remote pinning service returned http error %d", resp.StatusCode) } -func (c *Client) uploadFileViaPinata(ctx context.Context, file *os.File) (core.PinataResponseGetter, error) { - //ctx = context.WithValue(ctx, openapi.ContextServerIndex, 1) // index = 1 is the file pin url - - poster := c.client.FilepinApi.PinataFileUpload(ctx) - opt := openapi.NewPinataOptions() - opt.SetCidVersion(string(rune(c.cidVersion))) - - result, httpresp, err := poster.PinataOptions(*opt).File(file).Execute() - if err != nil { - err := httperr(httpresp, err) - return nil, err - } - - return core.NewPinataResponseGetter(*result), nil -} - -func (c *Client) uploadFileViaWeb3Storage(ctx context.Context, file *os.File) (core.Web3StorageResponseGetter, error) { - poster := c.client.FilepinApi.Web3StorageCarUpload(ctx) - result, httpresp, err := poster.Body(file).Execute() - if err != nil { - err := httperr(httpresp, err) - return nil, err - } - - return core.NewWeb3StorageResponseGetter(*result), nil +func (c *Client) uploadFileViaWeb3Storage(ctx context.Context, file *os.File) (cid.Cid, error) { + return c.w3up.UploadCarFile(file) } diff --git a/pinner.go b/pinner.go index 637a7b7..ae514d4 100644 --- a/pinner.go +++ b/pinner.go @@ -5,16 +5,16 @@ package pinner import ( + "log" + car "github.com/covalenthq/ipfs-pinner/car" "github.com/covalenthq/ipfs-pinner/core" "github.com/covalenthq/ipfs-pinner/coreapi" "github.com/covalenthq/ipfs-pinner/dag" "github.com/covalenthq/ipfs-pinner/pinclient" - logging "github.com/ipfs/go-log/v2" + "github.com/covalenthq/ipfs-pinner/w3up" ) -var logger = logging.Logger("ipfs-pinner") - type pinnerNode struct { ipfsCore coreapi.CoreExtensionAPI carExporter car.CarExporterAPI @@ -26,12 +26,28 @@ func NewPinnerNode(req PinnerNodeCreateRequest) PinnerNode { node := pinnerNode{} ipfsNode, err := core.CreateIpfsNode(req.cidComputeOnly) if err != nil { - logger.Fatal("error initializing ipfs node: ", err) + log.Fatal("error initializing ipfs node: ", err) } node.ipfsCore = coreapi.NewCoreExtensionApi(ipfsNode) - node.pinApiClient = pinclient.NewClient(req.pinServiceRequest, req.cidVersion) + //SETUP W3UP + log.Print("setting up w3up for uploads....") + w3up := w3up.NewW3up(req.pinServiceRequest.W3_AgentKey, req.pinServiceRequest.W3_AgentDid, req.pinServiceRequest.W3_DelegationProofPath) + agentDid, err := w3up.WhoAmI() + if err != nil { + log.Fatal("error getting agent did: ", err) + } + log.Printf("w3up agent did: %s\n", agentDid.String()) + + spaceDid, err := w3up.SpaceAdd() + if err != nil { + log.Fatal("error adding space: ", err) + } + log.Printf("w3up space did: %s\n", spaceDid.String()) + log.Print("w3up setup complete") + + node.pinApiClient = pinclient.NewClient(req.pinServiceRequest, req.cidVersion, w3up) node.carExporter = car.NewCarExporter(node.ipfsCore) node.unixfsService = dag.NewUnixfsAPI(node.ipfsCore, req.cidVersion, req.cidComputeOnly, req.ipfsFetchUrls) diff --git a/server/main.go b/server/main.go index a895ce6..663830d 100644 --- a/server/main.go +++ b/server/main.go @@ -9,7 +9,6 @@ import ( "io" "log" "net/http" - "net/url" "os" "os/signal" "strconv" @@ -22,6 +21,8 @@ import ( pinner "github.com/covalenthq/ipfs-pinner" "github.com/covalenthq/ipfs-pinner/core" client "github.com/covalenthq/ipfs-pinner/pinclient" + "github.com/web3-storage/go-ucanto/did" + "github.com/web3-storage/go-ucanto/principal/ed25519/signer" ) const ( @@ -38,53 +39,62 @@ func NewState() *State { return &State{status: OK} } +type Config struct { + portNumber int + w3AgentKey string + w3AgentDid did.DID + delegationProofPath string + ipfsGatewayUrls []string +} + var ( - emptyBytes = []byte("") - WEB3_JWT = "WEB3_JWT" + emptyBytes = []byte("") + DOWNLOAD_TIMEOUT = 12 * time.Minute // download can take a lot of time if it's not locally present UPLOAD_TIMEOUT = 60 * time.Second // uploads of around 6MB files happen in less than 10s typically ) func main() { portNumber := flag.Int("port", 3001, "port number for the server") - token := flag.String("jwt", "", "JWT token for web3.storage") + + w3AgentKey := flag.String("w3-agent-key", "", "w3 agent key") + w3DelegationFile := flag.String("w3-delegation-file", "", "w3 delegation file") + ipfsGatewayUrls := flag.String("ipfs-gateway-urls", "https://w3s.link/ipfs/%s,https://dweb.link/ipfs/%s,https://ipfs.io/ipfs/%s", "comma separated list of ipfs gateway urls") flag.Parse() core.Version() - setUpAndRunServer(*portNumber, *token, *ipfsGatewayUrls) -} -func setUpAndRunServer(portNumber int, token, ipfsGatewayUrls string) { - mux := http.NewServeMux() - httpState := NewState() - if token == "" { - var present bool - token, present = os.LookupEnv(WEB3_JWT) - if !present { - log.Fatalf("token (%s) not found in env", WEB3_JWT) - } + if *w3AgentKey == "" { + log.Fatalf("w3 agent key is required") } - var ipfsGatewayUrlArr []string - if ipfsGatewayUrls != "" { - ipfsGatewayUrlArr = strings.Split(ipfsGatewayUrls, ",") - for _, ipfsUrlStr := range ipfsGatewayUrlArr { - if !strings.Contains(ipfsUrlStr, "%s") { - log.Fatalf("ipfs gateway url %s does not contain %%s", ipfsUrlStr) - } + if *w3DelegationFile == "" { + log.Fatalf("w3 delegation file is required") + } - if _, err := url.Parse(fmt.Sprintf(ipfsUrlStr, "sample_cid")); err != nil { - log.Fatalf("ipfs gateway url %s is not a valid url", ipfsUrlStr) - } - } - } else { - log.Fatalf("ipfs gateway urls not found in params") + _, err := os.ReadFile(*w3DelegationFile) + if err != nil { + log.Fatalf("error reading delegation proof file: %v", err) } - clientCreateReq := client.NewClientRequest(core.Web3Storage).BearerToken(token) + agentSigner, err := signer.Parse(*w3AgentKey) + if err != nil { + log.Fatalf("error parsing agent signer: %v", err) + } + + log.Printf("agent did: %s", agentSigner.DID().DID().String()) + + setUpAndRunServer(Config{*portNumber, *w3AgentKey, agentSigner.DID().DID(), *w3DelegationFile, strings.Split(*ipfsGatewayUrls, ",")}) +} + +func setUpAndRunServer(config Config) { + mux := http.NewServeMux() + httpState := NewState() + + clientCreateReq := client.NewClientRequest(core.Web3Storage).W3AgentKey(config.w3AgentKey).W3AgentDid(config.w3AgentDid).DelegationProofPath(config.delegationProofPath) // check if cid compute true works with car uploads - nodeCreateReq := pinner.NewNodeRequest(clientCreateReq, ipfsGatewayUrlArr).CidVersion(1).CidComputeOnly(false) + nodeCreateReq := pinner.NewNodeRequest(clientCreateReq, config.ipfsGatewayUrls).CidVersion(1).CidComputeOnly(false) node := pinner.NewPinnerNode(*nodeCreateReq) mux.Handle("/upload", recoveryWrapper(uploadHttpHandler(node))) @@ -97,7 +107,7 @@ func setUpAndRunServer(portNumber int, token, ipfsGatewayUrls string) { log.Print("Listening...") serverStopCommandSetup() - err := http.ListenAndServe(":"+strconv.Itoa(portNumber), mux) + err := http.ListenAndServe(":"+strconv.Itoa(config.portNumber), mux) if err != nil { log.Println("error listening and serving on TCP network: %w", err) } @@ -280,12 +290,12 @@ func uploadHandler(contents string, node pinner.PinnerNode) (cid.Cid, error) { } defer carf.Close() // should delete the file due to unlink - - err = syscall.Unlink(carf.Name()) - if err != nil { - log.Printf("%v", err) - return cid.Undef, err - } + defer func() { + err := syscall.Unlink(carf.Name()) + if err != nil { + log.Printf("error in unlinking:%v", err) + } + }() log.Printf("car file location: %s\n", carf.Name()) diff --git a/w3up/w3up.go b/w3up/w3up.go new file mode 100644 index 0000000..15a3382 --- /dev/null +++ b/w3up/w3up.go @@ -0,0 +1,98 @@ +package w3up + +import ( + "encoding/json" + "fmt" + "log" + "os" + "os/exec" + "strings" + + "github.com/ipfs/go-cid" + "github.com/web3-storage/go-ucanto/did" +) + +type W3up struct { + principal string + agentDid did.DID + delegationProofPath string + w3up_dir string +} + +func NewW3up(principal string, did did.DID, delegationProofPath string) *W3up { + tmpDir, err := os.MkdirTemp("", "w3up_config") + if err != nil { + log.Fatal("error creating temp dir for w3up config: ", err) + } + return &W3up{principal: principal, agentDid: did, delegationProofPath: delegationProofPath, w3up_dir: tmpDir} +} + +func (w3up *W3up) WhoAmI() (did.DID, error) { + command := fmt.Sprintf("W3_STORE_NAME=%s W3_PRINCIPAL=\"%s\" w3 whoami", w3up.w3up_dir, w3up.principal) + cmd := exec.Command("bash", "-c", command) + output, err := cmd.CombinedOutput() + if err != nil { + log.Fatal("error initializing w3up: ", string(output)) + return did.DID{}, err + } + + pdid, err := did.Parse(strings.TrimSpace(string(output))) + if err != nil { + log.Fatal("error parsing did: ", err) + return did.DID{}, err + } + return pdid, nil +} + +func (w3up *W3up) SpaceAdd() (did.DID, error) { + command := fmt.Sprintf("W3_STORE_NAME=%s w3 space add %s", w3up.w3up_dir, w3up.delegationProofPath) + cmd := exec.Command("bash", "-c", command) + output, err := cmd.CombinedOutput() + if err != nil { + log.Fatal("error adding w3up space: ", string(output)) + return did.DID{}, err + } + + pdid, err := did.Parse(strings.TrimSpace(string(output))) + if err != nil { + log.Fatal("error parsing did: ", err) + return did.DID{}, err + } + + return pdid, nil +} + +func (w3up *W3up) UploadCarFile(carFile *os.File) (cid.Cid, error) { + command := fmt.Sprintf("W3_STORE_NAME=%s w3 up %s --json --no-wrap --car", w3up.w3up_dir, carFile.Name()) + cmd := exec.Command("bash", "-c", command) + output, err := cmd.CombinedOutput() + if err != nil { + log.Fatal("error uploading car file: ", string(output)) + return cid.Undef, err + } + + jsons := strings.TrimSpace(string(output)) + + log.Printf("w3 up output: %s\n", jsons) + + var result map[string]interface{} + err = json.Unmarshal([]byte(jsons), &result) + if err != nil { + log.Fatal("error parsing json: ", err) + return cid.Undef, err + } + + root, ok := result["root"].(map[string]interface{}) + if !ok { + log.Fatal("error type asserting root") + return cid.Undef, fmt.Errorf("error type asserting root") + } + + rcid, ok := root["/"].(string) + if !ok { + log.Fatal("error type asserting cid") + return cid.Undef, fmt.Errorf("error type asserting cid") + } + + return cid.Parse(rcid) +}