Skip to content

Commit

Permalink
add w3 pinning
Browse files Browse the repository at this point in the history
  • Loading branch information
tarassh committed Aug 14, 2024
1 parent 1daf179 commit 1245ccf
Show file tree
Hide file tree
Showing 9 changed files with 208 additions and 14 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,6 @@ bin/
# Go workspace file
go.work
go.work.sum

.env
delegation.proof
8 changes: 5 additions & 3 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,14 @@
"mode": "auto",
"program": "${workspaceFolder}/cmd/pinner/main.go",
"args": [
"--debug"
"--debug",
"--w3-agent-key", "<AGENT_KEY>",
"--w3-delegation-proof-path", "<DELEGATION_PROOF_PATH>",
],
"env": {
"TRUSTED_DIR": "${workspaceFolder}/test/data",
"IPFS_LOGGING": "info"
}
"IPFS_LOGGING": "info",
},
},
{
"name": "Pinner CLI: STORE",
Expand Down
14 changes: 10 additions & 4 deletions api/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,15 @@ import (
ipfsnode "github.com/covalenthq/das-ipfs-pinner/internal/ipfs-node"
)

type ServerConfig struct {
Addr string
W3AgentKey string
W3DelegationProofPath string
}

// StartServer initializes and starts the HTTP server.
func StartServer(addr string) {
ipfsnode, err := ipfsnode.NewIPFSNode()
func StartServer(config ServerConfig) {
ipfsnode, err := ipfsnode.NewIPFSNode(config.W3AgentKey, config.W3DelegationProofPath)
if err != nil {
log.Fatalf("Failed to initialize IPFS node: %v", err)
}
Expand All @@ -35,8 +41,8 @@ func StartServer(addr string) {
http.HandleFunc("/extract", extractHandler)

// Start the HTTP server
log.Printf("Starting daemon on %s...\n", addr)
if err := http.ListenAndServe(addr, nil); err != nil {
log.Printf("Starting daemon on %s...\n", config.Addr)
if err := http.ListenAndServe(config.Addr, nil); err != nil {
log.Fatalf("Could not start server: %v\n", err)
}
}
Expand Down
23 changes: 19 additions & 4 deletions cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,10 @@ import (
)

var (
debug bool
addr string
debug bool
addr string
w3AgentKey string
w3DelegationProofPath string
)

// rootCmd represents the base command
Expand Down Expand Up @@ -49,8 +51,13 @@ var rootCmd = &cobra.Command{
}
},
Run: func(cmd *cobra.Command, args []string) {
// Start the API server
api.StartServer(addr)
// Populate the ServerConfig struct
config := api.ServerConfig{
Addr: addr,
W3AgentKey: w3AgentKey,
W3DelegationProofPath: w3DelegationProofPath,
}
api.StartServer(config)
},
}

Expand All @@ -70,6 +77,14 @@ func init() {
cobra.OnInitialize(initConfig)
rootCmd.PersistentFlags().BoolVar(&debug, "debug", false, "Run in debug mode")
rootCmd.PersistentFlags().StringVar(&addr, "addr", getEnv("DAEMON_ADDR", "localhost:5080"), "Address to run the daemon")

// W3 agent flags
rootCmd.PersistentFlags().StringVar(&w3AgentKey, "w3-agent-key", "", "Key for the W3 agent")
rootCmd.PersistentFlags().StringVar(&w3DelegationProofPath, "w3-delegation-proof-path", "", "Path to the W3 delegation proof")

// Mark the flags as required
rootCmd.MarkPersistentFlagRequired("w3-agent-key")
rootCmd.MarkPersistentFlagRequired("w3-delegation-proof-path")
}

func initConfig() {
Expand Down
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,13 @@ require (
github.com/ipfs/go-block-format v0.2.0
github.com/ipfs/go-cid v0.4.1
github.com/ipfs/kubo v0.29.0
github.com/ipld/go-car v0.6.2
github.com/ipld/go-ipld-prime v0.21.0
github.com/multiformats/go-multicodec v0.9.0
github.com/multiformats/go-multihash v0.2.3
github.com/protolambda/go-kzg v0.0.0-20221224134646-c91cee5e954e
github.com/spf13/cobra v1.8.1
github.com/web3-storage/go-ucanto v0.1.0
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
)

Expand Down Expand Up @@ -97,7 +99,6 @@ require (
github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
github.com/ipfs/go-unixfsnode v1.9.0 // indirect
github.com/ipfs/go-verifcid v0.0.3 // indirect
github.com/ipld/go-car v0.6.2 // indirect
github.com/ipld/go-car/v2 v2.13.1 // indirect
github.com/ipld/go-codec-dagpb v1.6.0 // indirect
github.com/jackpal/go-nat-pmp v1.0.2 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -769,6 +769,8 @@ github.com/warpfork/go-testmark v0.12.1/go.mod h1:kHwy7wfvGSPh1rQJYKayD4AbtNaeyZ
github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ=
github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
github.com/web3-storage/go-ucanto v0.1.0 h1:Hg6jO7OLLeDLtmseQZWQGBFJMp+5t5OWAFVL5Y3LsOs=
github.com/web3-storage/go-ucanto v0.1.0/go.mod h1:XD6zahQ8HLh8Z2CSK7xYcTR0A7oCVYXTZB7fFtYqGF8=
github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc h1:BCPnHtcboadS0DvysUuJXZ4lWVv5Bh5i7+tbIyi+ck4=
github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM=
github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIfyDmc1Em5GqlNRzcdtlv4HTNmdpt7XH0=
Expand Down
13 changes: 12 additions & 1 deletion internal/ipfs-node/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,20 @@ import (
type IPFSNode struct {
Node *core.IpfsNode
API iface.CoreAPI
W3 *W3Storage
}

// NewIPFSNode initializes and returns a new IPFSNode instance.
func NewIPFSNode() (*IPFSNode, error) {
func NewIPFSNode(w3Key, w3DelegationProofPath string) (*IPFSNode, error) {
w3, err := NewW3Storage(w3Key, w3DelegationProofPath)
if err != nil {
return nil, err
}

if err := w3.Initialize(); err != nil {
return nil, err
}

buildConfig, err := initializeIPFSConfig()
if err != nil {
return nil, err
Expand All @@ -41,6 +51,7 @@ func NewIPFSNode() (*IPFSNode, error) {
return &IPFSNode{
Node: node,
API: api,
W3: w3,
}, nil
}

Expand Down
7 changes: 6 additions & 1 deletion internal/ipfs-node/pin.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,12 @@ func (ipfsNode *IPFSNode) Pin(ctx context.Context, root cid.Cid) error {
return err
}

// TODO: pin the CAR file to the web3.storage
cid, err := ipfsNode.W3.Pin(carFile)
if err != nil {
return err
}

log.Printf("Data stored successfully with CID: %s\n", cid)

return nil
}
149 changes: 149 additions & 0 deletions internal/ipfs-node/w3storage.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
package ipfsnode

import (
"encoding/json"
"fmt"
"log"
"os"
"os/exec"
"strings"

"github.com/ipfs/go-cid"
"github.com/web3-storage/go-ucanto/did"
"github.com/web3-storage/go-ucanto/principal/ed25519/signer"
)

// W3Storage struct encapsulates the web3.storage client.
type W3Storage struct {
agentKey string
agentDID did.DID
delegationProofPath string
w3Temp string
}

// Binary name for the `w3` command-line tool.
var binName = "w3"

// NewW3Storage initializes a new W3Storage instance.
func NewW3Storage(agentKey, delegationProofPath string) (*W3Storage, error) {
// Ensure the `w3` binary is available in the PATH.
if _, err := exec.LookPath(binName); err != nil {
return nil, fmt.Errorf("%s binary not found in PATH", binName)
}

// Create a temporary directory for `w3` usage.
tmpDir, err := os.MkdirTemp("", "w3up_config")
if err != nil {
return nil, fmt.Errorf("failed to create temporary directory: %w", err)
}

// Parse the agent key using the ed25519 signer.
agentSigner, err := signer.Parse(agentKey)
if err != nil {
return nil, fmt.Errorf("failed to parse agent key: %w", err)
}

// Verify that the delegation proof path exists.
if _, err := os.Stat(delegationProofPath); err != nil {
return nil, fmt.Errorf("delegation proof path does not exist: %w", err)
}

return &W3Storage{
agentKey: agentKey,
agentDID: agentSigner.DID().DID(),
delegationProofPath: delegationProofPath,
w3Temp: tmpDir,
}, nil
}

// Close cleans up resources used by W3Storage.
func (w3 *W3Storage) Close() {
_ = os.RemoveAll(w3.w3Temp) // Ignoring error as it's not critical
}

func (w3 *W3Storage) Initialize() error {
i, err := w3.whoAmI()
if err != nil {
return err
}
log.Printf("Initialized W3Storage with agent DID: %s\n", i)

spaceDID, err := w3.addSpace()
if err != nil {
return err
}

log.Printf("Added space with DID: %s\n", spaceDID)

return nil
}

// whoAmI returns the DID of the current W3Storage instance.
func (w3 *W3Storage) whoAmI() (did.DID, error) {
command := fmt.Sprintf("W3_STORE_NAME=%s W3_PRINCIPAL=\"%s\" %s whoami", w3.w3Temp, w3.agentKey, binName)
output, err := runCommand(command)
if err != nil {
return did.Undef, err
}

pdid, err := did.Parse(strings.TrimSpace(output))
if err != nil {
return did.Undef, fmt.Errorf("failed to parse DID from whoami output: %w", err)
}

return pdid, nil
}

// addSpace adds a space to the W3Storage instance using the delegation proof.
func (w3 *W3Storage) addSpace() (did.DID, error) {
command := fmt.Sprintf("W3_STORE_NAME=%s %s space add %s", w3.w3Temp, binName, w3.delegationProofPath)
output, err := runCommand(command)
if err != nil {
return did.Undef, err
}

pdid, err := did.Parse(strings.TrimSpace(output))
if err != nil {
return did.Undef, fmt.Errorf("failed to parse DID from add space output: %w", err)
}

return pdid, nil
}

// Pin uploads a CAR file and pins it, returning the resulting CID.
func (w3 *W3Storage) Pin(carFile *os.File) (cid.Cid, error) {
command := fmt.Sprintf("W3_STORE_NAME=%s %s up %s --json --no-wrap --car", w3.w3Temp, binName, carFile.Name())
output, err := runCommand(command)
if err != nil {
return cid.Undef, err
}

log.Printf("w3 up output: %s\n", output)

var result map[string]interface{}
if err := json.Unmarshal([]byte(output), &result); err != nil {
return cid.Undef, fmt.Errorf("failed to unmarshal w3 up output: %w", err)
}

root, ok := result["root"].(map[string]interface{})
if !ok {
return cid.Undef, fmt.Errorf("unexpected format: root key missing in w3 up output")
}

rcid, ok := root["/"].(string)
if !ok {
return cid.Undef, fmt.Errorf("unexpected format: CID key missing in root object")
}

return cid.Parse(rcid)
}

// runCommand is a helper function to execute shell commands and return the output.
func runCommand(command string) (string, error) {
cmd := exec.Command("bash", "-c", command)
output, err := cmd.CombinedOutput()
if err != nil {
return "", fmt.Errorf("command failed: %s, error: %w", string(output), err)
}
return strings.TrimSpace(string(output)), nil
}

0 comments on commit 1245ccf

Please sign in to comment.