Support for S3 as a storage backend

README.md

@@ -39,18 +39,23 @@ Set a remote store for the fat objects by editing `.gitfat`.
 
 This file should typically be committed to the repository so that others
 will automatically have their remote set. This remote address can use
-any protocol supported by rsync. 
+any protocol supported by rsync.
 
-Most users will configure it to use remote ssh in a directory with shared 
-access. To do this, set the `sshuser` and `sshport` variables in `.gitfat` 
+Most users will configure it to use remote ssh in a directory with shared
+access. To do this, set the `sshuser` and `sshport` variables in `.gitfat`
 configuration file. For example, to use rsync with ssh, with the default
-port (22) and authenticate with the user "_fat_", your configuration would 
-look like this: 
+port (22) and authenticate with the user "_fat_", your configuration would
+look like this:
 
     [rsync]
     remote = your.remote-host.org:/share/fat-store
     sshuser = fat
 
+To use an Amazon S3 bucket as the backend, you should first install the AWS CLI and configure it with a user that has access to the bucket. Your configuration would then look like:
+
+    [s3]
+    bucket = s3://your-s3-bucket
+
 # A worked example
 
 Before we start, let's turn on verbose reporting so we can see what's
@@ -145,7 +150,7 @@ selected history.
     1 file to consider
     1f218834a137f7b185b498924e7a030008aee2ae
             6449 100%    6.15MB/s    0:00:00 (xfer#1, to-check=0/1)
-    
+
     sent 30 bytes  received 6558 bytes  4392.00 bytes/sec
     total size is 6449  speedup is 0.98
     Restoring 1f218834a137f7b185b498924e7a030008aee2ae -> master.tar.gz

git-fat

@@ -46,6 +46,23 @@ except ImportError:
 
 BLOCK_SIZE = 4096
 
+def which(program):
+    def is_exe(fpath):
+        return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
+
+    fpath, fname = os.path.split(program)
+    if fpath:
+        if is_exe(program):
+            return program
+    else:
+        for path in os.environ["PATH"].split(os.pathsep):
+            exe_file = os.path.join(path, program)
+            if is_exe(exe_file):
+                return exe_file
+
+    return None
+
+
 def verbose_stderr(*args, **kwargs):
     return print(*args, file=sys.stderr, **kwargs)
 def verbose_ignore(*args, **kwargs):
@@ -158,7 +175,40 @@ class GitFat(object):
         if remote is None:
             raise RuntimeError('No rsync.remote in %s' % cfgpath)
         return remote, ssh_port, ssh_user, options
+
+    def get_aws_cmd(self, push, s3_bucket):
+        if not which('aws'):
+            sys.stderr.write('Could not find aws cli install.\n')
+            sys.exit(1)
+
+        if not s3_bucket.startswith('s3://'):
+            s3_bucket = "s3://{}".format(s3_bucket)
+
+        if push:
+            self.verbose('Pushing to %s' % (s3_bucket))
+            cmd = [
+                "aws",
+                "s3",
+                "sync",
+                self.objdir + "/",
+                s3_bucket + "/"
+            ]
+        else:
+            self.verbose('Pulling from %s' % (s3_bucket))
+            cmd = [
+                "aws",
+                "s3",
+                "sync",
+                s3_bucket + "/",
+                self.objdir + "/"
+            ]
+        return cmd
+
     def get_rsync_command(self,push):
+        cfgpath = os.path.join(self.gitroot,'.gitfat')
+        s3_bucket = gitconfig_get('s3.bucket', file=cfgpath)
+        if s3_bucket:
+            return self.get_aws_cmd(push, s3_bucket)
         (remote, ssh_port, ssh_user, options) = self.get_rsync()
         if push:
             self.verbose('Pushing to %s' % (remote))