Scp

README.md

@@ -180,6 +180,34 @@ afctl config add -d qubole -n demo -e https://api.qubole.com -c airflow_1102 -t
 afctl deploy qubole -n <name>
 ```
 
+### 7. Deploy project to a remote Airflow
+
+You can deploy your DAG's directly to a remote Airflow instance using SCP.  Your login must have
+enough permissions to write to the DAG folder; but since we don't know the installation, it isn't
+possible to manage any Python requirements using this deployer.
+
+In order to configure remote deployment, you must pass host and dag directory; you may set the
+login account, and either a password or identity file.
+
+```bash
+afctl config add -d remote -m airflow.example.com -d /var/lib/airflow/dags -e mypassword
+```
+or (say)
+
+```bash
+afctl config add -d remote -m airflow.example.com -d /var/lib/airflow/dags -u airflow -i ~/.ssh/id_rsa-airflow
+```
+
+This command will modify your config file. You can see your config file with the following command :
+```bash
+afctl config show
+
+* To deploy run the following command
+```bash
+afctl deploy remote
+```
+
+
 ### The following video also contains all the steps of deploying project using afctl - </br>
 https://www.youtube.com/watch?v=A4rcZDGtJME&feature=youtu.be
 
@@ -194,8 +222,9 @@ global:
 --access-token:
 deployment:
 -qubole:
---local:
----compose:
+-local:
+--compose:
+-remote:
 ```
 <br>
 
@@ -225,8 +254,6 @@ afctl <command> -h
 ```
 <br>
 
-### Caution
-Not yet ported for Windows.
 
 #### Credits
 Docker-compose file : https://github.com/puckel/docker-airflow 

afctl/_version.py

@@ -495,7 +495,7 @@ def get_versions():
         # versionfile_source is the relative path from the top of the source
         # tree (where the .git directory might live) to this file. Invert
         # this to find the root from __file__.
-        for i in cfg.versionfile_source.split('/'):
+        for i in cfg.versionfile_source.split(os.path.sep):
             root = os.path.dirname(root)
     except NameError:
         return {"version": "0+unknown", "full-revisionid": None,

afctl/meta.yml

@@ -7,4 +7,5 @@ hooks:
 deployment:
   qubole
   local
+  remote
 

afctl/parser_helpers.py

@@ -1,16 +1,19 @@
 from afctl.utils import Utility
 import os
-import subprocess
+import shutil
 from termcolor import colored
 from afctl.exceptions import AfctlParserException
+import git
+
+SEP = os.path.sep
 
 class ParserHelpers():
 
     @staticmethod
     def get_project_file_names(name):
         try:
             pwd = os.getcwd()
-            main_dir = pwd if name == '.' else os.path.join(pwd, name.lstrip('/').rstrip('/'))
+            main_dir = pwd if name == '.' else os.path.join(pwd, name.lstrip(SEP).rstrip(SEP))
             project_name = os.path.basename(main_dir)
             config_dir = Utility.CONSTS['config_dir']
             config_file = Utility.project_config(project_name)
@@ -35,11 +38,10 @@ def get_project_file_names(name):
     @staticmethod
     def add_git_config(files):
         try:
-            origin = subprocess.run(['git', '--git-dir={}'.format(os.path.join(files['main_dir'], '.git')), 'config',
-                                     '--get', 'remote.origin.url'],stdout=subprocess.PIPE)
-            origin = origin.stdout.decode('utf-8')[:-1]
-            if origin == '':
-                subprocess.run(['git', 'init', files['main_dir']])
+            try:
+                origin = git.Repo(os.path.join(files['main_dir']))
+            except git.exc.InvalidGitRepositoryError:
+                origin = git.Repo.init(os.path.join(files['main_dir']))
                 print(colored("Git origin is not set for this repository. Run 'afctl config global -o <origin>'", 'yellow'))
             else:
                 print("Updating git origin.")
@@ -71,19 +73,17 @@ def generate_all(files):
             #STEP - 2: create config file
             ParserHelpers.generate_config_file(files)
 
-            subprocess.run(['cp', '{}/templates/gitignore.txt'.format(os.path.dirname(os.path.abspath(__file__))),
-                            sub_file['.gitignore']])
-
+            shutil.copyfile('{}/templates/gitignore.txt'.format(os.path.dirname(os.path.abspath(__file__))),
+                            sub_file['.gitignore'])
         except Exception as e:
             raise AfctlParserException(e)
 
 
     @staticmethod
     def generate_config_file(files):
         try:
-            subprocess.run(['cp', '{}/plugins/deployments/deployment_config.yml'.format(os.path.dirname(os.path.abspath(__file__))),
-                            files['config_file']])
-
+            shutil.copyfile('{}/plugins/deployments/deployment_config.yml'.format(os.path.dirname(os.path.abspath(__file__))),
+                            files['config_file'])
             ParserHelpers.add_git_config(files)
 
         except Exception as e:

afctl/parsers.py

@@ -5,7 +5,6 @@
 from afctl import __version__
 from afctl.utils import Utility
 from afctl.exceptions import AfctlParserException
-import subprocess
 from afctl.plugins.deployments.deployment_config import DeploymentConfig
 from afctl.parser_helpers import ParserHelpers
 from termcolor import colored
@@ -44,7 +43,7 @@ def init(cls, args):
                 os.mkdir(files['config_dir'])
 
             if os.path.exists(files['main_dir']) and os.path.exists(files['config_file']):
-                cls.parser.error(colored("Project already exists. Please delete entry under /home/.afctl_congfis", 'red'))
+                cls.parser.error(colored("Project already exists. Please delete entry under ~/.afctl_config", 'red'))
 
             print(colored("Initializing new project...", 'green'))
 
@@ -123,9 +122,10 @@ def generate(cls, args):
             elif args.type == "module":
                 path = "{}/{}/dags/{}".format(project_path, project_name, args.n)
                 test_path = "{}/tests/{}".format(project_path, args.n)
-                mod_val = subprocess.call(['mkdir', path])
-                test_val = subprocess.call(['mkdir', test_path])
-                if mod_val != 0 or test_val != 0:
+                try:
+                    os.makedirs(path, exist_ok=True)
+                    os.makedirs(test_path, exist_ok=True)
+                except:
                     cls.parser.error(colored("Unable to generate.", 'red'))
 
             print(colored("Generated successfully.", 'green'))
@@ -178,13 +178,16 @@ def get_subparsers(cls):
                         ,
                 'args': [
                     ['type', {'choices':['add', 'update', 'show', 'global']}],
-                    ['-d', {'choices': ['qubole']}],
+                    ['-d', {'choices': ['qubole', 'remote']}],
+                    ['-m'],
+                    ['-i'],
                     ['-o'],
                     ['-p'],
                     ['-n'],
                     ['-e'],
                     ['-c'],
                     ['-t'],
+                    ['-u'],
                     ['-v']
                 ]
 
@@ -286,4 +289,4 @@ def act_on_configs(cls, args, project_name):
                 Utility.print_file(Utility.project_config(project_name))
 
         except Exception as e:
-            AfctlParserException(e)
+            raise AfctlParserException(e)

afctl/plugins/deployments/deployment_config.py

@@ -1,24 +1,27 @@
 from afctl.plugins.deployments.base_deployment_config import BaseDeploymentConfig
 from afctl.plugins.deployments.qubole.deployment_config import QuboleDeploymentConfig
 from afctl.plugins.deployments.docker.deployment_config import DockerDeploymentConfig
+from afctl.plugins.deployments.scp.deployment_config import ScpDeploymentConfig
 
 from afctl.exceptions import AfctlDeploymentException
 
 class DeploymentConfig(BaseDeploymentConfig):
     # Just append configs for other deployments here.
     CONFIG_DETAILS = QuboleDeploymentConfig.CONFIG_PARSER_USAGE+\
-                     DockerDeploymentConfig.CONFIG_PARSER_USAGE
+                     DockerDeploymentConfig.CONFIG_PARSER_USAGE+\
+                     ScpDeploymentConfig.CONFIG_PARSER_USAGE
 
     DEPLOY_DETAILS = QuboleDeploymentConfig.DEPLOY_PARSER_USAGE+\
-                     DockerDeploymentConfig.DEPLOY_PARSER_USAGE
+                     DockerDeploymentConfig.DEPLOY_PARSER_USAGE+\
+                     ScpDeploymentConfig.DEPLOY_PARSER_USAGE
 
     @classmethod
     def validate_configs(cls, args):
         try:
-
             if args.d == 'qubole':
                 return QuboleDeploymentConfig.validate_configs(args)
-
+            if args.d == 'remote':
+                return ScpDeploymentConfig.validate_configs(args)
         except Exception as e:
             raise AfctlDeploymentException(e)
 
@@ -34,5 +37,8 @@ def deploy_project(cls, args, project_name, project_path):
             if args.type == "qubole":
                 return QuboleDeploymentConfig.deploy_project(args, project_name)
 
+            if args.type == "remote":
+                return ScpDeploymentConfig.deploy_project(args, project_name)
+
         except Exception as e:
-            raise AfctlDeploymentException(e)
+            raise AfctlDeploymentException(e)

afctl/plugins/deployments/deployment_config.yml

@@ -6,4 +6,5 @@ global:
 deployment:
   qubole:
   local:
-    compose:
+    compose:
+  remote:

afctl/plugins/deployments/docker/deployment_config.py

@@ -4,7 +4,7 @@
 import os
 import yaml
 from afctl.utils import Utility
-import subprocess
+import docker
 
 # Yaml Structure
 #   deployment:
@@ -47,8 +47,10 @@ def deploy_project(cls, args, config_file):
             with open(Utility.project_config(config_file)) as file:
                 config = yaml.full_load(file)
 
-            val = subprocess.call(['docker', 'info'])
-            if val != 0:
+            try:
+                client = docker.from_env()
+                client.info()
+            except:
                 return True, "Docker is not running. Please start docker."
 
             if args.d:

afctl/plugins/deployments/qubole/deployment_config.py

@@ -127,4 +127,4 @@ def deploy_project(cls, args, config_file):
             return False, ""
 
         except Exception as e:
-            raise AfctlDeploymentException(e)
+            raise AfctlDeploymentException(e)

afctl/plugins/deployments/qubole/qubole_utils.py

@@ -1,25 +1,17 @@
-import subprocess
 from afctl.exceptions import AfctlDeploymentException
 from mako.template import Template
 from qds_sdk.qubole import Qubole
 from qds_sdk.commands import ShellCommand
 from urllib.parse import urlparse
-
+import git
 
 class QuboleUtils():
 
     @staticmethod
     def fetch_latest_commit(origin, branch):
         try:
-            commit = subprocess.run(['git', 'ls-remote', origin, 'refs/heads/{}'.format(branch), '|', 'cut', '-f', '1'],
-                                    stdout=subprocess.PIPE)
-            commit = commit.stdout.decode('utf-8')
-
-            if commit == '':
-                return None
-
-            return commit.split('\t')[0]
-
+            repo = git.Repo('.')
+            return repo.remotes.origin.refs[0].commit.hexsha
         except Exception as e:
             raise AfctlDeploymentException(e)
 
@@ -95,9 +87,8 @@ def generate_configs(configs, args):
             env = "{}/api".format(config['env'].rstrip('/'))
             cluster = config['cluster']
             token = config['token']
-            branch = subprocess.run(['git', 'symbolic-ref', '--short', 'HEAD'], stdout=subprocess.PIPE).stdout.decode(
-                'utf-8')[:-1]
-
+            repo = git.Repo('.')
+            branch = repo.active_branch.name
             return {
                 'name': name,
                 'env': env,