[kubernetes] Use KUBECONFIG if set (#1716)

* [kubernetes] Use KUBECONFIG if set

* [kubernetes] If neither KUBECONFIG nor KUBERNETES_SERVICE_HOST are set, default to loading kubeconfig to keep current behaviour

* [kubernetes] Document use of KUBECONFIG environment variable

metaflow/plugins/kubernetes/kubernetes_client.py

@@ -6,6 +6,7 @@
 
 from .kubernetes_job import KubernetesJob
 
+
 CLIENT_REFRESH_INTERVAL_SECONDS = 300
 
 
@@ -32,11 +33,18 @@ def __init__(self):
     def _refresh_client(self):
         from kubernetes import client, config
 
-        if os.getenv("KUBERNETES_SERVICE_HOST"):
+        if os.getenv("KUBECONFIG"):
+            # There are cases where we're running inside a pod, but can't use
+            # the kubernetes client for that pod's cluster: for example when
+            # running in Bitbucket Cloud or other CI system.
+            # In this scenario, the user can set a KUBECONFIG environment variable
+            # to load the kubeconfig, regardless of whether we're in a pod or not.
+            config.load_kube_config()
+        elif os.getenv("KUBERNETES_SERVICE_HOST"):
             # We are inside a pod, authenticate via ServiceAccount assigned to us
             config.load_incluster_config()
         else:
-            # Use kubeconfig, likely $HOME/.kube/config
+            # Default to using kubeconfig, likely $HOME/.kube/config
             # TODO (savin):
             #  1. Support generating kubeconfig on the fly using boto3
             #  2. Support auth via OIDC - https://docs.aws.amazon.com/eks/latest/userguide/authenticate-oidc-identity-provider.html