Merge pull request #4 from Factual/feature/simple-auth-hdfs

Feature/simple auth hdfs
Factual · Dec 6, 2021 · 6a658d2 · 6a658d2
2 parents 1170803 + 13bc4ca
commit 6a658d2
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 37 deletions.
diff --git a/project.clj b/project.clj
@@ -1,10 +1,18 @@
 (defproject uio/uio "1.2-SNAPSHOT"
   :description "uio is a Clojure library and a command line tool for accessing HDFS, S3, SFTP and other file systems."
 
-  :repositories {"cloudera" "https://repository.cloudera.com/content/groups/cdh-releases-rcs"}
-
-  :deploy-repositories [["clojars" {:url           "https://clojars.org/repo/"
-                                    :sign-releases false}]]
+  :repositories {"cloudera"  "https://repository.cloudera.com/content/groups/cdh-releases-rcs"
+                 "foursquare"  {:url      "https://foursquaredev.jfrog.io/foursquaredev/fsnexus"
+                                :username :env/MVN_USERNAME :password :env/MVN_PASSWORD}}
+
+  :deploy-repositories {"snapshots" {:id          "foursquare"
+                                     :url         "https://foursquaredev.jfrog.io/foursquaredev/fsfactual-snapshots-local"
+                                     :username :env/MVN_USERNAME :password :env/MVN_PASSWORD
+                                     :sign-releases false}
+                        "releases"  {:id          "foursquare"
+                                     :url         "https://foursquaredev.jfrog.io/foursquaredev/fsfactual-releases-local"
+                                     :username :env/MVN_USERNAME :password :env/MVN_PASSWORD
+                                     :sign-releases false}}
 
   :dependencies [[org.clojure/clojure "1.9.0"]
 

diff --git a/src/uio/fs/hdfs.clj b/src/uio/fs/hdfs.clj
@@ -29,41 +29,26 @@
   (let [c           (Configuration.)
         creds       (url->creds url)
 
-        principal   (:principal creds)
-        keytab-path (some-> (:keytab creds) path)
         aws-access  (:access creds)
         aws-secret  (:secret creds)]
 
     (when (and aws-access aws-secret)
-      (.set c "fs.s3a.impl"               "org.apache.hadoop.fs.s3a.S3AFileSystem")
       (.set c "fs.s3a.access.key"         aws-access)
       (.set c "fs.s3a.secret.key"         aws-secret)
 
-      (.set c "fs.s3n.impl"               "org.apache.hadoop.fs.s3native.NativeS3FileSystem")
       (.set c "fs.s3n.awsAccessKeyId"     aws-access)
       (.set c "fs.s3n.awsSecretAccessKey" aws-secret)
 
-      (.set c "fs.s3.impl"                "org.apache.hadoop.fs.s3.S3FileSystem")
       (.set c "fs.s3.awsAccessKeyId"      aws-access)
       (.set c "fs.s3.awsSecretAccessKey"  aws-secret))
 
     (doseq [url ["file:///etc/hadoop/conf/core-site.xml"
                  "file:///etc/hadoop/conf/hdfs-site.xml"]]
       (if (exists? url)
         (.addResource c (URL. url))))
-
-    (.set c "hadoop.security.authentication" "kerberos")
 
     (UserGroupInformation/setConfiguration c)
 
-    ; only use keytab creds if either user or keytab path was specified, otherwise rely on default auth (e.g. if ran from kinit/Yarn)
-    (when (or principal keytab-path)
-      (UserGroupInformation/loginUserFromKeytab principal keytab-path)
-
-      ; TODO is there a way to provide more information about the failure?
-      (if-not (UserGroupInformation/isLoginKeytabBased)
-        (die "Could not authenticate. Wrong or missing keytab?")))
-
     c))
 
 (defn ->fs [^String url]

diff --git a/src/uio/impl.clj b/src/uio/impl.clj
@@ -222,21 +222,11 @@
         c           (or config {})                          ; config -- for compatibility, credentials stored as keys
         e           (or env {})                             ; env    -- for compatibility, comes from JVM process (immutable, extracted as arg for testing)
 
-        nie         (fn [s] (if (str/blank? s) nil s))      ; nil-if-empty
-
-        ensure-url  (fn [k url-or-path]                     ; ensure-url
-                      (cond (nil? url-or-path) nil
-                            (str/starts-with? url-or-path default-delimiter) (str "file://" url-or-path)
-                            (url? url-or-path) url-or-path
-                            :else (die (str "Expected URL or path that starts with / for " k ", but got: " url-or-path))))
-
         creds       (if creds                               ; so it's the latest "url -> creds" version
                       creds
                       (case (scheme url)
                         ;        <current>             <obsolete>                    <obsolete>                  <obsolete>
-                        "hdfs" {:principal     (or (c :hdfs.keytab.principal)    (e "HDFS_KEYTAB_PRINCIPAL") (e "KEYTAB_PRINCIPAL"))
-                                :keytab        (or (c :hdfs.keytab.path)         (e "HDFS_KEYTAB_PATH")      (e "KEYTAB_FILE"))
-                                :access        (or (c :s3.access)                (e "AWS_ACCESS")            (e "AWS_ACCESS_KEY_ID"))
+                        "hdfs" {:access        (or (c :s3.access)                (e "AWS_ACCESS")            (e "AWS_ACCESS_KEY_ID"))
                                 :secret        (or (c :s3.secret)                (e "AWS_SECRET")            (e "AWS_SECRET_ACCESS_KEY"))}
 
                         "sftp" {:user          (or (c :sftp.user)                (e "SFTP_USER")             (e "SSH_USER"))
@@ -248,13 +238,7 @@
                         {}))]
     ; TODO post-validate pairs?
     ; TODO fail on unknown keys in `cr`?
-
-    ; if hdfs, replace empty strings with nil (required for proper work of HDFS API) + change path to URL
-    (case (scheme url)
-      "hdfs" (-> creds
-                 (update :principal nie)
-                 (update :keytab #(ensure-url :keytab (nie %))))
-      creds)))
+    creds))
 
 (defn url->creds [url]
   (url->creds' *config* (into {} (System/getenv)) url))