CAF::Download class: API proposal

pom.xml

@@ -85,6 +85,7 @@
                       <directory>src/main/perl</directory>
                       <includes>
                         <include>*.pm</include>
+                        <include>**/*.pm</include>
                       </includes>
                       <filtering>true</filtering>
                     </resource>
@@ -138,6 +139,7 @@
                       <directory>src/main/perl</directory>
                       <includes>
                         <include>*.pm</include>
+                        <include>**/*.pm</include>
                       </includes>
                       <filtering>true</filtering>
                     </resource>

src/main/pan/quattor/types/download.pan

@@ -0,0 +1,132 @@
+declaration template quattor/types/download;
+
+include 'pan/types';
+
+@documentation{
+    A string that represents a URL that can be handled by CAF::Download
+    Format: [auth+][method+]protocol://location
+    Protocols: http|file
+    Methods: lwp|curl transport
+    Auth: kinit|gssapi|x509 authentication
+    Location: anything
+}
+type caf_url_string = string with {
+    mp_l = split('://', SELF);
+
+    if (length(mp_l) != 2) {
+        error('invalid URL string requires ://, got' + SELF);
+        return(false);
+    };
+
+    if(! match(mp_l[0], '^((kinit|gssapi|x509)\+)?((lwp|curl)\+)?(https?|file)$')) {
+        error('invalid method+protocol for ' + mp_l[0]);
+        return(false);
+    };
+
+    a_m_p = split('\+', mp_l[0]);
+    protocol = a_m_p[length(a_m_p)-1];
+
+    if (protocol == 'file' && (!match(mp_l[1], '^/'))) {
+        error("location for file protocol has to start with /, got " + mp_l[1]);
+        return(false);
+    };
+
+    true;
+};
+
+# similar to kerberos_principal_string in ncm-ccm
+# http://web.mit.edu/kerberos/krb5-1.4/krb5-1.4.3/doc/krb5-user/Kerberos-Glossary.html
+type kerberos_primary = string with match(SELF, '^\w+$');
+type kerberos_realm = string with match(SELF, '^[A-Z][A-Z.-_]*$');
+type kerberos_instance = string with match(SELF, '^\w[\w.-]*$');
+
+# TODO: What if you want to use the defaults for all 4 settings?
+@documentation{
+    CAF::Download kerberos configuration
+}
+type caf_url_krb5 = {
+    'keytab' ? string
+    'primary' ? kerberos_primary
+    'realm' ? kerberos_realm
+    'instances' ? kerberos_instance[]
+} with {
+    if(exists(SELF['instances']) && ! exists(SELF['primary'])) {
+        error("Cannot have krb5 instance(s) without primary");
+    };
+    true;
+};
+
+@documentation{
+    CAF::Download X509 configuration
+}
+type caf_url_x509 = {
+    'cacert' ? string
+    'capath' ? string
+    'cert' ? string
+    'key' ? string
+} with {
+    if(exists(SELF['cacert']) && exists(SELF['capath'])) {
+        error('Both X509 cacert and capath defined, cannot have both');
+    };
+    true;
+};
+
+@documentation{
+    CAF::Download proxy configuration
+}
+type caf_url_proxy = {
+    'server' : type_hostname
+    'port' ? type_port
+    'reverse' ? boolean # reverse proxy (default is false, i.e. forward)
+};
+
+@documentation{
+    CAF::Download supported authentication: one of gssapi, kinit or lwp
+}
+type caf_url_auth = string with match(SELF, '^(gssapi|kinit|lwp)$');
+
+@documentation{
+    CAF::Download supported download method: one of lwp or curl
+}
+type caf_url_method = string with match(SELF, '^(lwp|curl)$');
+
+@documentation{
+    A URL that can be handled by CAF::Download
+}
+type caf_url = {
+    'auth' ? caf_url_auth[]
+    'method' ? caf_url_method[]
+    'proto' : string with match(SELF, '^(file|https?)$')
+    'server' ? type_hostname
+    'filename' : string
+
+    #'version' ? string
+
+    'timeout' : long(0..) = 600 # download timeout in seconds (600s * 1kB/s BW = 600kB document)
+    'head_timeout' ? long(0..) # timeout in seconds for initial request which checks for changes/existence
+
+    'retries' : long(0..) = 3 # number retries
+    'retry_wait' : long(0..) = 30 # number of seconds to wait before a retry
+
+    'krb5' ? caf_url_krb5
+    'x509' ? caf_url_x509
+    'proxy' ? caf_url_proxy
+} with {
+    # server is simply ignored with file protocol
+    if ((SELF['proto'] != 'file') && (!(exists(SELF['server'])))) {
+        error("caf url: cannot set server with file protocol");
+    };
+    if(exists(SELF['auth'])) {
+        foreach(idx; auth; SELF['auth']) {
+            if((auth == 'krb5' || auth == 'gssapi') &&
+               ! exists(SELF['krb5'])) {
+                error(format('Cannot set auth %s without setting krb5', auth));
+            };
+            if((auth == 'lwp') && ! exists(SELF['x509'])) {
+                error('Cannot set auth lwp without setting x509');
+            };
+        };
+    };
+
+    true;
+};

src/main/perl/Download.pm

@@ -0,0 +1,142 @@
+# ${license-info}
+# ${developer-info}
+# ${author-info}
+# ${build-info}
+
+package CAF::Download;
+
+use strict;
+use warnings;
+
+# For re-export only
+use CAF::Download::URL qw(set_url_defaults);
+use parent qw(CAF::ObjectText Exporter CAF::Download::URL CAF::Download::Retrieve);
+
+use Readonly;
+use LC::Exception qw (SUCCESS);
+
+our @EXPORT_OK = qw(set_url_defaults);
+
+# TODO: dependencies on curl and kinit
+
+Readonly::Hash my %DOWNLOAD_METHODS => {
+    http => [qw(lwp curl)], # try https, if not, try http
+    https => [qw(lwp curl)], # only https
+    file => [qw(lwp)],
+};
+
+Readonly::Array my @DOWNLOAD_PROTOCOLS => sort keys %DOWNLOAD_METHODS;
+
+# TODO: can we mix and match x509/krb5 also for security like TLS?
+# The GSSAPI doesn't require TLS, it has encryption
+# gssapi here means use the perl GSSAPI bindings to generate the tokens etc
+# kinit means to use commandline tools like kinit/kdestroy
+# x509/lwp means have LWP handle X509 (TLS + X509 auth)
+# TODO: does kinit imply GSSAPI usage?
+Readonly::Hash my %DOWNLOAD_AUTHENTICATION => {
+    krb5 => [qw(gssapi kinit)],
+    x509 => [qw(lwp)],
+};
+
+# Disclaimer: inspired by
+#    NCM::Component::download (15.8)
+#    EDG::WP4::CCM::Fetch (15.8)
+#    File::Fetch (0.48)
+
+=pod
+
+=head1 NAME
+
+CAF::Download - Class for downloading content from remote servers.
+
+=head1 SYNOPSIS
+
+    use CAF::Download;
+
+    my $dl = CAF::Download->new(['https://somewhere/myfile']);
+    print "$dl"; # stringification
+
+    $dl = CAF::TextRender->new(['https://somewhere/else']);
+    # return CAF::FileWriter instance (downloaded text already added)
+    my $fh = $dl->filewriter('/some/path');
+    die "Problem downloading the data" if (!defined($fh));
+    $fh->close();
+
+=head1 DESCRIPTION
+
+This class simplyfies the downloading of content located on remote servers.
+It handles things like authentication, decryption, creating the actual file, ...
+
+=head2 Methods
+
+=over
+
+=item C<_initialize>
+
+Initialize the download object. Arguments:
+
+=over
+
+=item urls
+
+A array reference of urls. Urls will be tried in order, first successful
+one is used. Providing more than one url can thus be used for failover.
+
+See C<prepare_urls> method for a description off the C<urls>.
+
+=back
+
+It takes some extra optional arguments:
+
+=over
+
+=item C<log>
+
+A C<CAF::Reporter> object to log to.
+
+=item C<setup>
+
+Boolean to run the setup (or not). Default/undef is to run setup.
+
+=item C<cleanup>
+
+Boolean to run the cleanup (or not). Default/undef is to run cleanup.
+
+=item destination
+
+The destination of the download, e.g. a filename. This is in particular required
+for download methods that can write to file themself, like C<curl>.
+
+=back
+
+=cut
+
+sub _initialize
+{
+    my ($self, $urls, %opts) = @_;
+
+    $self->{urls} = $self->parse_urls($urls);
+
+    %opts = () if !%opts;
+
+    $self->_initialize_textopts(%opts);
+
+    $self->{setup} = (! defined($opts{setup}) || $opts{setup}) ? 1 : 0;
+    $self->{cleanup} = (! defined($opts{cleanup}) || $opts{cleanup}) ? 1 : 0;
+    $self->debug(1, "setup $self->{setup} cleanup $self->{cleanup}");
+
+    if ($opts{destination}) {
+        $self->{destination} = $self->prepare_destination($opts{destination});
+        $self->debug(1, "download destination set to " . ($self->{destination} || '<UNDEF>'));
+    }
+
+    return SUCCESS;
+}
+
+=pod
+
+=back
+
+=cut
+
+1;