diff --git a/Gemfile b/Gemfile index 1488d263357..8e3c412b43c 100644 --- a/Gemfile +++ b/Gemfile @@ -253,6 +253,7 @@ unless ENV["APPLIANCE"] gem "capybara", "~>2.5.0", :require => false gem "coveralls", :require => false gem "factory_bot", "~>5.1", :require => false + gem "ftpd", "~> 2.1.0", :require => false # TODO: faker is used for url generation in git repository factory and the lenovo # provider, via a xclarity_client dependency diff --git a/config/brakeman.ignore b/config/brakeman.ignore index de2755302c0..219ae2fee6c 100644 --- a/config/brakeman.ignore +++ b/config/brakeman.ignore @@ -21,24 +21,44 @@ "note": "The chomp.to_i ensures we get a number and we protect against 0 with a conditional. The only other possible avenue for attack is if the attacker could replace pgrep, but then they already have root access, so it's a moot point." }, { - "warning_type": "File Access", - "warning_code": 16, - "fingerprint": "4e1918c2d5ff2beacc21db09f696af724d62f1a2a6a101e8e3cb564d0e8a94cd", - "check_name": "FileAccess", - "message": "Model attribute used in file name", - "file": "app/models/miq_report/import_export.rb", - "line": 85, - "link": "http://brakemanscanner.org/docs/warning_types/file_access/", - "code": "YAML.load_file(MiqReport.view_yaml_filename(db, current_user, options))", + "warning_type": "Command Injection", + "warning_code": 14, + "fingerprint": "6a9ec4613af89e29c750be8db27e7b64118ebef6a458357995c51614f26e4f4a", + "check_name": "Execute", + "message": "Possible command injection", + "file": "lib/mount/miq_generic_mount_session.rb", + "line": 34, + "link": "http://brakemanscanner.org/docs/warning_types/command_injection/", + "code": "`#{cmd_str} 2>&1`", "render_path": null, "location": { "type": "method", - "class": "MiqReport::ImportExport::ClassMethods", - "method": "load_from_view_options" + "class": "MiqGenericMountSession", + "method": "s(:self).runcmd" }, - "user_input": "MiqReport.view_yaml_filename(db, current_user, options)", + "user_input": "cmd_str", "confidence": "Medium", - "note": "Temporarily skipped, found in new brakeman version" + "note": "" + }, + { + "warning_type": "Command Injection", + "warning_code": 14, + "fingerprint": "84d4a4e5555b6b750216afadc01f9e385a8a1d56c97b1a8aa3f10925f446932b", + "check_name": "Execute", + "message": "Possible command injection", + "file": "lib/mount/miq_generic_mount_session.rb", + "line": 40, + "link": "http://brakemanscanner.org/docs/warning_types/command_injection/", + "code": "`sudo #{cmd_str} 2>&1`", + "render_path": null, + "location": { + "type": "method", + "class": "MiqGenericMountSession", + "method": "s(:self).runcmd" + }, + "user_input": "cmd_str", + "confidence": "Medium", + "note": "" }, { "warning_type": "Command Injection", @@ -81,6 +101,6 @@ "note": "Temporarily skipped, found in new brakeman version" } ], - "updated": "2017-11-01 11:16:49 -0400", + "updated": "2019-11-22 17:39:13 -0600", "brakeman_version": "3.7.2" } diff --git a/lib/evm_database_ops.rb b/lib/evm_database_ops.rb index 3b51fca3eb9..def9da12b66 100644 --- a/lib/evm_database_ops.rb +++ b/lib/evm_database_ops.rb @@ -2,7 +2,7 @@ require 'util/postgres_admin' require 'mount/miq_generic_mount_session' -require 'util/miq_object_storage' +require 'miq_object_storage' class EvmDatabaseOps include Vmdb::Logging diff --git a/lib/miq_file_storage.rb b/lib/miq_file_storage.rb new file mode 100644 index 00000000000..d756abf92b2 --- /dev/null +++ b/lib/miq_file_storage.rb @@ -0,0 +1,298 @@ +# This class is meant to be a abstract interface for defining a file_storage +# class. +# +# The storage class can either be of a type of "object storage", which includes: +# * protocols like FTP +# * document storage like s3 and OpenStack's Swift +# +# And mountable filesystems like: +# * NFS +# * SMB +# +# The class is meant to allow a shared interface for working with these +# different forms of file storage, while maintaining their differences in +# implementation where necessary. Connection will be handled separately by the +# subclasses, but they must conform to the top level interface. +# +class MiqFileStorage + class InvalidSchemeError < ArgumentError + def initialize(bad_scheme = nil) + super(error_message(bad_scheme)) + end + + def error_message(bad_scheme) + valid_schemes = ::MiqFileStorage.storage_interface_classes.keys.inspect + "#{bad_scheme} is not a valid MiqFileStorage uri scheme. Accepted schemes are #{valid_schemes}" + end + end + + def self.with_interface_class(opts) + klass = fetch_interface_class(opts) + block_given? ? yield(klass) : klass + end + + def self.fetch_interface_class(opts) + return nil unless opts[:uri] + + require 'uri' + scheme, _ = URI.split(URI::DEFAULT_PARSER.escape(opts[:uri])) + klass = storage_interface_classes[scheme] + + raise InvalidSchemeError, scheme if klass.nil? + + klass.new_with_opts(opts) + end + private_class_method :fetch_interface_class + + def self.storage_interface_classes + @storage_interface_classes ||= Interface.descendants.each_with_object({}) do |klass, memo| + memo[klass.uri_scheme] = klass if klass.uri_scheme + end + end + + class Interface + BYTE_HASH_MATCH = /^(?\d+(\.\d+)?)\s*(?K|M|G)?$/i + BYTE_HASH = { + "k" => 1.kilobyte, + "m" => 1.megabyte, + "g" => 1.gigabyte + }.freeze + + attr_reader :remote_file_path, :byte_count, :source_input, :input_writer + + def self.new_with_opts(opts) # rubocop:disable Lint/UnusedMethodArgument + raise NotImplementedError, "#{name}.new_with_opts is not defined" + end + + def self.uri_scheme + nil + end + + # :call-seq: + # add( remote_uri ) { |input_writer| ... } + # add( remote_uri, byte_count ) { |input_writer| ... } + # + # add( local_io, remote_uri ) + # add( local_io, remote_uri, byte_count ) + # + # Add a file to the destination URI. + # + # In the block form of the method, only the remote_uri is required, and it + # is assumed the input will be a generated in the executed block (most + # likely an external process) to a unix pipe that can be written to. The + # pipe generated by this method and passed in to the block as a file + # location to the `input_stream`). + # + # In the non-block form, a source must be provided as the first argument + # either as an IO object that can be read from, or a file path, and the + # second argument is the remote_uri as in the block form. + # + # An additional argument in both forms as the last argument is `byte_count` + # can also be included. If passed, it will be assumed that the resulting + # input will be split, and the naming for the splits will be: + # + # - filename.00001 + # - filename.00002 + # ... + # + # Block form: + # + # nfs_session.add("path/to/file", "200M") do |input_stream| + # `pg_dump -f #{input_stream} vmdb_production` + # end + # + # Non-block form: + # + # nfs_session.add("path/to/local_file", "path/to/remote_file") + # nfs_session.add("path/to/local_file", "path/to/remote_file", "200M") + # + def add(*upload_args, &block) + initialize_upload_vars(*upload_args) + mkdir(File.dirname(@remote_file_path)) + thread = handle_io_block(&block) + result = if byte_count + upload_splits + else + upload_single(@remote_file_path) + end + # `.join` will raise any errors from the thread, so we want to do that + # here (if a thread exists of course). + thread.join if thread + result + ensure + reset_vars + end + alias upload add + + def mkdir(dir) # rubocop:disable Lint/UnusedMethodArgument + raise NotImplementedError, "#{self.class}##{__callee__} is not defined" + end + + # :call-seq: + # download( local_io, remote_uri ) + # download( nil, remote_uri ) { |input_writer| ... } + # + # Download a file from a remote uri. + # + # In non-block form, the remote_uri is saved to the local_io. + # + # In block form, the local_io is omitted, and it is set to a PTY writer + # path that will assumed to be read by the block provided. + def download(local_file, remote_file_uri, &block) + @remote_file_path = remote_file_uri + if block_given? + thread = handle_io_block(&block) + download_single(remote_file_uri, input_writer) + input_writer.close + thread.join + else + download_single(remote_file_uri, local_file) + end + ensure + reset_vars + end + + # :call-seq: + # magic_number_for( remote_uri ) + # magic_number_for( remote_uri, {:accepted => {:key => "magic_str", ...} } ) + # + # Determine a magic number for a remote file. + # + # If no options[:accepted] is passed, then only the first 256 bytes of the + # file are downloaded, and just that data is returned. + # + # If a hash of magic number keys and values for those magic numbers is + # passed, then it will download the largest byte size for the magic number + # values, and compare against the list, returning the first match. + # + # Example: + # + # magics = { :pgdump => PostgresAdmin::PG_DUMP_MAGIC } + # + # magic_number_for("example.org/my_dump.gz", :accepted => magics) + # #=> :pgdump + # magic_number_for("example.org/my_file.rb", :accepted => magics) + # #=> nil + # + # NOTE: This is an extremely niave implementation for remote magic number + # checking, and is only really meant for working with the known magics + # PostgresAdmin. Many other use cases would need to be considered, since + # magic numbers can also checked against the tail of the file, and are not + # limited to the first 256 bytes as has been arbitrarily decided on here. + def magic_number_for(uri, options = {}) + # Amount of bytes to download for checking magic + @byte_count = options.fetch(:accepted, {}).values.map(&:length).max || 256 + uri_data_io = StringIO.new + download_single(uri, uri_data_io) + uri_data = uri_data_io.string + + if (magics = options[:accepted]) + result = magics.detect { |_, magic| uri_data.force_encoding(magic.encoding).start_with?(magic) } + result && result.first + else + uri_data + end + ensure + reset_vars + end + + private + + # NOTE: Needs to be overwritten in the subclass! + # + # Classes that inherit from `MiqFileStorage` need to make sure to create a + # method that overwrites this one to handle the specifics of uploading for + # their particular ObjectStore protocol or MountSession. + # + # `dest_uri` is the current file that will be uploaded. If file splitting + # is occurring, this will update the filename passed into `.add` to include + # a `.0000X` suffix, where the suffix is padded up to 5 digits in total. + # + # `#upload_single` doesn't need to worry about determining the file name + # itself for splitting, but if any relative path munging is necessary, that + # should be done here (see `MiqGenericMountSession#upload_single` for an + # example) + # + # `source_input` available as an attr_reader in this method, and will + # always be a local IO object that is available for reading. + # + # `byte_count` is also an attr_reader that is available, and will either be + # `nil` if no file splitting is occurring, or a integer representing the + # maximum number of bytes to uploaded for this particular `dest_uri`. + # + # + # Ideally, making use of `IO.copy_stream` will simplify this process + # significantly, as you can pass it `source_input`, `dest_uri`, and + # `byte_count` respectively, and it will automatically handle streaming the + # data from one IO object to the other. In mount based situations, where + # `dest_uri` is a file path (in `MiqGenericMountSession#upload_single`, + # this is converted to `relpath`), this does not need to be converted to a + # `File` IO object as `IO.copy_stream` will do that for you. + def upload_single(dest_uri) # rubocop:disable Lint/UnusedMethodArgument + raise NotImplementedError, "#{self.class}#upload_single is not defined" + end + + def upload_splits + @position = 0 + until source_input.eof? + upload_single(next_split_filename) + @position += byte_count + end + end + + def initialize_upload_vars(*upload_args) + upload_args.pop if (@byte_count = parse_byte_value(upload_args.last)) + @remote_file_path = upload_args.pop + + unless upload_args.empty? + source = upload_args.pop + @source_input = source.kind_of?(IO) ? source : File.open(source, "r") + end + end + + def parse_byte_value(bytes) + match = bytes.to_s.match(BYTE_HASH_MATCH) || return + + bytes = match[:BYTE_NUM].to_f + if match[:BYTE_QUALIFIER] + bytes *= BYTE_HASH[match[:BYTE_QUALIFIER].downcase] + end + bytes.to_i + end + + def handle_io_block + if block_given? + require "tmpdir" + + # create pathname, but don't create the file for it (next line) + fifo_path = Pathname.new(Dir::Tmpname.create("") {}) + File.mkfifo(fifo_path) + + # For #Reasons(TM), the reader must be opened first + @source_input = File.open(fifo_path.to_s, IO::RDONLY | IO::NONBLOCK) + @input_writer = File.open(fifo_path.to_s, IO::WRONLY | IO::NONBLOCK) + + Thread.new do + begin + yield fifo_path # send the path to the block to get executed + ensure + @input_writer.close # close the file so we know we hit EOF (for #add) + end + end + end + end + + def reset_vars + File.delete(@input_writer.path) if @input_writer + @position, @byte_count, @remote_file_path, @source_input, @input_writer = nil + end + + def next_split_filename + "#{remote_file_path}.#{'%05d' % (@position / byte_count + 1)}" + end + + def download_single(source, destination) # rubocop:disable Lint/UnusedMethodArgument + raise NotImplementedError, "#{self.class}#download_single is not defined" + end + end +end diff --git a/lib/miq_ftp_lib.rb b/lib/miq_ftp_lib.rb new file mode 100644 index 00000000000..16b934d20c9 --- /dev/null +++ b/lib/miq_ftp_lib.rb @@ -0,0 +1,76 @@ +require 'net/ftp' + +# Helper methods for net/ftp based classes and files. +# +# Will setup a `@ftp` attr_accessor to be used as the return value for +# `.connect`, the main method being provided in this class. +module MiqFtpLib + def self.included(klass) + klass.send(:attr_accessor, :ftp) + end + + def connect(cred_hash = nil) + host = URI(uri).hostname + + begin + _log.info("Connecting to FTP host #{host_ref}...") + @ftp = Net::FTP.new(host) + # Use passive mode to avoid firewall issues see http://slacksite.com/other/ftp.html#passive + @ftp.passive = true + # @ftp.debug_mode = true if settings[:debug] # TODO: add debug option + creds = cred_hash ? [cred_hash[:username], cred_hash[:password]] : login_credentials + @ftp.login(*creds) + _log.info("Successfully connected FTP host #{host_ref}...") + rescue SocketError => err + _log.error("Failed to connect. #{err.message}") + raise + rescue Net::FTPPermError => err + _log.error("Failed to login. #{err.message}") + raise + else + @ftp + end + end + + def file_exists?(file_or_directory) + !ftp.nlst(file_or_directory.to_s).empty? + rescue Net::FTPPermError + false + end + + private + + def host_ref + return @host_ref if @host_ref + @host_ref = URI(uri).hostname + @host_ref << " (#{name})" if respond_to?(:name) + @host_ref + end + + def create_directory_structure(directory_path) + pwd = ftp.pwd + directory_path.to_s.split('/').each do |directory| + unless ftp.nlst.include?(directory) + _log.info("creating #{directory}") + ftp.mkdir(directory) + end + ftp.chdir(directory) + end + rescue Net::FTPPermError + raise unless @username.nil? && @password.nil? + _log.info("introspection of directories disabled. skipping create_directory_structure") + ensure + ftp.chdir(pwd) if pwd + end + + def with_connection(cred_hash = nil) + raise _("no block given") unless block_given? + _log.info("Connecting through #{self.class.name}: [#{host_ref}]") + begin + connect(cred_hash) + yield @ftp + ensure + @ftp.try(:close) && @ftp = nil + end + end +end diff --git a/lib/miq_object_storage.rb b/lib/miq_object_storage.rb new file mode 100644 index 00000000000..f04bd6f7552 --- /dev/null +++ b/lib/miq_object_storage.rb @@ -0,0 +1,62 @@ +require 'net/protocol' +require 'miq_file_storage' + +class MiqObjectStorage < MiqFileStorage::Interface + require 'object_storage/miq_s3_storage' + require 'object_storage/miq_ftp_storage' + require 'object_storage/miq_swift_storage' + + attr_accessor :settings + attr_writer :logger + + DEFAULT_CHUNKSIZE = Net::BufferedIO::BUFSIZE + + def initialize(settings) + raise "URI missing" unless settings.key?(:uri) + @settings = settings.dup + end + + def logger + @logger ||= $log.nil? ? :: Logger.new(STDOUT) : $log + end + + private + + DONE_READING = "".freeze + def read_single_chunk(chunksize = DEFAULT_CHUNKSIZE) + @buf_left ||= byte_count + return DONE_READING.dup unless @buf_left.nil? || @buf_left.positive? + cur_readsize = if @buf_left.nil? || @buf_left - chunksize >= 0 + chunksize + else + @buf_left + end + buf = source_input.read(cur_readsize) + @buf_left -= chunksize if @buf_left + buf.to_s + end + + def write_single_split_file_for(file_io) + loop do + input_data = read_single_chunk + break if input_data.empty? + file_io.write(input_data) + end + clear_split_vars + end + + def write_chunk_proc(destination) + # We use a `proc` here instead of `lambda` because we are only concerned + # about the first argument, and other arguments (additional ones added by + # Excon's response_call signature, for example) are unneeded. + # + # `lambda do` will do argument checking, while `proc do` won't. + proc do |chunk| + destination.write(chunk.force_encoding(destination.external_encoding)) + end + end + + def clear_split_vars + @buf_left = nil + end +end diff --git a/lib/mount/miq_generic_mount_session.rb b/lib/mount/miq_generic_mount_session.rb new file mode 100644 index 00000000000..299ea20dfe3 --- /dev/null +++ b/lib/mount/miq_generic_mount_session.rb @@ -0,0 +1,501 @@ +require 'active_support/core_ext/object/blank' +require 'fileutils' +require 'logger' +require 'sys-uname' +require 'uri' + +require 'util/miq-exception' +require 'util/miq-uuid' +require 'miq_file_storage' + +class MiqGenericMountSession < MiqFileStorage::Interface + require 'mount/miq_local_mount_session' + require 'mount/miq_nfs_session' + require 'mount/miq_smb_session' + require 'mount/miq_glusterfs_session' + + attr_accessor :settings, :mnt_point, :logger + + def initialize(log_settings) + raise "URI missing" unless log_settings.key?(:uri) + @settings = log_settings.dup + @mnt_point = nil + end + + def logger + @logger ||= $log.nil? ? :: Logger.new(STDOUT) : $log + end + + def runcmd(cmd_str) + self.class.runcmd(cmd_str) + end + + def self.runcmd(cmd_str) + rv = `#{cmd_str} 2>&1` + + # If sudo is required, ensure you have /etc/sudoers.d/miq + # Cmnd_Alias MOUNTALL = /bin/mount, /bin/umount + # %wheel ALL = NOPASSWD: MOUNTALL + if $CHILD_STATUS.exitstatus == 1 && cmd_str =~ /^(mount|umount) / + rv = `sudo #{cmd_str} 2>&1` + end + + if $? != 0 + raise rv + end + end + + def self.in_depot_session(opts, &_block) + raise "No block provided!" unless block_given? + session = new_session(opts) + yield session + ensure + session.disconnect if session + end + + def self.new_session(opts) + klass = uri_scheme_to_class(opts[:uri]) + session = klass.new_with_opts(opts) + session.connect + session + end + + def self.new_with_opts(opts) + new(opts.slice(:uri, :username, :password)) + end + + def self.uri_scheme_to_class(uri) + require 'uri' + scheme, userinfo, host, port, registry, share, opaque, query, fragment = URI.split(URI.encode(uri)) + case scheme + when 'smb' + MiqSmbSession + when 'nfs' + MiqNfsSession + when 'glusterfs' + MiqGlusterfsSession + else + raise "unsupported scheme #{scheme} from uri: #{uri}" + end + end + + def mount_share + require 'tmpdir' + @mnt_point = settings_mount_point || Dir.mktmpdir("miq_") + end + + def get_ping_depot_options + @@ping_depot_options ||= begin + opts = ::VMDB::Config.new("vmdb").config[:log][:collection] if defined?(::VMDB) && defined?(::VMDB::CONFIG) + opts = {:ping_depot => false} + opts + end + end + + def ping_timeout + get_ping_depot_options + @@ping_timeout ||= (@@ping_depot_options[:ping_depot_timeout] || 20) + end + + def do_ping? + get_ping_depot_options + @@do_ping ||= @@ping_depot_options[:ping_depot] == true + end + + def pingable? + log_header = "MIQ(#{self.class.name}-pingable?)" + return true unless self.do_ping? + return true unless @settings[:ports].kind_of?(Array) + + res = false + require 'net/ping' + begin + # To prevent "no route to host" type issues, assume refused connection indicates the host is reachable + before = Net::Ping::TCP.econnrefused + Net::Ping::TCP.econnrefused = true + + @settings[:ports].each do |port| + logger.info("#{log_header} pinging: #{@host} on #{port} with timeout: #{ping_timeout}") + tcp1 = Net::Ping::TCP.new(@host, port, ping_timeout) + res = tcp1.ping + logger.info("#{log_header} pinging: #{@host} on #{port} with timeout: #{ping_timeout}...result: #{res}") + break if res == true + end + ensure + Net::Ping::TCP.econnrefused = before + end + + res == true + end + + def connect + log_header = "MIQ(#{self.class.name}-connect)" + + # Replace any encoded spaces back into spaces since the mount commands accepts quoted spaces + @mount_path = @mount_path.to_s.gsub('%20', ' ') + + # # Grab only the share part of a path such as: /temp/default_1/evm_1/current_default_1_evm_1_20091120_192429_20091120_225653.zip + # @mount_path = @mount_path.split("/")[0..1].join("/") + + begin + raise "Connect: Cannot communicate with: #{@host} - verify the URI host value and your DNS settings" unless self.pingable? + + mount_share + rescue MiqException::MiqLogFileMountPointMissing => err + logger.warn("#{log_header} Connecting to host: [#{@host}], share: [#{@mount_path}] encountered error: [#{err.class.name}] [#{err.message}]...retrying after disconnect") + disconnect + retry + rescue => err + if err.kind_of?(RuntimeError) && err.message =~ /No such file or directory/ + msg = "No such file or directory when connecting to host: [#{@host}] share: [#{@mount_path}]" + raise MiqException::MiqLogFileNoSuchFileOrDirectory, msg + end + msg = "Connecting to host: [#{@host}], share: [#{@mount_path}] encountered error: [#{err.class.name}] [#{err.message}]" + logger.error("#{log_header} #{msg}...#{err.backtrace.join("\n")}") + disconnect + raise + end + end + + def disconnect + self.class.disconnect(@mnt_point, logger) + @mnt_point = nil + end + + def self.disconnect(mnt_point, logger = $log) + return if mnt_point.nil? + log_header = "MIQ(#{self.class.name}-disconnect)" + logger.info("#{log_header} Disconnecting mount point: #{mnt_point}") if logger + begin + raw_disconnect(mnt_point) + rescue => err + # Ignore mount point not found/mounted messages + unless err.message =~ /not found|mounted/ + msg = "[#{err.class.name}] [#{err.message}], disconnecting mount point: #{@mnt_point}" + logger.error("#{log_header} #{msg}") if logger + raise + end + end + FileUtils.rmdir(mnt_point) if File.exist?(mnt_point) + + logger.info("#{log_header} Disconnecting mount point: #{mnt_point}...Complete") if logger + end + + def active? + !@mnt_point.nil? + end + + def reconnect! + disconnect + connect + end + + def with_test_file(&_block) + raise "requires a block" unless block_given? + file = '/tmp/miq_verify_test_file' + begin + `echo "testing" > #{file}` + yield file + ensure + FileUtils.rm(file, :force => true) + end + end + + def verify + log_header = "MIQ(#{self.class.name}-verify)" + logger.info("#{log_header} [#{@settings[:uri]}]...") + res = true + + begin + connect + relpath = File.join(@mnt_point, relative_to_mount(@settings[:uri])) + + test_path = 'miqverify/test' + to = File.join(test_path, 'test_file') + fq_file_path = File.join(relpath, to) + + current_test = "create nested directories" + logger.info("#{log_header} [#{@settings[:uri]}] Testing #{current_test}...") + FileUtils.mkdir_p(File.dirname(fq_file_path)) + logger.info("#{log_header} [#{@settings[:uri]}] Testing #{current_test}...complete") + + with_test_file do |from| + current_test = "copy file" + logger.info("#{log_header} [#{@settings[:uri]}] Testing #{current_test}...") + FileUtils.cp(from, fq_file_path) + logger.info("#{log_header} [#{@settings[:uri]}] Testing #{current_test}...complete") + end + + current_test = "delete file" + logger.info("#{log_header} [#{@settings[:uri]}] Testing #{current_test}...") + FileUtils.rm(fq_file_path, :force => true) + logger.info("#{log_header} [#{@settings[:uri]}] Testing #{current_test}...complete") + + current_test = "remove nested directories" + logger.info("#{log_header} [#{@settings[:uri]}] Testing #{current_test}...") + FileUtils.rmdir(File.dirname(fq_file_path)) + FileUtils.rmdir(File.dirname(File.dirname(fq_file_path))) + logger.info("#{log_header} [#{@settings[:uri]}] Testing #{current_test}...complete") + + rescue => err + logger.error("#{log_header} Verify [#{current_test}] failed with error [#{err.class.name}] [#{err}], [#{err.backtrace[0]}]") + res = false, err.to_s + else + res = true, "" + ensure + disconnect + end + logger.info("#{log_header} [#{@settings[:uri]}]...result: [#{res.first}]") + res + end + + def add(*upload_args) + dest_uri = nil + log_header = "MIQ(#{self.class.name}-add)" + + # Don't think this log line is possible when using MiqFileStorage::Interface + # + # logger.info("#{log_header} Source: [#{source}], Destination: [#{dest_uri}]...") + + begin + reconnect! + + dest_uri = super + rescue => err + msg = "Adding [#{source_for_log}] to [#{remote_file_path}], failed due to error: '#{err.message}'" + logger.error("#{log_header} #{msg}") + raise + ensure + disconnect + end + + logger.info("#{log_header} File URI added: [#{remote_file_path}] complete") + dest_uri + end + + def upload_single(dest_uri) + log_header = "MIQ(#{self.class.name}-upload_single)" + relpath = uri_to_local_path(dest_uri) + if File.exist?(relpath) + logger.info("#{log_header} Skipping add since URI: [#{dest_uri}] already exists") + return dest_uri + end + + logger.info("#{log_header} Copying file [#{source_for_log}] to [#{dest_uri}]...") + IO.copy_stream(source_input, relpath, byte_count) + logger.info("#{log_header} Copying file [#{source_for_log}] to [#{dest_uri}] complete") + dest_uri + end + + def download_single(remote_file, local_file) + log_header = "MIQ(#{self.class.name}-download)" + + logger.info("#{log_header} Target: [#{local_file}], Remote file: [#{remote_file}]...") + + begin + reconnect! + relpath = uri_to_local_path(remote_file) + unless File.exist?(relpath) + logger.warn("#{log_header} Remote file: [#{remote_file}] does not exist!") + return + end + + logger.info("#{log_header} Copying file [#{relpath}] to [#{local_file}]...") + IO.copy_stream(relpath, local_file, byte_count) + logger.info("#{log_header} Copying file [#{relpath}] to [#{local_file}] complete") + rescue => err + msg = "Downloading [#{remote_file}] to [#{local_file}], failed due to error: '#{err.message}'" + logger.error("#{log_header} #{msg}") + raise + ensure + disconnect + end + + logger.info("#{log_header} Download File: [#{remote_file}] complete") + local_file + end + + def log_uri_still_configured?(log_uri) + # Only remove the log file if the current depot @settings are based on the same base URI as the log_uri to be removed + return false if log_uri.nil? || @settings[:uri].nil? + + scheme, userinfo, host, port, registry, share, opaque, query, fragment = URI.split(URI.encode(@settings[:uri])) + scheme_log, userinfo_log, host_log, port_log, registry_log, share_log, opaque_log, query_log, fragment_log = URI.split(URI.encode(log_uri)) + + return false if scheme != scheme_log + return false if host != host_log + + # Since the depot URI is a base URI, remove all the directories in the log_uri from the base URI and check for empty? + return false unless (share.split("/") - share_log.split("/")).empty? + true + end + + def remove(log_uri) + log_header = "MIQ(#{self.class.name}-remove)" + + unless self.log_uri_still_configured?(log_uri) + logger.info("#{log_header} Skipping remove because log URI: [#{log_uri}] does not originate from the currently configured base URI: [#{@settings[:uri]}]") + return + end + + relpath = nil + begin + # Samba has issues mount directly in the directory of the file so mount on the parent directory + @settings.merge!(:uri => File.dirname(File.dirname(log_uri))) + reconnect! + + relpath = File.join(@mnt_point, relative_to_mount(log_uri)) + # path is now /temp/default_1/EVM_1/Archive_default_1_EVM_1_20091016_193633_20091016_204855.zip, trim the share and join with the mount point + # /mnt/miq_1258754934/default_1/EVM_1/Archive_default_1_EVM_1_20091016_193633_20091016_204855.zip + # relpath = File.join(@mnt_point, path.split('/')[2..-1] ) + + logger.info("#{log_header} URI: [#{log_uri}] using relative path: [#{relpath}] and mount path: [#{@mount_path}]...") + + unless File.exist?(relpath) + logger.info("#{log_header} Skipping since URI: [#{log_uri}] with relative path: [#{relpath}] does not exist") + return log_uri + end + + logger.info("#{log_header} Deleting [#{relpath}] on [#{log_uri}]...") + FileUtils.rm_rf(relpath) + logger.info("#{log_header} Deleting [#{relpath}] on [#{log_uri}]...complete") + rescue MiqException::MiqLogFileNoSuchFileOrDirectory => err + logger.warn("#{log_header} No such file or directory to delete: [#{log_uri}]") + rescue => err + msg = "Deleting [#{relpath}] on [#{log_uri}], failed due to err '#{err.message}'" + logger.error("#{log_header} #{msg}") + raise + ensure + disconnect + end + + logger.info("#{log_header} URI: [#{log_uri}]...complete") + log_uri + end + + def uri_to_local_path(remote_file) + File.join(@mnt_point, relative_to_mount(remote_file)) + end + + def local_path_to_uri(local_path) + relative_path = Pathname.new(local_path).relative_path_from(Pathname.new(@mnt_point)).to_s + File.join(@settings[:uri], relative_path) + end + + # + # These methods require an existing connection + # + + def glob(pattern) + with_mounted_exception_handling do + Dir.glob("#{mount_root}/#{pattern}").collect { |path| (path.split("/") - mount_root.split("/")).join("/") } + end + end + + def mkdir(path) + with_mounted_exception_handling do + log_header = "MIQ(#{self.class.name}-mkdir)" + new_path = uri_to_local_path(path) + logger.info("#{log_header} Building relative path: [#{new_path}]...") + FileUtils.mkdir_p(new_path) + logger.info("#{log_header} Building relative path: [#{new_path}]...complete") + end + end + + def stat(file) + with_mounted_exception_handling do + File.stat("#{mount_root}/#{file}") + end + end + + def read(file) + with_mounted_exception_handling do + File.read("#{mount_root}/#{file}") + end + end + + def write(file, contents) + with_mounted_exception_handling do + mkdir(File.dirname(file)) + open(file, "w") { |fd| fd.write(contents) } + end + end + + def delete(file_or_directory) + with_mounted_exception_handling do + FileUtils.rm_rf("#{mount_root}/#{file_or_directory}") + end + end + + def open(*args, &block) + with_mounted_exception_handling do + args[0] = "#{mount_root}/#{args[0]}" + File.open(*args, &block) + end + end + + def file?(file) + with_mounted_exception_handling do + File.file?("#{mount_root}/#{file}") + end + end + + protected + + def mount_root + @mnt_point + end + + def relative_to_mount(uri) + log_header = "MIQ(#{self.class.name}-relative_to_mount)" + logger.info("#{log_header} mount point [#{@mount_path}], uri: [#{uri}]...") + scheme, userinfo, host, port, registry, path, opaque, query, fragment = URI.split(URI.encode(uri)) + + # Replace any encoded spaces back into spaces since the mount commands accepts quoted spaces + path.gsub!('%20', ' ') + + raise "path: #{path} or mount_path #{@mount_path} is blank" if path.nil? || @mount_path.nil? || path.empty? || @mount_path.empty? + res = (path.split("/") - @mount_path.split("/")).join("/") + logger.info("#{log_header} mount point [#{@mount_path}], uri: [#{uri}]...relative: [#{res}]") + res + end + + def with_mounted_exception_handling + yield + rescue => err + err.message.gsub!(@mnt_point, @settings[:uri]) + raise + end + + def self.raw_disconnect(mnt_point) + case Sys::Platform::IMPL + when :macosx + runcmd("sudo umount #{mnt_point}") + when :linux + runcmd("umount #{mnt_point}") + else + raise "platform not supported" + end + end + + private + + def settings_read_only? + @settings[:read_only] == true + end + + def settings_mount_point + return nil if @settings[:mount_point].blank? # Check if settings contains the mount_point to use + FileUtils.mkdir_p(@settings[:mount_point]).first + end + + def source_for_log + if @input_writer + "" + elsif @source_input + @source_input.path + else + "" + end + end +end diff --git a/lib/mount/miq_glusterfs_session.rb b/lib/mount/miq_glusterfs_session.rb new file mode 100644 index 00000000000..03d45ca38eb --- /dev/null +++ b/lib/mount/miq_glusterfs_session.rb @@ -0,0 +1,41 @@ +require 'mount/miq_generic_mount_session' + +class MiqGlusterfsSession < MiqGenericMountSession + PORTS = [2049, 111].freeze + + def self.uri_scheme + "glusterfs".freeze + end + + def initialize(log_settings) + super(log_settings.merge(:ports => PORTS)) + end + + def connect + _scheme, _userinfo, @host, _port, _registry, @mount_path, _opaque, _query, _fragment = + URI.split(URI.encode(@settings[:uri])) + super + end + + def mount_share + super + + log_header = "MIQ(#{self.class.name}-mount_share)" + logger.info( + "#{log_header} Connecting to host: [#{@host}], share: [#{@mount_path}] using mount point: [#{@mnt_point}]...") + + mount = "mount" + mount << " -r" if settings_read_only? + + # Quote the host:exported directory since the directory can have spaces in it + case Sys::Platform::IMPL + when :macosx + runcmd("sudo #{mount} -t glusterfs -o resvport '#{@host}:#{@mount_path}' #{@mnt_point}") + when :linux + runcmd("#{mount} -t glusterfs '#{@host}:#{@mount_path}' #{@mnt_point}") + else + raise "platform not supported" + end + logger.info("#{log_header} Connecting to host: [#{@host}], share: [#{@mount_path}]...Complete") + end +end diff --git a/lib/mount/miq_local_mount_session.rb b/lib/mount/miq_local_mount_session.rb new file mode 100644 index 00000000000..fae29bb4f46 --- /dev/null +++ b/lib/mount/miq_local_mount_session.rb @@ -0,0 +1,28 @@ +require 'mount/miq_generic_mount_session' + +# MiqLocalMountSession is meant to be a representation of the local file system +# that conforms to the same interface as MiqLocalMountSession (and by proxy, +# MiqFileSystem::Interface). +# +# See MiqGenericMountSession for info on methods available. +class MiqLocalMountSession < MiqGenericMountSession + def self.uri_scheme + "file".freeze + end + + # no-op these since they are not relavent to the local file system + # + # rubocop:disable Style/SingleLineMethods, Layout/EmptyLineBetweenDefs + def connect; end # :nodoc: + def disconnect; end # :nodoc: + def mount_share; end # :nodoc: + # rubocop:enable Style/SingleLineMethods, Layout/EmptyLineBetweenDefs + + def relative_to_mount(remote_file) # :nodoc: + remote_file + end + + def uri_to_local_path(remote_file) # :nodoc: + File.expand_path(remote_file) + end +end diff --git a/lib/mount/miq_nfs_session.rb b/lib/mount/miq_nfs_session.rb new file mode 100644 index 00000000000..80b51acd0a8 --- /dev/null +++ b/lib/mount/miq_nfs_session.rb @@ -0,0 +1,41 @@ +require 'mount/miq_generic_mount_session' + +class MiqNfsSession < MiqGenericMountSession + PORTS = [2049, 111] + + def self.uri_scheme + "nfs".freeze + end + + def initialize(log_settings) + super(log_settings.merge(:ports => PORTS)) + end + + def connect + scheme, userinfo, @host, port, registry, @mount_path, opaque, query, fragment = URI.split(URI.encode(@settings[:uri])) + super + end + + def mount_share + super + + log_header = "MIQ(#{self.class.name}-mount_share)" + logger.info("#{log_header} Connecting to host: [#{@host}], share: [#{@mount_path}] using mount point: [#{@mnt_point}]...") + # URI: nfs://192.168.252.139/exported/miq + # mount 192.168.252.139:/exported/miq /mnt/miq + + mount = "mount" + mount << " -r" if settings_read_only? + + # Quote the host:exported directory since the directory can have spaces in it + case Sys::Platform::IMPL + when :macosx + runcmd("sudo #{mount} -t nfs -o resvport '#{@host}:#{@mount_path}' #{@mnt_point}") + when :linux + runcmd("#{mount} '#{@host}:#{@mount_path}' #{@mnt_point}") + else + raise "platform not supported" + end + logger.info("#{log_header} Connecting to host: [#{@host}], share: [#{@mount_path}]...Complete") + end +end diff --git a/lib/mount/miq_smb_session.rb b/lib/mount/miq_smb_session.rb new file mode 100644 index 00000000000..59a6b41b2e5 --- /dev/null +++ b/lib/mount/miq_smb_session.rb @@ -0,0 +1,57 @@ +require 'mount/miq_generic_mount_session' + +class MiqSmbSession < MiqGenericMountSession + PORTS = [445, 139] + + def self.uri_scheme + "smb".freeze + end + + def initialize(log_settings) + super(log_settings.merge(:ports => PORTS)) + raise "username is a required value!" if @settings[:username].nil? + raise "password is a required value!" if @settings[:password].nil? + end + + def connect + scheme, userinfo, @host, port, registry, @mount_root, opaque, query, fragment = URI.split(URI.encode(@settings[:uri])) + @mount_path = @mount_root.split("/")[0..1].join("/") + super + end + + def mount_root + File.join(@mnt_point, (@mount_root.split("/") - @mount_path.split("/"))) + end + + def mount_share + super + + log_header = "MIQ(#{self.class.name}-mount_share)" + # Convert backslashes to slashes in case the username is in domain\username format + @settings[:username] = @settings[:username].tr('\\', '/') + + # To work around 2.6.18 kernel issue where a domain could be passed along incorrectly if not specified, explicitly provide both the username and domain (set the domain 'null' if not provided) + # https://bugzilla.samba.org/show_bug.cgi?id=4176 + split_username = @settings[:username].split('/') + case split_username.length + when 1 + # No domain provided + user = split_username.first + domain = 'null' + when 2 + domain, user = split_username + else + raise "Expected 'domain/username' or 'domain\\username' format, received: '#{@settings[:username]}'" + end + + mount = "mount" + mount << " -r" if settings_read_only? + + logger.info("#{log_header} Connecting to host: [#{@host}], share: [#{@mount_path}], domain: [#{domain}], user: [#{user}], using mount point: [#{@mnt_point}]...") + # mount -t cifs //192.168.252.140/temp /media/windows_share/ -o rw,username=jrafaniello,password=blah,domain=manageiq.com + + # Quote the hostname and share and username since they have spaces in it + runcmd("#{mount} -t cifs '//#{File.join(@host, @mount_path)}' #{@mnt_point} -o rw,username='#{user}',password='#{@settings[:password]}',domain='#{domain}'") + logger.info("#{log_header} Connecting to host: [#{@host}], share: [#{@mount_path}]...Complete") + end +end diff --git a/lib/object_storage/miq_ftp_storage.rb b/lib/object_storage/miq_ftp_storage.rb new file mode 100644 index 00000000000..5ce61ad0c8f --- /dev/null +++ b/lib/object_storage/miq_ftp_storage.rb @@ -0,0 +1,102 @@ +require 'miq_ftp_lib' +require 'miq_object_storage' +require 'logger' + +class MiqFtpStorage < MiqObjectStorage + include MiqFtpLib + + attr_reader :uri, :username, :password + + def self.uri_scheme + "ftp".freeze + end + + def self.new_with_opts(opts) + new(opts.slice(:uri, :username, :password)) + end + + def initialize(settings) + super + @uri = @settings[:uri] + @username = @settings[:username] + @password = @settings[:password] + end + + # Override for connection handling + def add(*upload_args) + with_connection { super } + end + + # Override for connection handling + def download(*download_args) + with_connection { super } + end + + # Override for connection handling + def magic_number_for(*magic_number_for_args) + with_connection { super } + end + + # Specific version of Net::FTP#storbinary that doesn't use an existing local + # file, and only uploads a specific size (byte_count) from the input_file + def upload_single(dest_uri) + ftp.synchronize do + ftp.send(:with_binary, true) do + conn = ftp.send(:transfercmd, "STOR #{uri_to_relative(dest_uri)}") + IO.copy_stream(source_input, conn, byte_count) + conn.close + ftp.send(:voidresp) + end + end + dest_uri + rescue Errno::EPIPE + # EPIPE, in this case, means that the data connection was unexpectedly + # terminated. Rather than just raising EPIPE to the caller, check the + # response on the control connection. If getresp doesn't raise a more + # appropriate exception, re-raise the original exception. + ftp.send(:getresp) + raise + end + + def download_single(source, destination) + ftp.synchronize do + ftp.send(:with_binary, true) do + begin + conn = ftp.send(:transfercmd, "RETR #{uri_to_relative(source)}") + IO.copy_stream(conn.io, destination, byte_count) + conn.shutdown(Socket::SHUT_WR) + conn.read_timeout = 1 + ensure + conn.close if conn + end + end + end + end + + def mkdir(dir) + create_directory_structure(uri_to_relative(dir)) + end + + private + + def login_credentials + [username, password].compact + end + + # Currently assumes you have just connected and are at the root logged in + # dir. Net::FTP (or ftp in general) doesn't seem to have a concept of a + # "root dir" based on your login, so this should be used right after + # `.connect`, or shortly there after. + # + # Or, you should be returning to the directory you came from prior to using + # this method again, or not using `ftp.chdir` at all. + def uri_to_relative(filepath) + result = URI.split(filepath)[5] + result = result[1..-1] if result[0] == "/".freeze + result + end + + def _log + logger + end +end diff --git a/lib/object_storage/miq_s3_storage.rb b/lib/object_storage/miq_s3_storage.rb new file mode 100644 index 00000000000..64541a91160 --- /dev/null +++ b/lib/object_storage/miq_s3_storage.rb @@ -0,0 +1,105 @@ +require 'miq_object_storage' + +class MiqS3Storage < MiqObjectStorage + attr_reader :bucket_name + + def self.uri_scheme + "s3".freeze + end + + def self.new_with_opts(opts) + new(opts.slice(:uri, :username, :password, :region)) + end + + def initialize(settings) + super(settings) + + # NOTE: This line to be removed once manageiq-ui-class region change implemented. + @settings[:region] ||= "us-east-1" + @bucket_name = URI(@settings[:uri]).host + + raise "username, password, and region are required values!" if @settings[:username].nil? || @settings[:password].nil? || @settings[:region].nil? + end + + # Extract the path from the URI, so strip off the "s3://" scheme, the bucket + # hostname, leaving only the path minus the leading '/' + def uri_to_object_key(remote_file) + # `path` is `[5]` in the returned result of URI.split + URI.split(remote_file)[5][1..-1] + end + + def upload_single(dest_uri) + object_key = uri_to_object_key(dest_uri) + logger.debug("Writing [#{source_input}] to => Bucket [#{bucket_name}] Key [#{dest_uri}]") + + with_standard_s3_error_handling("uploading", source_input) do + bucket.object(object_key).upload_stream do |write_stream| + IO.copy_stream(source_input, write_stream, byte_count) + end + end + end + + def download_single(source, destination) + object_key = uri_to_object_key(source) + logger.debug("Downloading [#{source}] from bucket [#{bucket_name}] to local file [#{destination}]") + + with_standard_s3_error_handling("downloading", source) do + if destination.kind_of?(IO) || destination.kind_of?(StringIO) + get_object_opts = { + :bucket => bucket_name, + :key => object_key + } + # :range is indexed starting at zero + get_object_opts[:range] = "bytes=0-#{byte_count - 1}" if byte_count + + s3.client.get_object(get_object_opts, &write_chunk_proc(destination)) + else # assume file path + bucket.object(source).download_file(destination) + end + end + end + + # no-op mostly + # + # dirs don't need to be created ahead of time in s3, unlike mounted file + # systems. + # + # For convenience though, calling bucket, which will initialize and create + # (if needed) the s3 bucket to be used for this instance. + def mkdir(_dir) + bucket + end + + def bucket + @bucket ||= s3.bucket(bucket_name).tap do |bucket| + if bucket.exists? + logger.debug("Found bucket #{bucket_name}") + else + logger.debug("Bucket #{bucket_name} does not exist, creating.") + bucket.create + end + end + end + + private + + def s3 + require 'aws-sdk-s3' + + @s3 ||= Aws::S3::Resource.new(:region => @settings[:region], + :access_key_id => @settings[:username], + :secret_access_key => @settings[:password]) + end + + def with_standard_s3_error_handling(action, object) + yield + rescue Aws::S3::Errors::AccessDenied, Aws::S3::Errors::Forbidden => err + logger.error("Access to S3 bucket #{bucket_name} restricted. Try a different name. #{err}") + msg = "Access to S3 bucket #{bucket_name} restricted. Try a different name. #{err}" + raise err, msg, err.backtrace + rescue => err + logger.error("Error #{action} #{object} from S3. #{err}") + msg = "Error #{action} #{object} from S3. #{err}" + raise err, msg, err.backtrace + end +end diff --git a/lib/object_storage/miq_swift_storage.rb b/lib/object_storage/miq_swift_storage.rb new file mode 100644 index 00000000000..82ae907b276 --- /dev/null +++ b/lib/object_storage/miq_swift_storage.rb @@ -0,0 +1,184 @@ +require 'miq_object_storage' + +class MiqSwiftStorage < MiqObjectStorage + attr_reader :container_name + + def self.uri_scheme + "swift".freeze + end + + def self.new_with_opts(opts) + new(opts.slice(:uri, :username, :password)) + end + + def initialize(settings) + super(settings) + @bucket_name = URI(@settings[:uri]).host + + raise "username and password are required values!" if @settings[:username].nil? || @settings[:password].nil? + _scheme, _userinfo, @host, @port, _registry, path, _opaque, query, _fragment = URI.split(URI.encode(@settings[:uri])) + query_params(query) if query + @swift = nil + @username = @settings[:username] + @password = @settings[:password] + + # Omit leading slash (if it exists), and grab the rest of the characters + # before the next file separator + @container_name = path.gsub(/^\/?([^\/]+).*/, '\1') + end + + def uri_to_object_path(remote_file) + # Strip off the leading "swift://" and the container name from the URI" + # Also remove the leading delimiter. + object_file_with_bucket = URI.split(URI.encode(remote_file))[5] + object_file_with_bucket.split(File::Separator)[2..-1].join(File::Separator) + end + + def upload_single(dest_uri) + # + # Get the remote path, and parse out the bucket name. + # + object_file = uri_to_object_path(dest_uri) + # + # write dump file to swift + # + logger.debug("Writing [#{source_input}] to => Bucket [#{container_name}] using object file name [#{object_file}]") + + with_standard_swift_error_handling("uploading") do + swift_file = container.files.new(:key => object_file) + params = { + :expects => [201, 202], + :headers => {}, + :request_block => -> { read_single_chunk }, + :idempotent => false, + :method => "PUT", + :path => "#{Fog::OpenStack.escape(swift_file.directory.key)}/#{Fog::OpenStack.escape(swift_file.key)}" + } + # + # Because of how `Fog::OpenStack` (and probably `Fog::Core`) is designed, + # it has hidden the functionality to provide a block for streaming + # uploads that is available out of the box with Excon. + # + # we use .send here because #request is private + # + # we can't use #put_object (public) directly because it doesn't allow a + # 202 response code, which is what swift responds with when we pass it + # the :request_block (This allows us to stream the response in chunks) + # + swift_file.service.send(:request, params) + + clear_split_vars + end + end + + def download_single(source, destination) + object_file = uri_to_object_path(source) + logger.debug("Downloading [#{source}] from Container [#{container_name}] to local file [#{destination}]") + + with_standard_swift_error_handling("downloading") do + container_key = container.key # also makes sure 'fog/openstack' is loaded + params = { + :expects => [200, 206], + :headers => {}, + :response_block => write_chunk_proc(destination), + :method => "GET", + :path => "#{Fog::OpenStack.escape(container_key)}/#{Fog::OpenStack.escape(object_file)}" + } + # Range is indexed starting at zero + params[:headers]['Range'] = "bytes=0-#{byte_count - 1}" if byte_count + swift.send(:request, params) + end + end + + def mkdir(_dir) + container + end + + # + # Some calls to Fog::Storage::OpenStack::Directories#get will + # return 'nil', and not return an error. This would cause errors down the + # line in '#upload' or '#download'. + # + # Instead of investigating further, we created a new method that is in charge of + # OpenStack container creation, '#create_container', and that is called from '#container' + # if 'nil' is returned from 'swift.directories.get(container_name)', or in the rescue case + # for 'NotFound' to cover that scenario as well + # + + def container(create_if_missing = true) + @container ||= begin + container = swift.directories.get(container_name) + logger.debug("Swift container [#{container}] found") if container + raise Fog::Storage::OpenStack::NotFound unless container + container + rescue Fog::Storage::OpenStack::NotFound + if create_if_missing + logger.debug("Swift container #{container_name} does not exist. Creating.") + create_container + else + msg = "Swift container #{container_name} does not exist. #{err}" + logger.error(msg) + raise err, msg, err.backtrace + end + rescue => err + msg = "Error getting Swift container #{container_name}. #{err}" + logger.error(msg) + raise err, msg, err.backtrace + end + end + + private + + def auth_url + URI::Generic.build( + :scheme => @security_protocol == 'non-ssl' ? "http" : "https", + :host => @host, + :port => @port.to_i, + :path => "/#{@api_version}#{@api_version == "v3" ? "/auth" : ".0"}/tokens" + ).to_s + end + + def swift + return @swift if @swift + require 'fog/openstack' + + connection_params = { + :openstack_auth_url => auth_url, + :openstack_username => @username, + :openstack_api_key => @password, + :openstack_project_domain_id => @domain_id, + :openstack_user_domain_id => @domain_id, + :openstack_region => @region, + :connection_options => { :debug_request => true } + } + + @swift = Fog::Storage::OpenStack.new(connection_params) + end + + def create_container + container = swift.directories.create(:key => container_name) + logger.debug("Swift container [#{container_name}] created") + container + rescue => err + msg = "Error creating Swift container #{container_name}. #{err}" + logger.error(msg) + raise err, msg, err.backtrace + end + + def query_params(query_string) + parts = URI.decode_www_form(query_string).to_h + @region, @api_version, @domain_id, @security_protocol = parts.values_at("region", "api_version", "domain_id", "security_protocol") + end + + def with_standard_swift_error_handling(action) + yield + rescue Excon::Errors::Unauthorized => err + msg = "Access to Swift container #{@container_name} failed due to a bad username or password. #{err}" + logger.error(msg) + raise err, msg, err.backtrace + rescue => err + msg = "Error #{action} #{source_input} to Swift container #{@container_name}. #{err}" + logger.error(msg) + raise err, msg, err.backtrace + end +end diff --git a/spec/lib/miq_file_storage_spec.rb b/spec/lib/miq_file_storage_spec.rb new file mode 100644 index 00000000000..a6ccb394382 --- /dev/null +++ b/spec/lib/miq_file_storage_spec.rb @@ -0,0 +1,435 @@ +require "mount/miq_generic_mount_session" +require "miq_object_storage" + +describe MiqFileStorage do + def opts_for_nfs + opts[:uri] = "nfs://example.com/share/path/to/file.txt" + end + + def opts_for_smb + opts[:uri] = "smb://example.com/share/path/to/file.txt" + opts[:username] = "user" + opts[:password] = "pass" + end + + def opts_for_glusterfs + opts[:uri] = "glusterfs://example.com/share/path/to/file.txt" + end + + def opts_for_ftp + opts[:uri] = "ftp://example.com/share/path/to/file.txt" + end + + def opts_for_swift_without_params + opts[:uri] = "swift://example.com/share/path/to/file.txt" + opts[:username] = "user" + opts[:password] = "pass" + end + + def opts_for_swift_with_params + opts[:uri] = "swift://example.com/share/path/to/file.txt?region=foo" + opts[:username] = "user" + opts[:password] = "pass" + end + + def opts_for_fakefs + opts[:uri] = "foo://example.com/share/path/to/file.txt" + end + + describe ".with_interface_class" do + let(:opts) { {} } + + shared_examples ".with_interface_class implementation" do |class_name| + let(:klass) { Object.const_get(class_name) } + + it "instanciates as #{class_name}" do + interface_instance = described_class.with_interface_class(opts) + expect(interface_instance.class).to eq(klass) + end + + it "with a block, passes the instance, and returns the result" do + instance_double = double(class_name.to_s) + interface_block = ->(instance) { instance.add } + + expect(klass).to receive(:new).and_return(instance_double) + expect(instance_double).to receive(:add).and_return(:foo) + + expect(described_class.with_interface_class(opts, &interface_block)).to eq(:foo) + end + end + + context "with a nil uri" do + it "returns nil" do + expect(described_class.with_interface_class(opts)).to eq(nil) + end + end + + context "with an nfs:// uri" do + before { opts_for_nfs } + + include_examples ".with_interface_class implementation", "MiqNfsSession" + end + + context "with an smb:// uri" do + before { opts_for_smb } + + include_examples ".with_interface_class implementation", "MiqSmbSession" + end + + context "with an glusterfs:// uri" do + before { opts_for_glusterfs } + + include_examples ".with_interface_class implementation", "MiqGlusterfsSession" + end + + context "with an ftp:// uri" do + before { opts_for_ftp } + + include_examples ".with_interface_class implementation", "MiqFtpStorage" + end + + context "with an swift:// uri" do + before { opts_for_swift_with_params } + + include_examples ".with_interface_class implementation", "MiqSwiftStorage" + end + + context "with an swift:// uri and no query params" do + before { opts_for_swift_without_params } + + include_examples ".with_interface_class implementation", "MiqSwiftStorage" + end + + context "with an unknown uri scheme" do + before { opts_for_fakefs } + + it "raises an MiqFileStorage::InvalidSchemeError" do + valid_schemes = MiqFileStorage.storage_interface_classes.keys + error_class = MiqFileStorage::InvalidSchemeError + error_message = "foo is not a valid MiqFileStorage uri scheme. Accepted schemes are #{valid_schemes}" + + expect { described_class.with_interface_class(opts) }.to raise_error(error_class).with_message(error_message) + end + end + end + + ##### Interface Methods ##### + + describe MiqFileStorage::Interface do + shared_examples "an interface method" do |method_str, *args| + subject { method_str[0] == "#" ? described_class.new : described_class } + let(:method) { method_str[1..-1] } + + it "raises NotImplementedError" do + expected_error_message = "MiqFileStorage::Interface#{method_str} is not defined" + expect { subject.send(method, *args) }.to raise_error(NotImplementedError, expected_error_message) + end + end + + shared_examples "upload functionality" do |method| + let(:local_io) { IO.pipe.first } + let(:remote_file_path) { "baz/bar/foo" } + let(:byte_count) { 1234 } + let(:args) { [local_io, remote_file_path] } + + before do + subject.instance_variable_set(:@position, 0) + expect(subject).to receive(:initialize_upload_vars).with(*args).and_call_original + expect(subject).to receive(:handle_io_block).with(no_args) + expect(subject).to receive(:mkdir).with("baz/bar") + end + + it "resets all vars" do + subject.instance_variable_set(:@position, 10) + subject.instance_variable_set(:@byte_count, 10) + allow(subject).to receive(:upload_single) + allow(subject).to receive(:upload_splits) + + subject.send(method, *args) + + expect(subject.instance_variable_get(:@position)).to be nil + expect(subject.byte_count).to be nil + expect(subject.remote_file_path).to be nil + expect(subject.source_input).to be nil + expect(subject.input_writer).to be nil + end + + context "without a byte_count" do + it "calls #upload_single" do + expect(subject).to receive(:upload_single).with(remote_file_path).once + expect(subject).to receive(:upload_splits).never + subject.send(method, *args) + end + end + + context "with a byte_count" do + let(:args) { [local_io, remote_file_path, byte_count] } + + it "calls #upload_splits" do + expect(subject).to receive(:upload_splits).once + expect(subject).to receive(:upload_single).never + subject.send(method, *args) + end + end + end + + describe "#add" do + include_examples "upload functionality", :add + end + + describe "#upload" do + include_examples "upload functionality", :upload + end + + describe "#mkdir" do + it_behaves_like "an interface method", "#mkdir", "foo/bar/baz" + end + + describe "#upload_single" do + it_behaves_like "an interface method", "#upload_single", "path/to/file" + end + + describe "#download_single" do + it_behaves_like "an interface method", "#download_single", "nfs://1.2.3.4/foo", "foo" + end + + describe ".new_with_opts" do + it_behaves_like "an interface method", ".new_with_opts", {} + end + + describe ".uri_scheme" do + it "returns nil by default" do + expect(described_class.uri_scheme).to eq(nil) + end + end + + describe "#upload_splits" do + let(:file_name) { "path/to/file" } + + it "uploads multiple files of the byte count size" do + subject.instance_variable_set(:@position, 0) + subject.instance_variable_set(:@byte_count, 10) + subject.instance_variable_set(:@remote_file_path, file_name) + + source_input_stub = double('@source_input') + allow(subject).to receive(:source_input).and_return(source_input_stub) + allow(source_input_stub).to receive(:eof?).and_return(false, false, true) + + expect(subject).to receive(:upload_single).with("#{file_name}.00001") + expect(subject).to receive(:upload_single).with("#{file_name}.00002") + + subject.send(:upload_splits) + end + end + + describe "#initialize_upload_vars (private)" do + let(:local_io) { File.open(local_io_str) } + let(:local_io_str) { Tempfile.new.path } + let(:remote_path) { "/path/to/remote_file" } + let(:byte_count_int) { 1024 } + let(:byte_count_str) { "5M" } + let(:upload_args) { [] } + let(:pty_master) { double("pty_master") } + let(:pty_slave) { double("pty_slave") } + + before do + subject.send(:initialize_upload_vars, *upload_args) + end + after { FileUtils.rm_rf local_io_str } + + context "with byte_count passed" do + let(:upload_args) { [remote_path, byte_count_int] } + + it "assigns @byte_count to the parse value" do + expect(subject.byte_count).to eq(1024) + end + + it "assigns @remote_file_path" do + expect(subject.remote_file_path).to eq("/path/to/remote_file") + end + + it "assigns @source_input nil (set in #handle_io_block)" do + expect(subject.source_input).to eq(nil) + end + + it "assigns @input_writer nil (set in #handle_io_block)" do + expect(subject.input_writer).to eq(nil) + end + + context "with local_io as an IO object passed" do + let(:upload_args) { [local_io, remote_path, byte_count_str] } + + it "assigns @byte_count to the parse value" do + expect(subject.byte_count).to eq(5.megabytes) + end + + it "assigns @source_input to the passed value" do + expect(subject.source_input).to eq(local_io) + end + + it "@input_writer is nil" do + expect(subject.input_writer).to eq(nil) + end + end + + context "with local_io passed" do + let(:upload_args) { [local_io_str, remote_path, byte_count_str] } + + it "assigns @byte_count to the parse value" do + expect(subject.byte_count).to eq(5.megabytes) + end + + it "assigns @source_input to the passed value" do + expect(File.identical?(subject.source_input, local_io_str)).to be true + end + + it "@input_writer is nil" do + expect(subject.input_writer).to eq(nil) + end + end + end + + context "without byte_count passed" do + let(:upload_args) { [remote_path] } + + it "@byte_count is nil" do + expect(subject.byte_count).to eq(nil) + end + + it "assigns @remote_file_path" do + expect(subject.remote_file_path).to eq("/path/to/remote_file") + end + + it "assigns @source_input nil (set in #handle_io_block)" do + expect(subject.source_input).to eq(nil) + end + + it "assigns @input_writer nil (set in #handle_io_block)" do + expect(subject.input_writer).to eq(nil) + end + + context "with local_io passed" do + let(:upload_args) { [local_io, remote_path] } + + it "assigns @byte_count to the parse value" do + expect(subject.byte_count).to eq(nil) + end + + it "assigns @source_input to the passed value" do + expect(subject.source_input).to eq(local_io) + end + + it "@input_writer is nil" do + expect(subject.input_writer).to eq(nil) + end + end + + context "with local_io passed" do + let(:upload_args) { [local_io_str, remote_path] } + + it "assigns @byte_count to the parse value" do + expect(subject.byte_count).to eq(nil) + end + + it "assigns @source_input to the passed value" do + expect(File.identical?(subject.source_input, local_io_str)).to be true + end + + it "@input_writer is nil" do + expect(subject.input_writer).to eq(nil) + end + end + end + end + + describe "#parse_byte_value (private)" do + it "returns 2 for '2'" do + expect(subject.send(:parse_byte_value, "2")).to eq(2) + end + + it "returns 2048 for '2k'" do + expect(subject.send(:parse_byte_value, "2k")).to eq(2048) + end + + it "returns 1536 for '1.5K'" do + expect(subject.send(:parse_byte_value, "1.5K")).to eq(1536) + end + + it "returns 3145728 for '3M'" do + expect(subject.send(:parse_byte_value, "3M")).to eq(3.megabytes) + end + + it "returns 1073741824 for '1g'" do + expect(subject.send(:parse_byte_value, "1g")).to eq(1.gigabyte) + end + + it "returns nil for nil" do + expect(subject.send(:parse_byte_value, nil)).to eq(nil) + end + + it "returns 100 for 100 (integer)" do + expect(subject.send(:parse_byte_value, 100)).to eq(100) + end + end + + describe "#handle_io_block" do + let(:input_writer) { Tempfile.new } + let(:source_input) { Tempfile.new } + + after do + input_writer.unlink + source_input.unlink + end + + context "with a block" do + let(:block) { ->(_input_writer) { sleep 0.1 } } + + before do + expect(File).to receive(:mkfifo) + expect(File).to receive(:open).and_return(source_input, input_writer) + end + + it "creates a thread for handling the input IO" do + thread_count = Thread.list.count + thread = subject.send(:handle_io_block, &block) + expect(Thread.list.count).to eq(thread_count + 1) + thread.join + end + + it "closes input_writer" do + expect(input_writer.closed?).to eq(false) + thread = subject.send(:handle_io_block, &block) + thread.join + expect(input_writer.closed?).to eq(true) + end + end + + context "without a block" do + it "doesn't create a new thread for IO generation" do + thread_count = Thread.list.count + nil_result = subject.send(:handle_io_block) + + expect(nil_result).to be(nil) + expect(Thread.list.count).to eq(thread_count) + end + end + + context "with a block that causes an error" do + let(:err_block) { ->(_input_writer) { raise "err-mah-gerd" } } + + before do + skip "currently fails consistenly on Travis" + expect(File).to receive(:mkfifo) + expect(File).to receive(:open).and_return(source_input, input_writer) + end + + it "does not hang the process and closes the writer" do + expect(input_writer.closed?).to eq(false) + thread = subject.send(:handle_io_block, &err_block) + expect { thread.join }.to raise_error StandardError + expect(input_writer.closed?).to eq(true) + end + end + end + end +end diff --git a/spec/lib/miq_ftp_lib_spec.rb b/spec/lib/miq_ftp_lib_spec.rb new file mode 100644 index 00000000000..fb462657894 --- /dev/null +++ b/spec/lib/miq_ftp_lib_spec.rb @@ -0,0 +1,193 @@ +require 'miq_ftp_lib' +require 'logger' # probably loaded elsewhere, but for the below classes + +class FTPKlass + include MiqFtpLib + + attr_accessor :uri + + def self.instance_logger + Logger.new(File::NULL) # null logger (for testing) + end + + private + + def _log + self.class.instance_logger + end +end + +class OtherFTPKlass + include MiqFtpLib + + attr_accessor :uri + + def _log + private_log_method + end + + private + + def private_log_method + Logger.new(File::NULL) # null logger (for testing) + end + + def login_credentials + %w(ftpuser ftppass) + end +end + +shared_examples "connecting" do |valid_cred_hash| + let(:cred_hash) { valid_cred_hash } + + before { subject.uri = "ftp://localhost" } + + it "logs in with valid credentials" do + expect { subject.connect(cred_hash) }.not_to raise_error + end + + it "sets the connection to passive" do + subject.connect(cred_hash) + expect(subject.ftp.passive).to eq(true) + end + + context "with an invalid ftp credentials" do + let(:cred_hash) { { :username => "invalid", :password => "alsoinvalid" } } + + it "raises a Net::FTPPermError" do + expect { subject.connect(cred_hash) }.to raise_error(Net::FTPPermError) + end + end +end + +shared_examples "with a connection" do |valid_cred_hash| + let(:cred_hash) { valid_cred_hash } + let(:error_msg) { "no block given" } + + before do + subject.uri = "ftp://localhost" + allow(subject).to receive(:_).with(error_msg).and_return(error_msg) + end + + def with_connection(&block) + subject.send(:with_connection, cred_hash, &block) + end + + def get_socket(ftp) + ftp.instance_variable_get(:@sock).instance_variable_get(:@io) + end + + it "passes the ftp object to the block" do + with_connection do |ftp| + expect(ftp).to be_a(Net::FTP) + expect(subject.ftp).to be(ftp) + end + end + + it "closes the ftp connection after the block is finished" do + ftp_instance = subject.connect(cred_hash) + # stub further calls to `#connect` + expect(subject).to receive(:connect).and_return(ftp_instance) + + with_connection { |ftp| } + expect(subject.ftp).to eq(nil) + expect(ftp_instance.closed?).to eq(true) + end + + it "raises an error if no block is given" do + expect { with_connection }.to raise_error(RuntimeError, error_msg) + end +end + +describe MiqFtpLib do + subject { FTPKlass.new } + + describe "when included" do + it "has a `ftp` accessor" do + ftp_instance = Net::FTP.new + subject.ftp = ftp_instance + + expect(subject.ftp).to eq ftp_instance + end + end + + describe "#connect", :with_ftp_server do + context "with credentials hash" do + subject { FTPKlass.new } + + include_examples "connecting", :username => "ftpuser", :password => "ftppass" + end + + context "with login_credentials method" do + subject { OtherFTPKlass.new } + + include_examples "connecting" + end + end + + describe "#with_connection", :with_ftp_server do + context "with credentials hash" do + subject { FTPKlass.new } + + include_examples "with a connection", :username => "ftpuser", :password => "ftppass" + end + + context "with login_credentials method" do + subject { OtherFTPKlass.new } + + include_examples "with a connection" + end + end + + describe "#file_exists?", :with_ftp_server do + let(:existing_file) { File.basename(existing_ftp_file) } + + subject { FTPKlass.new.tap { |ftp| ftp.uri = "ftp://localhost" } } + before { subject.connect(valid_ftp_creds) } + + it "returns true if the file exists" do + expect(subject.file_exists?(existing_file)).to eq(true) + end + + it "returns false if the file does not exist" do + expect(subject.file_exists?("#{existing_file}.fake")).to eq(false) + end + end + + # Note: Don't use `file_exists?` to try and test the directory existance. + # Most FTP implementations will send the results of `nlst` as the contents of + # a directory if a directory is given. + # + # In our current implementation, this will return a empty list if the + # directory is empty, thus causing the check to fail. Testing against the + # `ftp.nlst(parent_dir)` will make sure the directory in question is included + # in it's parent. + describe "#create_directory_structure", :with_ftp_server do + subject { OtherFTPKlass.new.tap { |ftp| ftp.uri = "ftp://localhost" } } + before { subject.connect(valid_ftp_creds) } + + it "creates a new nested directory" do + new_dir = "foo/bar/baz" + parent_dir = File.dirname(new_dir) + + expect(subject.ftp.nlst(parent_dir).include?("baz")).to eq(false) + subject.send(:create_directory_structure, new_dir) + expect(subject.ftp.nlst(parent_dir).include?("baz")).to eq(true) + end + + context "to an existing directory" do + it "creates the nested directory without messing with the existing" do + existing_dir = existing_ftp_dir + new_dir = File.join(existing_ftp_dir, "foo/bar/baz") + parent_dir = File.dirname(new_dir) + + expect(subject.ftp.nlst.include?(existing_dir)).to eq(true) + expect(subject.ftp.nlst(parent_dir).include?("baz")).to eq(false) + + subject.send(:create_directory_structure, new_dir) + expect(subject.ftp.nlst.include?(existing_dir)).to eq(true) + expect(subject.ftp.nlst(parent_dir).include?("baz")).to eq(true) + end + end + end +end diff --git a/spec/lib/miq_object_storage_spec.rb b/spec/lib/miq_object_storage_spec.rb new file mode 100644 index 00000000000..86dd9945eda --- /dev/null +++ b/spec/lib/miq_object_storage_spec.rb @@ -0,0 +1,120 @@ +require "fileutils" +require "mount/miq_generic_mount_session" +require "miq_object_storage" + +class MockLocalFileStorage < MiqObjectStorage + def initialize(source_path = nil, byte_count = 2.megabytes) + @byte_count = byte_count + @source_input = File.open(source_path, "rb") if source_path + @root_dir = Dir.tmpdir + end + + def mkdir(dir) + FileUtils.mkdir_p(File.join(@root_dir, dir)) + end +end + +describe MiqObjectStorage do + describe "#write_single_split_file_for (private)" do + include_context "generated tmp files" + + subject { MockLocalFileStorage.new source_path } + let(:dest_path) { Dir::Tmpname.create("") { |name| name } } + + it "copies file to splits" do + expected_splitfiles = (1..5).map do |suffix| + "#{dest_path}.0000#{suffix}" + end + + expected_splitfiles.each do |file| + subject.send(:write_single_split_file_for, File.open(file, "wb")) + end + + expected_splitfiles.each do |filename| + expect(File.exist?(filename)).to be true + expect(Pathname.new(filename).lstat.size).to eq(2.megabytes) + end + end + + context "with slightly a slightly smaller input file than 10MB" do + let(:tmpfile_size) { 10.megabytes - 1.kilobyte } + subject { MockLocalFileStorage.new source_path, 1.megabyte } + + it "properly chunks the file" do + expected_splitfiles = (1..10).map do |suffix| + "#{dest_path}.%05d" % {:suffix => suffix} + end + + expected_splitfiles.each do |file| + subject.send(:write_single_split_file_for, File.open(file, "wb")) + end + + expected_splitfiles[0, 9].each do |filename| + expect(File.exist?(filename)).to be true + expect(File.size(filename)).to eq(1.megabytes) + end + + last_split = expected_splitfiles.last + expect(File.exist?(last_split)).to be true + expect(File.size(last_split)).to eq(1.megabyte - 1.kilobyte) + end + end + + context "non-split files (byte_count == nil)" do + subject { MockLocalFileStorage.new source_path, byte_count } + let(:byte_count) { nil } + + it "streams the whole file over" do + subject.send(:write_single_split_file_for, File.open(dest_path, "wb")) + expect(File.exist?(dest_path)).to be true + expect(Pathname.new(dest_path).lstat.size).to eq(tmpfile_size) + end + end + end + + describe "#read_single_chunk (private)" do + include_context "generated tmp files" + + subject { MockLocalFileStorage.new source_path } + let(:dest_path) { Dir::Tmpname.create("") { |name| name } } + let(:chunksize) { MockLocalFileStorage::DEFAULT_CHUNKSIZE } + + it "reads 16384 by default" do + chunk_of_data = subject.send(:read_single_chunk) + expect(chunk_of_data).to eq("0" * chunksize) + end + + it "reads the amount of data equal to chunksize when that is passed" do + chunk_of_data = subject.send(:read_single_chunk, 1.kilobyte) + expect(chunk_of_data).to eq("0" * 1.kilobyte) + end + + context "near the end of the split file" do + let(:data_left) { 123 } + let(:penultimate_chunkize) { chunksize - data_left } + + before do + # read an odd amount of data + read_times = 2.megabytes / chunksize + (read_times - 1).times { subject.send(:read_single_chunk) } + subject.send(:read_single_chunk, penultimate_chunkize) + end + + it "reads only what is necessary to finish the split file" do + chunk_of_data = subject.send(:read_single_chunk) + expect(chunk_of_data).to eq("0" * data_left) + end + + it "stops reading until `#clear_split_vars` is called" do + expect(subject.send(:read_single_chunk)).to eq("0" * data_left) + expect(subject.send(:read_single_chunk)).to eq("") + expect(subject.send(:read_single_chunk)).to eq("") + expect(subject.send(:read_single_chunk)).to eq("") + + subject.send(:clear_split_vars) + + expect(subject.send(:read_single_chunk)).to eq("0" * chunksize) + end + end + end +end diff --git a/spec/lib/mount/miq_generic_mount_session_spec.rb b/spec/lib/mount/miq_generic_mount_session_spec.rb new file mode 100644 index 00000000000..0ab6226ccda --- /dev/null +++ b/spec/lib/mount/miq_generic_mount_session_spec.rb @@ -0,0 +1,28 @@ +require "mount/miq_generic_mount_session" + +describe MiqGenericMountSession do + it "#connect returns a string pointing to the mount point" do + allow(described_class).to receive(:raw_disconnect) + s = described_class.new(:uri => '/tmp/abc') + s.logger = Logger.new("/dev/null") + + result = s.connect + expect(result).to be_kind_of(String) + expect(result).to_not be_blank + + s.disconnect + end + + it "#mount_share is unique" do + expect(described_class.new(:uri => '/tmp/abc').mount_share).to_not eq(described_class.new(:uri => '/tmp/abc').mount_share) + end + + it ".runcmd will retry with sudo if needed" do + cmd = "mount X Y" + expect(described_class).to receive(:`).once.with("#{cmd} 2>&1") + expect(described_class).to receive(:`).with("sudo #{cmd} 2>&1") + expect($CHILD_STATUS).to receive(:exitstatus).once.and_return(1) + + described_class.runcmd(cmd) + end +end diff --git a/spec/lib/mount/miq_local_mount_session_spec.rb b/spec/lib/mount/miq_local_mount_session_spec.rb new file mode 100644 index 00000000000..1999f14783f --- /dev/null +++ b/spec/lib/mount/miq_local_mount_session_spec.rb @@ -0,0 +1,112 @@ +require 'mount/miq_local_mount_session' +require 'tempfile' + +describe MiqLocalMountSession do + let!(:dest_path) { Pathname.new(Dir::Tmpname.create("") {}) } + + subject { described_class.new(:uri => "file://") } + + describe "#add" do + include_context "generated tmp files" + + it "copies single files" do + expect(subject.add(source_path.to_s, dest_path.to_s)).to eq(dest_path.to_s) + expect(File.exist?(dest_path)).to be true + expect(Pathname.new(dest_path).lstat.size).to eq(10.megabytes) + end + + it "copies file to splits" do + expected_splitfiles = (1..5).map do |suffix| + source_path.dirname.join("#{dest_path.basename}.0000#{suffix}") + end + + File.open(source_path) do |f| # with an IO object this time + subject.add(f, dest_path.to_s, "2M") + end + + expected_splitfiles.each do |filename| + expect(File.exist?(filename)).to be true + expect(Pathname.new(filename).lstat.size).to eq(2.megabytes) + end + end + + it "can take input from a command" do + expected_splitfiles = (1..5).map do |suffix| + source_path.dirname.join("#{dest_path.basename}.0000#{suffix}") + end + + subject.add(dest_path.to_s, "2M") do |input_writer| + `#{Gem.ruby} -e "File.write('#{input_writer}', '0' * #{tmpfile_size})"` + end + + expected_splitfiles.each do |filename| + expect(File.exist?(filename)).to be true + expect(Pathname.new(filename).lstat.size).to eq(2.megabytes) + end + end + + context "with a slightly smaller input file than 10MB" do + let(:tmpfile_size) { 10.megabytes - 1.kilobyte } + + it "properly chunks the file" do + expected_splitfiles = (1..10).map do |suffix| + name = "#{dest_path.basename}.%05d" % {:suffix => suffix} + source_path.dirname.join(name) + end + + # using pathnames this time + subject.add(source_path, dest_path.to_s, 1.megabyte) + + expected_splitfiles[0, 9].each do |filename| + expect(File.exist?(filename)).to be true + expect(Pathname.new(filename).lstat.size).to eq(1.megabyte) + end + + last_split = expected_splitfiles.last + expect(File.exist?(last_split)).to be true + expect(Pathname.new(last_split).lstat.size).to eq(1.megabyte - 1.kilobyte) + end + end + end + + describe "#download" do + include_context "generated tmp files" + + it "downloads the file" do + subject.download(dest_path.to_s, source_path.to_s) + expect(File.exist?(dest_path)).to be true + expect(Pathname.new(dest_path).lstat.size).to eq(10.megabytes) + end + + it "can take input from a command" do + source_data = nil + subject.download(nil, source_path) do |input_writer| + source_data = `#{Gem.ruby} -e "print File.read('#{input_writer}')"` + end + + expect(File.exist?(dest_path)).to be false + expect(source_data.size).to eq(10.megabytes) + expect(source_data).to eq(File.read(source_path)) + end + end + + describe "#magic_number_for" do + include_context "generated tmp files" + + it "returns 256 bytes by default" do + result = subject.magic_number_for(source_path) + + expect(result.size).to eq(256) + expect(result).to eq("0" * 256) + end + + describe "with a hash of accepted magics" do + it "returns key for the passed in magic number value" do + magics = { :zero => "000", :one => "1", :foo => "bar" } + result = subject.magic_number_for(source_path, :accepted => magics) + + expect(result).to eq(:zero) + end + end + end +end diff --git a/spec/lib/object_storage/miq_ftp_storage_spec.rb b/spec/lib/object_storage/miq_ftp_storage_spec.rb new file mode 100644 index 00000000000..a1097fe4a89 --- /dev/null +++ b/spec/lib/object_storage/miq_ftp_storage_spec.rb @@ -0,0 +1,140 @@ +require 'object_storage/miq_ftp_storage.rb' + +describe MiqFtpStorage, :with_ftp_server do + subject { described_class.new(ftp_creds.merge(:uri => "ftp://localhost")) } + let(:ftp_creds) { { :username => "ftpuser", :password => "ftppass" } } + + describe "#add" do + include_context "generated tmp files" + + shared_examples "adding files" do |dest_path| + let(:dest_path) { dest_path } + + it "copies single files" do + expect(subject.add(source_path.to_s, dest_path.to_s)).to eq(dest_path.to_s) + expect(dest_path).to exist_on_ftp_server + expect(dest_path).to have_size_on_ftp_server_of(10.megabytes) + end + + it "copies file to splits" do + expected_splitfiles = (1..5).map do |suffix| + "#{dest_path}.0000#{suffix}" + end + + File.open(source_path) do |f| # with an IO object this time + subject.add(f, dest_path.to_s, "2M") + end + + expected_splitfiles.each do |filename| + expect(filename).to exist_on_ftp_server + expect(filename).to have_size_on_ftp_server_of(2.megabytes) + end + end + + it "can take input from a command" do + expected_splitfiles = (1..5).map do |suffix| + "#{dest_path}.0000#{suffix}" + end + + subject.add(dest_path.to_s, "2M") do |input_writer| + `#{Gem.ruby} -e "File.write('#{input_writer}', '0' * #{tmpfile_size})"` + end + + expected_splitfiles.each do |filename| + expect(filename).to exist_on_ftp_server + expect(filename).to have_size_on_ftp_server_of(2.megabytes) + end + end + + context "with slightly a slightly smaller input file than 10MB" do + let(:tmpfile_size) { 10.megabytes - 1.kilobyte } + + it "properly chunks the file" do + expected_splitfiles = (1..10).map do |suffix| + "#{dest_path}.%05d" % {:suffix => suffix} + end + + # using pathnames this time + subject.add(source_path, dest_path.to_s, 1.megabyte) + + expected_splitfiles[0, 9].each do |filename| + expect(filename).to exist_on_ftp_server + expect(filename).to have_size_on_ftp_server_of(1.megabytes) + end + + last_split = expected_splitfiles.last + expect(last_split).to exist_on_ftp_server + expect(last_split).to have_size_on_ftp_server_of(1.megabyte - 1.kilobyte) + end + end + end + + context "using a 'relative path'" do + include_examples "adding files", "path/to/file" + end + + context "using a 'absolute path'" do + include_examples "adding files", "/path/to/my_file" + end + + context "using a uri" do + include_examples "adding files", "ftp://localhost/foo/bar/baz" + end + end + + describe "#download" do + let(:dest_path) { Dir::Tmpname.create("") { |name| name } } + let(:source_file) { existing_ftp_file(10.megabytes) } + let(:source_path) { File.basename(source_file.path) } + + after { File.delete(dest_path) if File.exist?(dest_path) } + + it "downloads the file" do + subject.download(dest_path, source_path) + + # Sanity check that what we are downloading is the size we expect + expect(source_path).to exist_on_ftp_server + expect(source_path).to have_size_on_ftp_server_of(10.megabytes) + + expect(File.exist?(dest_path)).to be true + expect(File.stat(dest_path).size).to eq(10.megabytes) + end + + it "can take input from a command" do + source_data = nil + subject.download(nil, source_path) do |input_writer| + source_data = `#{Gem.ruby} -e "print File.read('#{input_writer}')"` + end + + # Sanity check that what we are downloading is the size we expect + # (and we didn't actually download the file to disk) + expect(File.exist?(dest_path)).to be false + expect(source_path).to exist_on_ftp_server + expect(source_path).to have_size_on_ftp_server_of(10.megabytes) + + # Nothing written, just printed the streamed file in the above command + expect(source_data.size).to eq(10.megabytes) + end + end + + describe "#magic_number_for" do + let(:source_file) { existing_ftp_file(10.megabytes) } + let(:source_path) { File.basename(source_file.path) } + + it "returns 256 bytes by default" do + result = subject.magic_number_for(source_path) + + expect(result.size).to eq(256) + expect(result).to eq("0" * 256) + end + + describe "with a hash of accepted magics" do + it "returns key for the passed in magic number value" do + magics = { :zero => "000", :one => "1", :foo => "bar" } + result = subject.magic_number_for(source_path, :accepted => magics) + + expect(result).to eq(:zero) + end + end + end +end diff --git a/spec/lib/object_storage/miq_s3_storage_spec.rb b/spec/lib/object_storage/miq_s3_storage_spec.rb new file mode 100644 index 00000000000..50b6328cbf5 --- /dev/null +++ b/spec/lib/object_storage/miq_s3_storage_spec.rb @@ -0,0 +1,13 @@ +require "object_storage/miq_s3_storage" + +describe MiqS3Storage do + before(:each) do + @uri = "s3://tmp/abc/def" + @session = described_class.new(:uri => @uri, :username => 'user', :password => 'pass', :region => 'region') + end + + it "#uri_to_object_path returns a new object path" do + result = @session.uri_to_object_key(@uri) + expect(result).to eq("abc/def") + end +end diff --git a/spec/lib/object_storage/miq_swift_storage_spec.rb b/spec/lib/object_storage/miq_swift_storage_spec.rb new file mode 100644 index 00000000000..6874c44a497 --- /dev/null +++ b/spec/lib/object_storage/miq_swift_storage_spec.rb @@ -0,0 +1,111 @@ +require "object_storage/miq_swift_storage" + +describe MiqSwiftStorage do + let(:object_storage) { described_class.new(:uri => uri, :username => 'user', :password => 'pass') } + + describe "#initialize" do + context "using a uri with query parameters" do + let(:uri) { "swift://foo.com:5678/abc/def?region=region&api_version=v3&security_protocol=non-ssl" } + + it "sets the container_name" do + expect(object_storage.container_name).to eq("abc") + end + + it "#uri_to_object_path returns a new object path" do + result = object_storage.uri_to_object_path(uri) + expect(result).to eq("def") + end + end + + context "using a uri without query parameters" do + let(:uri) { "swift://foo.com/abc/def/my_file.tar.gz" } + + it "sets the container_name" do + expect(object_storage.container_name).to eq("abc") + end + + it "#uri_to_object_path returns a new object path" do + result = object_storage.uri_to_object_path(uri) + expect(result).to eq("def/my_file.tar.gz") + end + end + + context "using a uri with only a container_name" do + let(:uri) { "swift://foo.com/container_name" } + + it "sets the container_name" do + expect(object_storage.container_name).to eq("container_name") + end + end + + context "using a uri with only a container_name and params" do + let(:uri) { "swift://foo.com/container_name?region=region&api_version=v3&security_protocol=non-ssl" } + + it "sets the container_name" do + expect(object_storage.container_name).to eq("container_name") + end + + it "sets the region" do + expect(object_storage.instance_variable_get(:@region)).to eq("region") + end + + it "sets the api_version" do + expect(object_storage.instance_variable_get(:@api_version)).to eq("v3") + end + + it "sets the security_protocol" do + expect(object_storage.instance_variable_get(:@security_protocol)).to eq("non-ssl") + end + end + end + + describe "#auth_url (private)" do + context "with non-ssl security protocol" do + let(:uri) { "swift://foo.com:5678/abc/def?region=region&api_version=v3&security_protocol=non-ssl" } + + it "sets the scheme to http" do + expect(URI(object_storage.send(:auth_url)).scheme).to eq("http") + end + + it "sets the host to foo.com" do + expect(URI(object_storage.send(:auth_url)).host).to eq("foo.com") + end + + it "unsets the query string" do + expect(URI(object_storage.send(:auth_url)).query).to eq(nil) + end + end + + context "with ssl security protocol" do + let(:uri) { "swift://foo.com:5678/abc/def?region=region&api_version=v3&security_protocol=ssl" } + + it "sets the scheme to https" do + expect(URI(object_storage.send(:auth_url)).scheme).to eq("https") + end + + it "sets the host to foo.com" do + expect(URI(object_storage.send(:auth_url)).host).to eq("foo.com") + end + + it "unsets the query string" do + expect(URI(object_storage.send(:auth_url)).query).to eq(nil) + end + end + + context "with v3 api version" do + let(:uri) { "swift://foo.com:5678/abc/def?region=region&api_version=v3&security_protocol=ssl" } + + it "sets the path to a v3 path" do + expect(URI(object_storage.send(:auth_url)).path).to eq("/v3/auth/tokens") + end + end + + context "with v2 api version" do + let(:uri) { "swift://foo.com:5678/abc/def?region=region&api_version=v2&security_protocol=ssl" } + + it "sets the path to a v2 path" do + expect(URI(object_storage.send(:auth_url)).path).to eq("/v2.0/tokens") + end + end + end +end diff --git a/spec/support/custom_matchers/exist_on_ftp_server.rb b/spec/support/custom_matchers/exist_on_ftp_server.rb new file mode 100644 index 00000000000..980f6c01940 --- /dev/null +++ b/spec/support/custom_matchers/exist_on_ftp_server.rb @@ -0,0 +1,52 @@ +# Assumes this in run in a :with_ftp_server context so an FTP server on +# localhost is available. +# +# See spec/support/with_ftp_server.rb for more info. +RSpec::Matchers.define :exist_on_ftp_server do + match do |actual| + !list_in_ftp(actual).empty? + end + + failure_message do |actual| + fail_msg(actual) + end + + failure_message_when_negated do |actual| + fail_msg(actual, :negated => true) + end + + def with_connection + Net::FTP.open("localhost") do |ftp| + ftp.login("ftpuser", "ftppass") + yield ftp + end + end + + # Do searches with Net::FTP instead of normal directory scan (even though we + # could) just so we are exercising the FTP interface as expected. + def list_in_ftp(file_or_dir) + with_connection do |ftp| + begin + ftp.nlst(to_path_string(file_or_dir)) + rescue Net::FTPPermError + [] + end + end + end + + def fail_msg(actual, negated: false) + dir = File.dirname(actual) + entries = list_in_ftp(dir) + exist = negated ? "not exist" : "exist" + <<~MSG + expected: #{to_path_string(actual)} to #{exist} in ftp directory" + + Entries for #{dir}: + #{entries.empty? ? " []" : entries.map { |e| " #{e}" }.join("\n")} + MSG + end + + def to_path_string(path) + path.try(:path) || URI.split(path.to_s)[5] + end +end diff --git a/spec/support/custom_matchers/have_size_on_ftp_server_of.rb b/spec/support/custom_matchers/have_size_on_ftp_server_of.rb new file mode 100644 index 00000000000..277aa6f5dde --- /dev/null +++ b/spec/support/custom_matchers/have_size_on_ftp_server_of.rb @@ -0,0 +1,30 @@ +# Assumes this in run in a :with_ftp_server context so an FTP server on +# localhost is available. +# +# See spec/support/with_ftp_server.rb for more info. +RSpec::Matchers.define :have_size_on_ftp_server_of do |expected| + match do |filepath| + size = size_on_ftp(filepath) + size == expected + end + + def with_connection + Net::FTP.open("localhost") do |ftp| + ftp.login("ftpuser", "ftppass") + yield ftp + end + end + + # Do searches with Net::FTP instead of normal directory scan (even though we + # could) just so we are exercising the FTP interface as expected. + def size_on_ftp(file_or_dir) + path = file_or_dir.try(:path) || URI.split(file_or_dir.to_s)[5] + with_connection do |ftp| + begin + ftp.size(path) + rescue Net::FTPPermError + 0 + end + end + end +end diff --git a/spec/support/examples_group/generated_tmp_files.rb b/spec/support/examples_group/generated_tmp_files.rb new file mode 100644 index 00000000000..2300e5e69cf --- /dev/null +++ b/spec/support/examples_group/generated_tmp_files.rb @@ -0,0 +1,28 @@ +shared_context "generated tmp files" do + let!(:tmpfile_size) { 10.megabytes } + let!(:source_path) { Pathname.new(source_file.path) } + let!(:source_file) do + Tempfile.new("source_file").tap do |file| + file.write("0" * tmpfile_size) + file.close + end + end + + after do + # When source_file.unlink is called, it will make it so `source_file.path` + # returns `nil`. Cache it's value incase it hasn't been accessed in the + # tests so we can clear out the generated files properly. + tmp_source_path = source_path + + source_file.unlink + Dir["#{tmp_source_path.expand_path}.*"].each do |file| + File.delete(file) + end + + if defined?(dest_path) && dest_path.to_s.include?(Dir.tmpdir) + Dir["#{dest_path}*"].each do |file| + File.delete(file) + end + end + end +end diff --git a/spec/support/examples_group/with_ftp_server.rb b/spec/support/examples_group/with_ftp_server.rb new file mode 100644 index 00000000000..0add97244ae --- /dev/null +++ b/spec/support/examples_group/with_ftp_server.rb @@ -0,0 +1,90 @@ +require "ftpd" +require "tmpdir" +require "tempfile" +require "fileutils" + +class FtpSingletonServer + class << self + attr_reader :driver + end + + def self.run_ftp_server + @driver = FTPServerDriver.new + @ftp_server = Ftpd::FtpServer.new(@driver) + @ftp_server.on_exception do |e| + STDOUT.puts e.inspect + end + @ftp_server.start + end + + def self.bound_port + @ftp_server.bound_port + end + + def self.stop_ftp_server + @ftp_server.stop + @ftp_server = nil + + @driver.cleanup + @driver = nil + end +end + +class FTPServerDriver + attr_reader :existing_file, :existing_dir + + def initialize + create_tmp_dir + end + + def authenticate(username, password) + username == "ftpuser" && password == "ftppass" + end + + def file_system(_user) + Ftpd::DiskFileSystem.new(@ftp_dir) + end + + def cleanup + FileUtils.remove_entry(@ftp_dir) + end + + def create_existing_file(size = 0) + @existing_file ||= Tempfile.new("", @ftp_dir).tap { |tmp| tmp.puts "0" * size } + end + + # Create a dir under the @ftp_dir, but only return the created directory name + def create_existing_dir + @existing_dir ||= Dir.mktmpdir(nil, @ftp_dir).sub("#{@ftp_dir}/", "") + end + + private + + def create_tmp_dir + @ftp_dir = Dir.mktmpdir + end +end + +shared_context "with ftp server", :with_ftp_server do + before(:all) { FtpSingletonServer.run_ftp_server } + after(:all) { FtpSingletonServer.stop_ftp_server } + + # HACK: Avoid permission denied errors with `ftpd` starting on port 21, but + # our FTP lib always assuming that we are using the default port + # + # The hack basically sets the default port for `Net::FTP` to the bound port + # of the running server + before(:each) do + stub_const("Net::FTP::FTP_PORT", FtpSingletonServer.bound_port) + end + + let(:valid_ftp_creds) { { :username => "ftpuser", :password => "ftppass" } } + + def existing_ftp_file(size = 0) + FtpSingletonServer.driver.create_existing_file(size) + end + + def existing_ftp_dir + FtpSingletonServer.driver.create_existing_dir + end +end