diff --git a/pom.xml b/pom.xml index 576f8105a..086b9d97e 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ org.auscope.portal portal-core jar - 2.8.0-SNAPSHOT + 2.8.1-SNAPSHOT Portal-Core Core functionality common to various AuScope portals. http://portal.auscope.org @@ -77,42 +77,6 @@ - - - - - commons-codec - commons-codec - 1.15 - - - commons-beanutils - commons-beanutils - 1.9.4 - - - io.netty - netty-codec-http - 4.1.92.Final - - - io.netty - netty-codec - 4.1.92.Final - - - org.yaml - snakeyaml - 1.33 - - - com.squareup.okhttp3 - okhttp - 4.10.0 - - - - ${project.artifactId}-${project.version} @@ -316,15 +280,9 @@ org.springframework.boot spring-boot-starter-data-elasticsearch - - com.amazonaws - aws-java-sdk - 1.12.468 - org.apache.commons commons-lang3 - 3.12.0 org.json @@ -347,30 +305,6 @@ xmlunit-legacy 2.9.1 test - - - org.apache.logging.log4j - log4j-core - 2.20.0 - - - javax.mail - mail - - - javax.jms - jms - - - com.sun.jdmk - jmxtools - - - com.sun.jmx - jmxri - - - runtime org.jmock @@ -384,80 +318,34 @@ - - org.slf4j - slf4j-log4j12 - false - edu.ucar netcdf4 5.3.3 false - - org.apache.jclouds - jclouds-all - 2.5.0 - - - javax.annotation - jsr250-api - - - javax.annotation javax.annotation-api 1.3.2 - - commons-io - commons-io - 2.11.0 - - - joda-time - joda-time - 2.12.5 - org.apache.jena apache-jena-libs 4.10.0 pom - - - com.github.openstack4j.core - openstack4j-core - 3.9 com.google.auth google-auth-library-oauth2-http 0.23.0 - - org.apache.lucene - lucene-core - 9.4.1 - - - org.apache.lucene - lucene-queryparser - 9.6.0 - - - org.apache.lucene - lucene-suggest - 9.6.0 - com.google.http-client google-http-client-apache-v2 1.43.2 - - + + com.esotericsoftware kryo 5.5.0 @@ -467,6 +355,6 @@ asm - + diff --git a/src/main/java/org/auscope/portal/core/cloud/CloudDirectoryInformation.java b/src/main/java/org/auscope/portal/core/cloud/CloudDirectoryInformation.java deleted file mode 100644 index 811df35dd..000000000 --- a/src/main/java/org/auscope/portal/core/cloud/CloudDirectoryInformation.java +++ /dev/null @@ -1,97 +0,0 @@ -package org.auscope.portal.core.cloud; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Stack; - -import com.fasterxml.jackson.annotation.JsonIgnore; - - -/** - * Represent a directory in cloud storage, may contain other directories and/or files. - * - * @author woo392 - * - */ -public class CloudDirectoryInformation implements Serializable { - - private static final long serialVersionUID = 8154906555042742255L; - - // Name of directory, this will be the path fragment relative to the parent - private String name; - // Complete path from root to this directory, will be null if this is the job output root directory - private String path; - @JsonIgnore - private CloudDirectoryInformation parent; - // List of files within this directory - private ArrayList files = new ArrayList(); - // List of directories within this directory - private ArrayList directories = new ArrayList(); - - public CloudDirectoryInformation(String name, CloudDirectoryInformation parent) { - this.name = name; - this.parent = parent; - } - - public String getName() { - return this.name; - } - - public void setName(String name) { - this.name = name; - } - - // Path will be built top down from all parents' directory names - public String getPath() { - Stack stack = new Stack(); - CloudDirectoryInformation cloudDir = this; - while (cloudDir != null) { - if(cloudDir.getName() != "") { - stack.push(cloudDir.getName() + "/"); - } - cloudDir = cloudDir.getParent(); - } - String path = ""; - while (!stack.empty()) { - path += stack.pop(); - } - return path; - } - - public void setPath(String path) { - this.path = path; - } - - public CloudDirectoryInformation getParent() { - return parent; - } - - public void setParent(CloudDirectoryInformation parent) { - this.parent = parent; - } - - public ArrayList getFiles() { - return files; - } - - public void setFiles(ArrayList files) { - this.files = files; - } - - public void addFile(CloudFileInformation file) { - this.files.add(file); - } - - public ArrayList getDirectories() { - return directories; - } - - public void setDirectories(ArrayList directories) { - this.directories = directories; - } - - public void addDirectory(CloudDirectoryInformation directory) { - this.directories.add(directory); - } - -} diff --git a/src/main/java/org/auscope/portal/core/cloud/CloudFileInformation.java b/src/main/java/org/auscope/portal/core/cloud/CloudFileInformation.java deleted file mode 100644 index 043203969..000000000 --- a/src/main/java/org/auscope/portal/core/cloud/CloudFileInformation.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * This file is part of the AuScope Virtual Rock Lab (VRL) project. - * Copyright (c) 2009 ESSCC, The University of Queensland - * - * Licensed under the terms of the GNU Lesser General Public License. - */ -package org.auscope.portal.core.cloud; - -import java.io.Serializable; - -/** - * Simple bean class that stores information about a file in cloud storage. - * - * @author Cihan Altinay - * @author Joshua Vote - */ -public class CloudFileInformation implements Serializable { - - /** - * Generated 2012-06-07 - */ - private static final long serialVersionUID = -2300795656821477004L; - /** The file size in bytes */ - private long size; - /** cloud storage key */ - private String cloudKey = ""; - /** - * URL where the file can be accessed by anyone (only valid if file is publicly readable) - */ - private String publicUrl = ""; - /** - * The hash information of the currently stored file (implementation depends on cloud provider). Can be null/empty - */ - private String fileHash = ""; - - /** - * Constructor with name and size - */ - public CloudFileInformation(String cloudKey, long size, String publicUrl) { - this(cloudKey, size, publicUrl, null); - } - - /** - * Constructor with name and size - */ - public CloudFileInformation(String cloudKey, long size, String publicUrl, String fileHash) { - this.cloudKey = cloudKey; - this.size = size; - this.publicUrl = publicUrl; - this.fileHash = fileHash; - } - - /** - * Returns the filename. - * - * @return The filename. - */ - public String getName() { - String[] keyParts = cloudKey.split("/"); - return keyParts[keyParts.length - 1]; - } - - /** - * Returns the file size. - * - * @return The file size in bytes. - */ - public long getSize() { - return size; - } - - /** - * Gets the underlying cloud key representing this file - * - * @return - */ - public String getCloudKey() { - return cloudKey; - } - - /** - * Sets the underlying cloud key representing this file - * - * @param cloudKey - */ - public void setCloudKey(String cloudKey) { - this.cloudKey = cloudKey; - } - - /** - * Gets the public URL where this file can be accessed (assuming the file has its ACL set to public read) - * - * @return - */ - public String getPublicUrl() { - return publicUrl; - } - - /** - * Sets the public URL where this file can be accessed (assuming the file has its ACL set to public read) - * - * @param publicUrl - */ - public void setPublicUrl(String publicUrl) { - this.publicUrl = publicUrl; - } - - /** - * The hash information of the currently stored file (implementation depends on cloud provider). Can be null/empty - * @return - */ - public String getFileHash() { - return fileHash; - } - - /** - * The hash information of the currently stored file (implementation depends on cloud provider). Can be null/empty - * @param hash - */ - public void setFileHash(String fileHash) { - this.fileHash = fileHash; - } - -} diff --git a/src/main/java/org/auscope/portal/core/cloud/CloudFileOwner.java b/src/main/java/org/auscope/portal/core/cloud/CloudFileOwner.java deleted file mode 100644 index 33faf760a..000000000 --- a/src/main/java/org/auscope/portal/core/cloud/CloudFileOwner.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.auscope.portal.core.cloud; - -/** - * Implementors of this class are capable of "owning" a portion of cloud object storage. - * - * @see org.auscope.portal.core.services.cloud.CloudStorageService - * @author Josh Vote - * - */ -public interface CloudFileOwner { - /** - * A user name to associate with these files - * - * @return - */ - public String getUser(); - - /** - * Unique ID identifying this owner - * - * @return - */ - public Integer getId(); - - /** - * The key prefix for all files associated with this object in the specified storage bucket - * - * @return - */ - public String getStorageBaseKey(); - - /** - * The key prefix for all files associated with this job in the specified storage bucket - * - * @param storageBaseKey - * The base key to set. Can be null/empty - * @return - */ - public void setStorageBaseKey(String baseKey); - - /** - * Return job property - * @param key - * The property key - * @return the property value - */ - public String getProperty(String key); - - /** - * Returns the storage bucket to be used to store files. If null, no specific bucket is - * required by this cloud file and the default bucket of the underlying service should be used. - * @return The storage bucket or null - */ - public String getStorageBucket(); -} diff --git a/src/main/java/org/auscope/portal/core/cloud/CloudJob.java b/src/main/java/org/auscope/portal/core/cloud/CloudJob.java deleted file mode 100644 index 4d9c44bde..000000000 --- a/src/main/java/org/auscope/portal/core/cloud/CloudJob.java +++ /dev/null @@ -1,388 +0,0 @@ -package org.auscope.portal.core.cloud; - -import java.io.Serializable; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.HashMap; -import java.util.Map; - -import jakarta.persistence.GeneratedValue; -import jakarta.persistence.GenerationType; -import jakarta.persistence.Id; -import jakarta.persistence.MappedSuperclass; - -/** - * Base class representing the base state of a job that is sent to the cloud for processing. - * - * @author Josh Vote - */ -@MappedSuperclass -public class CloudJob implements Serializable, StagedFileOwner, CloudFileOwner { - - /** - * Generated 2012-06-07 - */ - private static final long serialVersionUID = -3796627138526394662L; - - /** The format used for representing cloud job dates as string */ - public static final String DATE_FORMAT = "yyyyMMdd_HHmmss"; - - /** Unique ID identifying this job */ - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - protected Integer id; - /** Descriptive name of this job */ - protected String name; - /** Long description of this job */ - protected String description; - /** Email address of job submitter */ - protected String emailAddress; - /** user name of job submitter */ - protected String user; - /** date/time when this job was submitted */ - protected Date submitDate; - /** date/time when this job was processed */ - protected Date processDate; - /** descriptive status of this job */ - protected String status; - - /** the ID of the VM that will be used to run this job */ - protected String computeVmId; - /** the ID of the VM instance that is running this job (will be null if no job is currently running) */ - protected String computeInstanceId; - /** The type of the compute instance to start (size of memory, number of CPUs etc) - eg m1.large. Can be null */ - protected String computeInstanceType; - /** The name of the key to inject into the instance at startup for root access. Can be null */ - protected String computeInstanceKey; - /** The unique ID of the storage service this job has been using */ - protected String computeServiceId; - - /** The key prefix for all files associated with this job in the specified storage bucket */ - protected String storageBaseKey; - /** The unique ID of the storage service this job has been using */ - protected String storageServiceId; - - transient protected Map properties = new HashMap<>(); - - public final static String PROPERTY_STS_ARN = "sts_arn"; - public final static String PROPERTY_CLIENT_SECRET = "client_secret"; - public final static String PROPERTY_S3_ROLE = "s3_role"; - - /** - * Creates a new cloud job will null entries for every field - */ - protected CloudJob() { - super(); - } - - /** - * Creates a new cloud job with the following fields - * - * @param id - * Unique ID identifying this job - */ - public CloudJob(Integer id) { - super(); - this.id = id; - } - - public String setProperty(String key, String value) { - if (value == null) { - String oldValue = properties.get(key); - properties.remove(key); - return oldValue; - } - return properties.put(key, value); - } - - @Override - public String getProperty(String key) { - return properties.get(key); - } - - /** - * Unique ID identifying this job - * - * @return - */ - @Override - public Integer getId() { - return id; - } - - /** - * Unique ID identifying this job - * - * @param id - */ - public void setId(Integer id) { - this.id = id; - } - - /** - * Descriptive name of this job - * - * @return - */ - public String getName() { - return name; - } - - /** - * Descriptive name of this job - * - * @param name - */ - public void setName(String name) { - this.name = name; - } - - /** - * Long description of this job - * - * @return - */ - public String getDescription() { - return description; - } - - /** - * Long description of this job - * - * @param description - */ - public void setDescription(String description) { - this.description = description; - } - - /** - * Email address of job submitter - * - * @return - */ - public String getEmailAddress() { - return emailAddress; - } - - /** - * Email address of job submitter - * - * @param emailAddress - */ - public void setEmailAddress(String emailAddress) { - this.emailAddress = emailAddress; - } - - /** - * user name of job submitter - * - * @return - */ - @Override - public String getUser() { - return user; - } - - /** - * user name of job submitter - * - * @param user - */ - public void setUser(String user) { - this.user = user; - } - - /** - * date/time when this job was submitted - * - * @return - */ - public Date getSubmitDate() { - return submitDate; - } - - /** - * date/time when this job was submitted - * - * @param submitDate - */ - public void setSubmitDate(Date submitDate) { - this.submitDate = submitDate; - } - - /** - * date/time when this job was processed - * - * @return - */ - public Date getProcessDate() { - return processDate; - } - - /** - * date/time when this job was processed - * - * @param processDate - */ - public void setProcessDate(Date processDate) { - this.processDate = processDate; - } - - /** - * date/time when this job was submitted (expects a date in the format CloudJob.DATE_FORMAT) - * - * @param submitDate - * @throws ParseException - */ - public void setSubmitDate(String submitDate) throws ParseException { - SimpleDateFormat sdf = new SimpleDateFormat(DATE_FORMAT); - this.setSubmitDate(sdf.parse(submitDate)); - } - - /** - * descriptive status of this job - * - * @return - */ - public String getStatus() { - return status; - } - - /** - * descriptive status of this job - * - * @param status - */ - public void setStatus(String status) { - this.status = status; - } - - /** - * the ID of the VM that will be used to run this job - * - * @return - */ - public String getComputeVmId() { - return computeVmId; - } - - /** - * the ID of the VM that will be used to run this job - * - * @param computeVmId - */ - public void setComputeVmId(String computeVmId) { - this.computeVmId = computeVmId; - } - - /** - * the ID of the VM instance that is running this job (will be null if no job is currently running) - * - * @return - */ - public String getComputeInstanceId() { - return computeInstanceId; - } - - /** - * the ID of the VM instance that is running this job (will be null if no job is currently running) - * - * @param computeInstanceId - */ - public void setComputeInstanceId(String computeInstanceId) { - this.computeInstanceId = computeInstanceId; - } - - /** - * The type of the compute instance to start (size of memory, number of CPUs etc) - eg m1.large. Can be null - */ - public String getComputeInstanceType() { - return computeInstanceType; - } - - /** - * The type of the compute instance to start (size of memory, number of CPUs etc) - eg m1.large. Can be null - */ - public void setComputeInstanceType(String computeInstanceType) { - this.computeInstanceType = computeInstanceType; - } - - /** - * The name of the key to inject into the instance at startup for root access. Can be null - */ - public String getComputeInstanceKey() { - return computeInstanceKey; - } - - /** - * The name of the key to inject into the instance at startup for root access. Can be null - */ - public void setComputeInstanceKey(String computeInstanceKey) { - this.computeInstanceKey = computeInstanceKey; - } - - /** - * The unique ID of the compute service this job has been using - * - * @return - */ - public String getComputeServiceId() { - return computeServiceId; - } - - /** - * The unique ID of the compute service this job has been using - * - * @param computeServiceId - */ - public void setComputeServiceId(String computeServiceId) { - this.computeServiceId = computeServiceId; - } - - /** - * The unique ID of the storage service this job has been using - * - * @return - */ - public String getStorageServiceId() { - return storageServiceId; - } - - /** - * The unique ID of the storage service this job has been using - * - * @param storageServiceId - */ - public void setStorageServiceId(String storageServiceId) { - this.storageServiceId = storageServiceId; - } - - /** - * The key prefix for all files associated with this job in the specified storage bucket - * - * @return - */ - @Override - public String getStorageBaseKey() { - return storageBaseKey; - } - - /** - * The key prefix for all files associated with this job in the specified storage bucket - * - * @param storageBaseKey - */ - @Override - public void setStorageBaseKey(String storageBaseKey) { - this.storageBaseKey = storageBaseKey; - } - - /** - * Default behaviour is to offload bucket requirements to the CloudStorageService. - */ - @Override - public String getStorageBucket() { - return null; - } -} diff --git a/src/main/java/org/auscope/portal/core/cloud/ComputeType.java b/src/main/java/org/auscope/portal/core/cloud/ComputeType.java deleted file mode 100644 index 501c0f857..000000000 --- a/src/main/java/org/auscope/portal/core/cloud/ComputeType.java +++ /dev/null @@ -1,145 +0,0 @@ -package org.auscope.portal.core.cloud; - -import java.io.Serializable; - -/** - * A compute type (or compute flavor) is a simplified collection of virtual machine resources. - * - * VM Images can be instantiated at a compute node using a VM type or flavor that contains these resources. - * - * The values of this class (with the exception of id) are NOT authoritive and exist only for descriptive purposes - * - * @author Josh Vote - * - */ -public class ComputeType implements Serializable { - private static final long serialVersionUID = 5143102635586852517L; - - /** Name of this compute type (valid only at parent compute provider) */ - private String id; - /** Human readable short description of this compute type */ - private String description; - /** How many virtual CPU's does this compute type offer */ - private Integer vcpus; - /** How much RAM (roughly) in MB does this compute type offer */ - private Integer ramMB; - /** How much does the root disk of this compute type offer (in GB) */ - private Integer rootDiskGB; - /** How much does the Ephemeral disk of this compute type offer (in GB) */ - private Integer ephemeralDiskGB; - - public ComputeType(String id, int vcpus, int ramMB) { - this.id=id; - this.vcpus=vcpus; - this.ramMB=ramMB; - } - /** - * - * @param id - * Name of this compute type (valid only at parent compute provider) - */ - public ComputeType(String id) { - super(); - this.id = id; - } - - /** - * Human readable short description of this compute type - * - * @return - */ - public String getDescription() { - return description; - } - - /** - * Human readable short description of this compute type - * - * @param description - */ - public void setDescription(String description) { - this.description = description; - } - - /** - * How many virtual CPU's does this compute type offer - * - * @return - */ - public Integer getVcpus() { - return vcpus; - } - - /** - * How many virtual CPU's does this compute type offer - * - * @param vcpus - */ - public void setVcpus(Integer vcpus) { - this.vcpus = vcpus; - } - - /** - * How much RAM (roughly) in MB does this compute type offer - * - * @return - */ - public Integer getRamMB() { - return ramMB; - } - - /** - * How much RAM (roughly) in MB does this compute type offer - * - * @param ramMB - */ - public void setRamMB(Integer ramMB) { - this.ramMB = ramMB; - } - - /** - * How much does the root disk of this compute type offer (in GB) - * - * @return - */ - public Integer getRootDiskGB() { - return rootDiskGB; - } - - /** - * How much does the root disk of this compute type offer (in GB) - * - * @param rootDiskGB - */ - public void setRootDiskGB(Integer rootDiskGB) { - this.rootDiskGB = rootDiskGB; - } - - /** - * How much does the Ephemeral disk of this compute type offer (in GB) - * - * @return - */ - public Integer getEphemeralDiskGB() { - return ephemeralDiskGB; - } - - /** - * How much does the Ephemeral disk of this compute type offer (in GB) - * - * @param ephemeralDiskGB - */ - public void setEphemeralDiskGB(Integer ephemeralDiskGB) { - this.ephemeralDiskGB = ephemeralDiskGB; - } - - /** - * Name of this compute type (valid only at parent compute provider) - * - * @return - */ - public String getId() { - return id; - } - -} diff --git a/src/main/java/org/auscope/portal/core/cloud/MachineImage.java b/src/main/java/org/auscope/portal/core/cloud/MachineImage.java deleted file mode 100644 index 552d183a5..000000000 --- a/src/main/java/org/auscope/portal/core/cloud/MachineImage.java +++ /dev/null @@ -1,155 +0,0 @@ -package org.auscope.portal.core.cloud; - -import java.io.Serializable; -import java.util.Collection; -import java.util.HashSet; -import java.util.Set; - -/** - * Represents a single virtual machine image that can be used for spawning worker instances. - * - * Contains descriptive information about the image itself which will be shown to a user. - * - * @author Josh Vote - * - */ -public class MachineImage implements Serializable { - private static final long serialVersionUID = -2005086055711041647L; - - /** The unique id of the cloud image - will be used for spawning instances of this image */ - private String imageId; - /** Descriptive short name of this image */ - private String name; - /** Longer description of this image */ - private String description; - /** (Possibly empty) List of descriptive keywords for this image */ - private String[] keywords; - /** The minimum root disk size (in GB) that this image can be run on. Null if this is N/A */ - private Integer minimumDiskGB; - /** The (possibly null) run command that should be used to execute python scripts. If null, most providers will default to 'python'*/ - private String runCommand; - /** A set of optional String annotations for image and/or provider specific metadata. */ - private Set annotations; - - /** - * Creates a new VglMachineImage object - * - * @param imageId - */ - public MachineImage(String imageId) { - super(); - this.imageId = imageId; - this.name = imageId; - this.keywords = new String[0]; - this.annotations = new HashSet(); - } - - /** - * Descriptive short name of this image - * - * @return - */ - public String getName() { - return name; - } - - /** - * The (possibly null) run command that should be used to execute python scripts. If null, most providers will default to 'python' - * @return - */ - public String getRunCommand() { - return runCommand; - } - - /** - * The (possibly null) run command that should be used to execute python scripts. If null, most providers will default to 'python' - * @param runCommand - */ - public void setRunCommand(String runCommand) { - this.runCommand = runCommand; - } - - /** - * A set of optional String annotations for image and/or provider specific metadata. - */ - public Set getAnnotations() { - return this.annotations; - } - - /** - * A set of optional String annotations for image and/or provider specific metadata. - */ - public void setAnnotations(Collection annotations) { - this.annotations.clear(); - this.annotations.addAll(annotations); - } - - /** - * Descriptive short name of this image - * - * @param name - */ - public void setName(String name) { - this.name = name; - } - - /** - * Longer description of this image - * - * @return - */ - public String getDescription() { - return description; - } - - /** - * Longer description of this image - * - * @param description - */ - public void setDescription(String description) { - this.description = description; - } - - /** - * (Possibly empty) List of descriptive keywords for this image - * - * @return - */ - public String[] getKeywords() { - return keywords; - } - - /** - * (Possibly empty) List of descriptive keywords for this image - * - * @param keywords - */ - public void setKeywords(String[] keywords) { - this.keywords = keywords; - } - - /** - * The unique id of the cloud image - will be used for spawning instances of this image - * - * @return - */ - public String getImageId() { - return imageId; - } - - /** The minimum root disk size (in GB) that this image can be run on. Null if this is N/A */ - public Integer getMinimumDiskGB() { - return minimumDiskGB; - } - - /** The minimum root disk size (in GB) that this image can be run on. Null if this is N/A */ - public void setMinimumDiskGB(Integer minimumDiskGB) { - this.minimumDiskGB = minimumDiskGB; - } - - @Override - public String toString() { - return "MachineImage [imageId=" + imageId + ", name=" + name + "]"; - } -} diff --git a/src/main/java/org/auscope/portal/core/cloud/StagedFile.java b/src/main/java/org/auscope/portal/core/cloud/StagedFile.java deleted file mode 100644 index dfd344c57..000000000 --- a/src/main/java/org/auscope/portal/core/cloud/StagedFile.java +++ /dev/null @@ -1,115 +0,0 @@ -package org.auscope.portal.core.cloud; - -import java.io.File; - -/** - * Represents a local file (stored somewhere on the file system) that belongs to a job. - * - * @author Joshua - * - */ -public class StagedFile { - /** The job that owns this staged file */ - private StagedFileOwner owner; - /** The name of this staged file (unique per job) */ - private String name; - /** can be null - the underlying reference to the HDD where this file is staged */ - private File file; - - /** - * Creates a new instance - * - * @param owner - * The job that owns this staged file - * @param name - * The name of this staged file (unique per job) - * @param file - * can be null - the underlying reference to the HDD where this file is staged - */ - public StagedFile(StagedFileOwner owner, String name, File file) { - super(); - this.owner = owner; - this.name = name; - this.file = file; - } - - /** - * The job that owns this staged file - * - * @return - */ - public StagedFileOwner getOwner() { - return owner; - } - - /** - * The job that owns this staged file - * - * @param owner - */ - public void setOwner(StagedFileOwner owner) { - this.owner = owner; - } - - /** - * The name of this staged file (unique per job) - * - * @return - */ - public String getName() { - return name; - } - - /** - * The name of this staged file (unique per job) - * - * @param name - */ - public void setName(String name) { - this.name = name; - } - - /** - * can be null - the underlying reference to the HDD where this file is staged - * - * @return - */ - public File getFile() { - return file; - } - - /** - * can be null - the underlying reference to the HDD where this file is staged - * - * @param file - */ - public void setFile(File file) { - this.file = file; - } - - @Override - public String toString() { - return "StagedFile [owner=" + owner + ", name=" + name + "]"; - } - - /** - * Returns true if obj is a StagedFile with equal name AND owner - */ - @Override - public boolean equals(Object obj) { - if (obj instanceof StagedFile) { - return owner.equals(((StagedFile) obj).owner) && name.equals(((StagedFile) obj).name); - } - - return false; - } - - /** - * Generates a hashcode based on owner and name - */ - @Override - public int hashCode() { - return owner.hashCode() ^ name.hashCode(); - } - -} diff --git a/src/main/java/org/auscope/portal/core/cloud/StagedFileOwner.java b/src/main/java/org/auscope/portal/core/cloud/StagedFileOwner.java deleted file mode 100644 index 9834a2f29..000000000 --- a/src/main/java/org/auscope/portal/core/cloud/StagedFileOwner.java +++ /dev/null @@ -1,21 +0,0 @@ -/** - * - */ -package org.auscope.portal.core.cloud; - -/** - * To be implemented by classes that want to own files from the FileStagingService - * - * @author fri096 - * - */ -public interface StagedFileOwner { - - /** - * Return unique identifier - * - * @return unique identifier - */ - Integer getId(); - -} diff --git a/src/main/java/org/auscope/portal/core/cloud/StagingInformation.java b/src/main/java/org/auscope/portal/core/cloud/StagingInformation.java deleted file mode 100644 index 40acb58ca..000000000 --- a/src/main/java/org/auscope/portal/core/cloud/StagingInformation.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.auscope.portal.core.cloud; - -/** - * A simple POJO for storing all information about where a portal can stage in job files - * - * @author Josh Vote - * - */ -public class StagingInformation { - private String stageInDirectory; - - /** - * Creates a new instance - * - * @param stageInDirectory - * Gets where the portal can add/remove directories with impunity for the purpose of staging in new job directories - */ - public StagingInformation(String stageInDirectory) { - super(); - this.stageInDirectory = stageInDirectory; - } - - /** - * Gets where the portal can add/remove directories with impunity for the purpose of staging in new job directories - * - * @return - */ - public String getStageInDirectory() { - return stageInDirectory; - } - - /** - * Sets where the portal can add/remove directories with impunity for the purpose of staging in new job directories - * - * @param stageInDirectory - */ - public void setStageInDirectory(String stageInDirectory) { - this.stageInDirectory = stageInDirectory; - } -} diff --git a/src/main/java/org/auscope/portal/core/services/cloud/CloudComputeService.java b/src/main/java/org/auscope/portal/core/services/cloud/CloudComputeService.java deleted file mode 100644 index d4dd0377f..000000000 --- a/src/main/java/org/auscope/portal/core/services/cloud/CloudComputeService.java +++ /dev/null @@ -1,298 +0,0 @@ -package org.auscope.portal.core.services.cloud; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.auscope.portal.core.cloud.CloudJob; -import org.auscope.portal.core.cloud.ComputeType; -import org.auscope.portal.core.cloud.MachineImage; -import org.auscope.portal.core.services.PortalServiceException; - -/** - * Service class wrapper for interacting with a remote cloud compute service using CloudJob objects. - * - * @author Josh Vote - */ -abstract public class CloudComputeService { - - public enum ProviderType { - /** Connect to an Openstack instance via the Keystone Identity service */ - NovaKeystone, - /** Connect to an Amazon Web Services instance via EC2 */ - AWSEc2, - /** Connect to GADI NCI HPC */ - GADI - } - - /** - * The status of a compute instance (not the job status) as reported by the remote cloud. - * @author Josh Vote (CSIRO) - * - */ - public enum InstanceStatus { - /** - * Job is still waiting to start - */ - Pending, - /** - * Instance is running - */ - Running, - /** - * The instance could not be found or it's in a terminated state. - */ - Missing, - } - - @SuppressWarnings("unused") - private final Log logger = LogFactory.getLog(getClass()); - - /** Unique ID for distinguishing instances of this class - can be null */ - private String id; - /** A short descriptive name for human identification of this service */ - private String name; - /** What type of cloud service are we communicating with */ - private ProviderType provider; - - /** A group name that all jobs will be assigned to */ - private String groupName = "portal-cloud-compute-service"; - - /** An array of images that are available through this compute service */ - private MachineImage[] availableImages = new MachineImage[0]; - - /** - * Name of the developers' keypair to inject into instances on this provider. - */ - private String keypair; - - /** Cloud endpoint to connect to */ - private String endpoint; - - public ProviderType getProvider() { - return provider; - } - - public String getEndpoint() { - return endpoint; - } - - public void setEndpoint(String endpoint) { - this.endpoint = endpoint; - } - - public String getApiVersion() { - return apiVersion; - } - - public void setApiVersion(String apiVersion) { - this.apiVersion = apiVersion; - } - - /** - * Cloud API version - */ - private String apiVersion; - - /** - * Creates a new instance with the specified credentials (no endpoint specified - ensure provider type has a fixed endpoint) - * - * @param accessKey - * The Compute Access key (user name) - * @param secretKey - * The Compute Secret key (password) - * - */ - public CloudComputeService(ProviderType provider) { - this(provider, null, null); - } - - /** - * Creates a new instance with the specified credentials - * - * @param endpoint - * (URL) The location of the Compute (Nova) service - * @param accessKey - * The Compute Access key (user name) - * @param secretKey - * The Compute Secret key (password) - * - */ - public CloudComputeService(ProviderType provider, String endpoint) { - this(provider, endpoint, null); - } - - /** - * Creates a new instance with the specified credentials - * - * @param endpoint - * (URL) The location of the Compute (Nova) service - * @param accessKey - * The Compute Access key (user name) - * @param secretKey - * The Compute Secret key (password) - * @param apiVersion - * The API version - */ - public CloudComputeService(ProviderType provider, String endpoint, String apiVersion) { - this.provider = provider; - this.endpoint = endpoint; - this.apiVersion = apiVersion; - } - - /** - * Unique ID for distinguishing instances of this class - can be null - * - * @return - */ - public String getId() { - return id; - } - - /** - * Unique ID for distinguishing instances of this class - can be null - * - * @param id - */ - public void setId(String id) { - this.id = id; - } - - /** A group name that all jobs will be assigned to */ - public String getGroupName() { - return groupName; - } - - /** A group name that all jobs will be assigned to */ - public void setGroupName(String groupName) { - this.groupName = groupName; - } - - /** - * An array of images that are available through this compute service - * - * @return - */ - public MachineImage[] getAvailableImages() { - return availableImages; - } - - /** - * An array of images that are available through this compute service - * - * @param availableImages - */ - public void setAvailableImages(MachineImage[] availableImages) { - this.availableImages = availableImages; - } - - /** - * A short descriptive name for human identification of this service - * - * @return - */ - public String getName() { - return name; - } - - /** - * A short descriptive name for human identification of this service - * - * @param name - */ - public void setName(String name) { - this.name = name; - } - - /** - * Begins execution of the specified job and returns the ID of the started instance. - * - * This function will create a VM to run the job which will be responsible for decoding the userDataString and downloading any input files from the - * JobStorageService - * - * @param job - * The job to execute - * @param userDataString - * A string that is made available to the job when it starts execution (this will be Base64 encoded before being sent to the VM) - * @return null if execution fails or the instance ID of the running VM - */ - abstract public String executeJob(CloudJob job, String userDataString) throws PortalServiceException; - - /** - * Makes a request that the VM started by job be terminated - * - * @param job - * The job whose execution should be terminated - * @throws PortalServiceException - */ - abstract public void terminateJob(CloudJob job) throws PortalServiceException; - - public ComputeType[] getAvailableComputeTypes() { - return getAvailableComputeTypes(null, null, null); - } - - /** - * An array of compute types that are available through this compute service - */ - abstract public ComputeType[] getAvailableComputeTypes(Integer minimumVCPUs, Integer minimumRamMB, Integer minimumRootDiskGB); - - /** - * Return the ssh keypair to be used with the VM - * @return - */ - public String getKeypair() { - return keypair; - } - - /** - * Sets the ssh keypair to be used with the VM - * @param keypair - */ - public void setKeypair(String keypair) { - this.keypair = keypair; - } - - /** - * Will attempt to tail and return the last 1000 lines from the given servers console. - * - * @param job - * the job which has been executed by this service - * @param numLines - * the number of console lines to return - * @return console output as string or null - * @return - */ - public String getConsoleLog(CloudJob job) throws PortalServiceException { - return getConsoleLog(job, 1000); - } - - /** - * Will attempt to tail and return the last {@code numLines} from the given servers console. - * - * @param job - * the job which has been executed by this service - * @param numLines - * the number of console lines to return - * @return console output as string or null - * @return - */ - abstract public String getConsoleLog(CloudJob job, int numLines) throws PortalServiceException; - - /** - * Attempts to lookup low level status information about this job's compute instance from the remote cloud. - * - * Having no computeInstanceId set will result in an exception being thrown. - * - * @param job - * @return - * @throws PortalServiceException - */ - abstract public InstanceStatus getJobStatus(CloudJob job) throws PortalServiceException; - - /** - * Return all VM types that can run the specified image - * - * @param machineImageId - * @return all VM types that can run the specified image - * @throws PortalServiceException - */ - abstract public ComputeType[] getAvailableComputeTypes(String machineImageId) throws PortalServiceException; -} diff --git a/src/main/java/org/auscope/portal/core/services/cloud/CloudComputeServiceAws.java b/src/main/java/org/auscope/portal/core/services/cloud/CloudComputeServiceAws.java deleted file mode 100644 index 9cc60da4d..000000000 --- a/src/main/java/org/auscope/portal/core/services/cloud/CloudComputeServiceAws.java +++ /dev/null @@ -1,465 +0,0 @@ -package org.auscope.portal.core.services.cloud; - -import java.io.UnsupportedEncodingException; -import java.security.CodeSource; -import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.List; -import java.util.concurrent.TimeUnit; - -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.SAXParserFactory; -import javax.xml.transform.TransformerFactory; -import javax.xml.xpath.XPathFactory; - -import org.apache.http.HttpStatus; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.auscope.portal.core.cloud.CloudJob; -import org.auscope.portal.core.cloud.ComputeType; -import org.auscope.portal.core.services.PortalServiceException; -import org.auscope.portal.core.util.TextUtil; - -import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.auth.BasicSessionCredentials; -import com.amazonaws.services.ec2.AmazonEC2; -import com.amazonaws.services.ec2.AmazonEC2Client; -import com.amazonaws.services.ec2.model.BlockDeviceMapping; -import com.amazonaws.services.ec2.model.CreateTagsRequest; -import com.amazonaws.services.ec2.model.DescribeImagesRequest; -import com.amazonaws.services.ec2.model.DescribeImagesResult; -import com.amazonaws.services.ec2.model.DescribeInstanceStatusRequest; -import com.amazonaws.services.ec2.model.DescribeInstanceStatusResult; -import com.amazonaws.services.ec2.model.GetConsoleOutputRequest; -import com.amazonaws.services.ec2.model.GetConsoleOutputResult; -import com.amazonaws.services.ec2.model.IamInstanceProfileSpecification; -import com.amazonaws.services.ec2.model.Image; -import com.amazonaws.services.ec2.model.Instance; -import com.amazonaws.services.ec2.model.RunInstancesRequest; -import com.amazonaws.services.ec2.model.RunInstancesResult; -import com.amazonaws.services.ec2.model.Tag; -import com.amazonaws.services.ec2.model.TerminateInstancesRequest; -import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient; -import com.amazonaws.services.securitytoken.model.AssumeRoleRequest; -import com.amazonaws.services.securitytoken.model.AssumeRoleResult; - -/** - * Service class wrapper for interacting with a remote cloud compute service - * using CloudJob objects. - * - * @author Josh Vote - */ -public class CloudComputeServiceAws extends CloudComputeService { - /** - * Any getStatus request on a job whose submission time is less than STATUS_PENDING_SECONDS seconds - * away from the current time will be forced to return a Pending status (ignoring any status checks) - * - * This is to avoid missing errors occurring when AWS hasn't fully caught up to the new VM. - */ - public static final long STATUS_PENDING_SECONDS = 30; - - private final Log logger = LogFactory.getLog(getClass()); - - private String devAccessKey; - - private String devSecretKey; - - private STSRequirement stsRequirement = STSRequirement.Permissable; - - private String devSessionKey; - - /** - * Creates a new instance with the specified credentials (no endpoint - * specified - ensure provider type has a fixed endpoint) - * - * @param accessKey - * The Compute Access key (user name) - * @param secretKey - * The Compute Secret key (password) - * @param sessionKey - * The Compute Session key (password) - * - */ - public CloudComputeServiceAws(String accessKey, String secretKey, String sessionKey) { - this(null, accessKey, secretKey, null, sessionKey); - } - - /** - * Creates a new instance with the specified credentials (no endpoint - * specified - ensure provider type has a fixed endpoint) - * - * @param accessKey - * The Compute Access key (user name) - * @param secretKey - * The Compute Secret key (password) - * - */ - public CloudComputeServiceAws(String accessKey, String secretKey) { - this(null, accessKey, secretKey, null, null); - } - - private static String getJaxpImplementationInfo(String componentName, Class componentClass) { - CodeSource source = componentClass.getProtectionDomain().getCodeSource(); - return MessageFormat.format("{0} implementation: {1} loaded from: {2}", componentName, componentClass.getName(), - source == null ? "Java Runtime" : source.getLocation()); - } - - /** - * Creates a new instance with the specified credentials - * - * @param endpoint - * (URL) The location of the Compute (Nova) service - * @param accessKey - * The Compute Access key (user name) - * @param secretKey - * The Compute Secret key (password) - * @param apiVersion - * The API version - */ - public CloudComputeServiceAws(String endpoint, String accessKey, String secretKey, String apiVersion, String sessionKey) { - super(ProviderType.AWSEc2, endpoint, apiVersion); - this.devAccessKey = accessKey; - this.devSecretKey = secretKey; - this.devSessionKey = sessionKey; - - logger.debug( - getJaxpImplementationInfo("DocumentBuilderFactory", DocumentBuilderFactory.newInstance().getClass())); - logger.debug(getJaxpImplementationInfo("XPathFactory", XPathFactory.newInstance().getClass())); - logger.debug(getJaxpImplementationInfo("TransformerFactory", TransformerFactory.newInstance().getClass())); - logger.debug(getJaxpImplementationInfo("SAXParserFactory", SAXParserFactory.newInstance().getClass())); - - } - - /** - * Returns whether AWS cross account authorization is mandatory, optional or forced off - * @return - */ - public STSRequirement getStsRequirement() { - return stsRequirement; - } - - /** - * Sets whether AWS cross account authorization is mandatory, optional or forced off - * @param stsRequirement - */ - public void setStsRequirement(STSRequirement stsRequirement) { - this.stsRequirement = stsRequirement; - } - - protected AWSCredentials getCredentials(String arn, String clientSecret) throws PortalServiceException { - if (stsRequirement == STSRequirement.ForceNone) { - arn = null; - clientSecret = null; - } - - if (!TextUtil.isNullOrEmpty(arn)) { - if (TextUtil.isNullOrEmpty(clientSecret)) - throw new PortalServiceException("Job ARN set, but no client secret"); - - AWSSecurityTokenServiceClient stsClient; - - if (!TextUtil.isAnyNullOrEmpty(devAccessKey, devSecretKey)) { - AWSCredentials awsCredentials; - - if(!TextUtil.isAnyNullOrEmpty(devSessionKey)) { - awsCredentials = new BasicSessionCredentials(devAccessKey, devSecretKey, devSessionKey); - } else { - awsCredentials = new BasicAWSCredentials(devAccessKey, devSecretKey); - } - stsClient = new AWSSecurityTokenServiceClient(awsCredentials); - } else { - stsClient = new AWSSecurityTokenServiceClient(); - } - - AssumeRoleRequest assumeRequest = new AssumeRoleRequest().withRoleArn(arn).withDurationSeconds(3600) - .withExternalId(clientSecret).withRoleSessionName("vgl"); - - AssumeRoleResult assumeResult = stsClient.assumeRole(assumeRequest); - - // Step 2. AssumeRole returns temporary security credentials for - // the IAM role. - - return new BasicSessionCredentials(assumeResult.getCredentials().getAccessKeyId(), - assumeResult.getCredentials().getSecretAccessKey(), - assumeResult.getCredentials().getSessionToken()); - } else if (stsRequirement == STSRequirement.Mandatory) { - throw new PortalServiceException("AWS cross account authorization required, but not configured"); - } else if (!TextUtil.isAnyNullOrEmpty(devAccessKey, devSecretKey)) { - if(!TextUtil.isAnyNullOrEmpty(devSessionKey)) { - return new BasicSessionCredentials(devAccessKey, devSecretKey, devSessionKey); - } else { - return new BasicAWSCredentials(devAccessKey, devSecretKey); - } - } - return null; - } - - protected AmazonEC2 getEc2Client(CloudJob job) throws PortalServiceException { - String arn = job.getProperty(CloudJob.PROPERTY_STS_ARN); - String clientSecret = job.getProperty(CloudJob.PROPERTY_CLIENT_SECRET); - - AWSCredentials creds = getCredentials(arn, clientSecret); - AmazonEC2 ec2 = creds == null ? new AmazonEC2Client() : new AmazonEC2Client(creds); - - if (!TextUtil.isNullOrEmpty(getEndpoint())) - ec2.setEndpoint(getEndpoint()); - return ec2; - } - - /** - * Returns true if the Compute VM ID in job has one or more persistent volumes (volumes that - * won't delete on termination). - * - * @param job The job whose Compute VM will be checked for persistent volumes - * @throws PortalServiceException If there is an error communicating with AWS - * @return - */ - public boolean containsPersistentVolumes(CloudJob job) throws PortalServiceException { - String vmId = job.getComputeVmId(); - if (vmId.contains("/")) { - vmId = vmId.substring(vmId.lastIndexOf("/") + 1); - } - - try { - DescribeImagesRequest dir = new DescribeImagesRequest().withImageIds(vmId); - DescribeImagesResult imageDescs = getEc2Client(job).describeImages(dir); - if(imageDescs.getImages().isEmpty()) { - throw new PortalServiceException("Could not get description for image: " + vmId); - } - - Image imageDesc = imageDescs.getImages().get(0); - for (BlockDeviceMapping bdm : imageDesc.getBlockDeviceMappings()) { - Boolean deleteOnTermination = bdm.getEbs().getDeleteOnTermination(); - if( deleteOnTermination != null && deleteOnTermination == false) { - return true; - } - } - return false; - } catch (AmazonClientException ex) { - logger.error("Unable to describe images: " + ex.getMessage()); - logger.debug("Error:", ex); - throw new PortalServiceException("Unable to describe images: " + ex.getMessage(), ex); - } - } - - @Override - public String executeJob(CloudJob job, String userDataString) throws PortalServiceException { - String vmId = job.getComputeVmId(); - if (vmId.contains("/")) { - vmId = vmId.substring(vmId.lastIndexOf("/") + 1); - } - try { - userDataString = com.amazonaws.util.Base64.encodeAsString(userDataString.getBytes("Utf-8")); - } catch (UnsupportedEncodingException e) { - // can't happen - } - - RunInstancesRequest runInstancesRequest = new RunInstancesRequest() - .withInstanceType(job.getComputeInstanceType()).withImageId(vmId).withMinCount(1).withMaxCount(1) - .withInstanceInitiatedShutdownBehavior("terminate").withUserData(userDataString); - - String instanceProfileArn = job.getProperty(CloudJob.PROPERTY_S3_ROLE); - if ( (stsRequirement != STSRequirement.ForceNone) && - (!TextUtil.isNullOrEmpty(instanceProfileArn))) { - IamInstanceProfileSpecification iamInstanceProfile = new IamInstanceProfileSpecification() - .withArn(instanceProfileArn); - runInstancesRequest = runInstancesRequest.withIamInstanceProfile(iamInstanceProfile); - } - - // Check for a keypair in the CloudJob first, then fall back onto the - // default one for this instance. - String keypair = null; - if (!TextUtil.isNullOrEmpty(job.getComputeInstanceKey())) { - keypair = job.getComputeInstanceKey(); - } - if (!TextUtil.isNullOrEmpty(getKeypair())) { - keypair = getKeypair(); - } - if (keypair != null) { - runInstancesRequest = runInstancesRequest.withKeyName(keypair); - } - - AmazonEC2 ec2 = getEc2Client(job); - - if (!TextUtil.isNullOrEmpty(getEndpoint())) { - ec2.setEndpoint(getEndpoint()); - // runInstancesRequest=runInstancesRequest.withPlacement(new - // Placement(getZone())); - } - - // Wrap any AWS exception into a PortalServiceException - RunInstancesResult runInstances; - try { - runInstances = ec2.runInstances(runInstancesRequest); - } - catch (AmazonServiceException ex) { - throw new PortalServiceException("AWS RunInstances request failed", ex); - } - - // TAG EC2 INSTANCES - List instances = runInstances.getReservation().getInstances(); - if (instances.isEmpty()) - throw new PortalServiceException("AWS Vm start failed without error message"); - - Instance instance = instances.get(0); - CreateTagsRequest createTagsRequest = new CreateTagsRequest(); - createTagsRequest.withResources(instance.getInstanceId()) // - .withTags(new Tag("Name", "VGL - Job: " + job.getId())); - ec2.createTags(createTagsRequest); - - return instance.getInstanceId(); - } - - /** - * Makes a request that the VM started by job be terminated - * - * @param job - * The job whose execution should be terminated - * @throws PortalServiceException - */ - @Override - public void terminateJob(CloudJob job) throws PortalServiceException { - AmazonEC2 ec2 = getEc2Client(job); - - TerminateInstancesRequest terminateInstancesRequest = new TerminateInstancesRequest() - .withInstanceIds(job.getComputeInstanceId()); - ec2.terminateInstances(terminateInstancesRequest); - } - - final static ComputeType[] COMPUTE_TYPES = { - new ComputeType("x1.32xlarge", 128, 1920000), - new ComputeType("c4.8xlarge", 36, 60000), - new ComputeType("c4.4xlarge", 16, 30000), - new ComputeType("c4.2xlarge", 8, 15000), - new ComputeType("c4.xlarge", 4, 7500), - new ComputeType("c4.large", 2, 3750), - new ComputeType("m4.10xlarge", 40, 160000), - new ComputeType("m4.4xlarge", 16, 64000), - new ComputeType("m4.2xlarge", 8, 32000), - new ComputeType("m4.xlarge", 4, 16000), - new ComputeType("m4.large", 2, 8000) - }; - - /** - * An array of compute types that are available through this compute service - */ - @Override - public ComputeType[] getAvailableComputeTypes(Integer minimumVCPUs, Integer minimumRamMB, - Integer minimumRootDiskGB) { - - ArrayList result = new ArrayList<>(); - - for (ComputeType type : COMPUTE_TYPES) { - if( (minimumVCPUs == null || type.getVcpus()>= minimumVCPUs) && (minimumRamMB == null || type.getRamMB()>= minimumRamMB)) { - result.add(type); - } - } - return result.toArray(new ComputeType[result.size()]); - } - - /** - * Will attempt to tail and return the last {@code numLines} from the given - * servers console. - * - * @param job - * the job which has been executed by this service - * @param numLines - * the number of console lines to return - * @return console output as string or null - * @return - */ - @Override - public String getConsoleLog(CloudJob job, int numLines) throws PortalServiceException { - GetConsoleOutputRequest req = new GetConsoleOutputRequest(job.getComputeInstanceId()); - - GetConsoleOutputResult res = getEc2Client(job).getConsoleOutput(req); - - try { - return res.getDecodedOutput(); - } catch (NullPointerException e) { - return null; - } - } - - /** - * Attempts to lookup low level status information about this job's compute instance from the remote cloud. - * - * Having no computeInstanceId set will result in an exception being thrown. - * - * @param job - * @return - * @throws PortalServiceException - */ - @Override - public InstanceStatus getJobStatus(CloudJob job) throws PortalServiceException { - - //If the job has just been submitted - don't go checking with AWS, we'll probably get a missing VM message - //Let the VM have a chance to propogate through AWS - //See also ANVGL-112 - Date submitDate = job.getSubmitDate(); - if (submitDate != null) { - Date now = new Date(); - if (TimeUnit.MILLISECONDS.toSeconds(now.getTime() - submitDate.getTime()) < STATUS_PENDING_SECONDS) { - return InstanceStatus.Pending; - } - } - - if (StringUtils.isEmpty(job.getComputeInstanceId())) { - logger.debug("Unexpected missing job ID in getJobStatus(). Will return 'pending'. Local status: "+job.getStatus()); - return InstanceStatus.Pending; - } - - DescribeInstanceStatusRequest request = new DescribeInstanceStatusRequest(); - request.setInstanceIds(Arrays.asList(job.getComputeInstanceId())); - - try { - AmazonEC2 ec2 = getEc2Client(job); - - DescribeInstanceStatusResult result = ec2.describeInstanceStatus(request); - List statuses = result.getInstanceStatuses(); - if (statuses.isEmpty()) { - return InstanceStatus.Missing; - } - String status = statuses.get(0).getInstanceState().getName(); - switch(status) { - case "pending": - return InstanceStatus.Pending; - case "running": - return InstanceStatus.Running; - default: - return InstanceStatus.Missing; - } - } catch (AmazonServiceException ex) { - //Some of the "expected" AWS responses come from parsing exceptions - switch (ex.getErrorCode()) { - case "InvalidInstanceID.NotFound": - return InstanceStatus.Missing; - default: - // ignore all other cases - break; - } - - switch (ex.getStatusCode()) { - case HttpStatus.SC_FORBIDDEN: - return InstanceStatus.Missing; - default: - throw new PortalServiceException("Unable to lookup status code for :" + job.getComputeInstanceId(), ex); - } - } catch (Exception ex) { - throw new PortalServiceException("Unable to lookup status code for :" + job.getComputeInstanceId(), ex); - } - } - - @Override - public ComputeType[] getAvailableComputeTypes(String machineImageId) throws PortalServiceException { - // Carsten: As far as I know AWS images have no specific limitation on what ComputeType can run them. To be implemented properly if this - // turns out to be wrong - return getAvailableComputeTypes(); - } -} diff --git a/src/main/java/org/auscope/portal/core/services/cloud/CloudComputeServiceNectar.java b/src/main/java/org/auscope/portal/core/services/cloud/CloudComputeServiceNectar.java deleted file mode 100644 index 6d2739211..000000000 --- a/src/main/java/org/auscope/portal/core/services/cloud/CloudComputeServiceNectar.java +++ /dev/null @@ -1,431 +0,0 @@ -package org.auscope.portal.core.services.cloud; - -import static com.google.common.base.Predicates.not; -import static org.jclouds.compute.predicates.NodePredicates.RUNNING; -import static org.jclouds.compute.predicates.NodePredicates.TERMINATED; -import static org.jclouds.compute.predicates.NodePredicates.inGroup; - -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Properties; -import java.util.Set; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.auscope.portal.core.cloud.CloudJob; -import org.auscope.portal.core.cloud.ComputeType; -import org.auscope.portal.core.services.PortalServiceException; -import org.jclouds.ContextBuilder; -import org.jclouds.compute.ComputeService; -import org.jclouds.compute.ComputeServiceContext; -import org.jclouds.compute.RunNodesException; -import org.jclouds.compute.domain.Hardware; -import org.jclouds.compute.domain.NodeMetadata; -import org.jclouds.compute.domain.NodeMetadata.Status; -import org.jclouds.compute.domain.Processor; -import org.jclouds.compute.domain.Template; -import org.jclouds.compute.domain.Volume; -import org.jclouds.compute.options.TemplateOptions; -import org.jclouds.openstack.keystone.config.KeystoneProperties; -import org.jclouds.openstack.nova.v2_0.NovaApi; -import org.jclouds.openstack.nova.v2_0.compute.options.NovaTemplateOptions; -import org.jclouds.openstack.nova.v2_0.domain.Image; -import org.jclouds.openstack.nova.v2_0.features.ImageApi; -import org.openstack4j.api.OSClient; -import org.openstack4j.model.common.Identifier; -import org.openstack4j.openstack.OSFactory; - -import com.google.common.base.Predicate; -import com.google.common.base.Predicates; - - -/** - * Service class wrapper for interacting with a remote cloud compute service using CloudJob objects. - * - * @author Josh Vote - */ -public class CloudComputeServiceNectar extends CloudComputeService { - private final Log logger = LogFactory.getLog(getClass()); - - private ComputeService computeService; - private ComputeServiceContext context; - private NovaApi novaApi; //will be null for non nova API's - - private Set skippedZones = new HashSet<>(); - - private Predicate terminateFilter; - - /** Name of accessKey for authentication */ - private String accessKey; - /** Name of secretKey for authentication */ - private String secretKey; - /** Cloud endpoint to connect to */ - private String endpoint; - - private ContextBuilder builder; - - private String zone; //can be null - - private String adminEmail = "cg-admin@csiro.au"; - - private ImageApi imageApi; - - /** - * @return the adminEmail - */ - public String getAdminEmail() { - return adminEmail; - } - - /** - * @param adminEmail the adminEmail to set - */ - public void setAdminEmail(String adminEmail) { - this.adminEmail = adminEmail; - } - - /** - * Gets the region (if any) where this compute service will be restricted to run in - * - * Will be ignored if skipped zones is specified - * @return - */ - public String getZone() { - return zone; - } - - /** - * Sets the region (if any) where this compute service will be restricted to run in - * - * Will be ignored if skipped zones is specified - * - * @param zone - */ - public void setZone(String zone) { - this.zone = zone; - } - - /** - * Creates a new instance with the specified credentials (no endpoint specified - ensure provider type has a fixed endpoint) - * - * @param accessKey - * The Compute Access key (user name) - * @param secretKey - * The Compute Secret key (password) - * - */ - public CloudComputeServiceNectar(String accessKey, String secretKey) { - this(null, accessKey, secretKey, null); - } - - /** - * Creates a new instance with the specified credentials - * - * @param endpoint - * (URL) The location of the Compute (Nova) service - * @param accessKey - * The Compute Access key (user name) - * @param secretKey - * The Compute Secret key (password) - * - */ - public CloudComputeServiceNectar(String endpoint, String accessKey, String secretKey) { - this(endpoint, accessKey, secretKey, null); - } - - /** - * Creates a new instance with the specified credentials - * - * @param endpoint - * (URL) The location of the Compute (Nova) service - * @param accessKey - * The Compute Access key (user name) - * @param secretKey - * The Compute Secret key (password) - * @param apiVersion - * The API version - */ - @SuppressWarnings("unchecked") - public CloudComputeServiceNectar(String endpoint, String accessKey, String secretKey, String apiVersion) { - super(ProviderType.NovaKeystone, endpoint, apiVersion); - this.endpoint = endpoint; - this.accessKey = accessKey; - this.secretKey = secretKey; - - String[] accessParts = this.accessKey.split(":"); - String projectName = accessParts[0]; - String userName = accessParts[1]; - String typeString = "openstack-nova"; - - Properties overrides = new Properties(); - if(endpoint.contains("keystone") && endpoint.contains("v3")) { - overrides.put(KeystoneProperties.KEYSTONE_VERSION, "3"); - overrides.put(KeystoneProperties.SCOPE, "project:" + projectName); - } - this.builder = ContextBuilder.newBuilder(typeString) - .overrides(overrides); - - if(accessKey!=null && secretKey!=null) { - if(endpoint.contains("keystone") && endpoint.contains("v3")) { - builder.credentials("default:"+userName, secretKey); - } else { - builder.credentials(userName, secretKey); - } - } - - if (getApiVersion() != null) { - builder.apiVersion(getApiVersion()); - } - - if (endpoint != null) { - builder.endpoint(endpoint); - } - - this.novaApi = builder.buildApi(NovaApi.class); - - Set regions = novaApi.getConfiguredRegions(); - String region = null; - if(!regions.isEmpty()) - region = regions.iterator().next(); - - this.imageApi = novaApi.getImageApi(region); - this.context = builder.buildView(ComputeServiceContext.class); - this.computeService = this.context.getComputeService(); - this.terminateFilter = Predicates.and(not(TERMINATED), not(RUNNING), inGroup(getGroupName())); - } - - public CloudComputeServiceNectar(ComputeService computeService, NovaApi novaApi, - Predicate terminPredicate) { - super(ProviderType.NovaKeystone); - this.computeService = computeService; - this.novaApi = novaApi; - this.terminateFilter = terminPredicate; - } - - /** - * Begins execution of the specified job and returns the ID of the started instance. - * - * This function will create a VM to run the job which will be responsible for decoding the userDataString and downloading any input files from the - * JobStorageService - * - * @param job - * The job to execute - * @param userDataString - * A string that is made available to the job when it starts execution (this will be Base64 encoded before being sent to the VM) - * @return null if execution fails or the instance ID of the running VM - */ - @Override - public String executeJob(CloudJob job, String userDataString) throws PortalServiceException { - - // We have different template options depending on provider - NodeMetadata result; - Set results = Collections.emptySet(); - TemplateOptions options = ((NovaTemplateOptions) computeService.templateOptions()) - .keyPairName(getKeypair()) - .userData(userDataString.getBytes(Charset.forName("UTF-8"))); - - Template template = computeService.templateBuilder().imageId(job.getComputeVmId()) - .hardwareId(job.getComputeInstanceType()).options(options).build(); - - try { - results = computeService.createNodesInGroup(getGroupName(), 1, template); - } catch (RunNodesException e) { - logger.error(String.format("Launch failure from service '%4$s' for job %3$s image '%1$s' type '%2$s'", job.getComputeVmId(), job.getComputeInstanceType(), job.getId(), this.getId())); - logger.debug(e.getMessage()); - try { - // FIXME: - // I think this could possibly delete EVERY NODE RUN - // from PORTAL-CORE... - // JClouds is not very clever here - - // issue: how do you delete thing you didnt name and - // dont have an ID for?? - Set destroyedNodes = computeService.destroyNodesMatching(this.terminateFilter); - logger.warn(String.format("cleaned up %1$s nodes: %2$s", destroyedNodes.size(), destroyedNodes)); - } catch (Exception z) { - logger.error("couldnt clean it up"); - } - } - - if (results.isEmpty()) { - throw new PortalServiceException( - "An unexpected error has occured while executing your job. Most likely this is from the lack of available resources. Please try using" - + "a smaller virtual machine", - "Please report it to " +getAdminEmail()+"."); - } else { - result = results.iterator().next(); - } - - logger.trace(String.format("Successful launch from service '%4$s' for job %3$s image '%1$s' type '%2$s'", job.getComputeVmId(), job.getComputeInstanceType(), job.getId(), this.getId())); - - return result.getId(); - } - - /** - * Makes a request that the VM started by job be terminated - * - * @param job - * The job whose execution should be terminated - */ - @Override - public void terminateJob(CloudJob job) { - computeService.destroyNode(job.getComputeInstanceId()); - } - - @Override - public ComputeType[] getAvailableComputeTypes(String machineImageId) throws PortalServiceException { - String imageId = machineImageId.split("/")[1]; - - Image image = imageApi.get(imageId); - - return getAvailableComputeTypes(null, image.getMinRam(), image.getMinDisk()); - } - - /** - * An array of compute types that are available through this compute service - */ - @Override - public ComputeType[] getAvailableComputeTypes(Integer minimumVCPUs, Integer minimumRamMB, Integer minimumRootDiskGB) { - Set hardwareSet = computeService.listHardwareProfiles(); - - List computeTypes = new ArrayList<>(); - - for (Hardware hw : hardwareSet) { - ComputeType ct = new ComputeType(hw.getId()); - - ct.setDescription(hw.getName()); - double vCpus = 0; - for (Processor p : hw.getProcessors()) { - vCpus += p.getCores(); - } - ct.setVcpus((int) vCpus); - ct.setRamMB(hw.getRam()); - - double rootDiskGB = 0; - double ephemeralDiskGB = 0; - for (Volume v : hw.getVolumes()) { - if (v.isBootDevice()) { - rootDiskGB += v.getSize(); - } else { - ephemeralDiskGB += v.getSize(); - } - } - ct.setRootDiskGB((int) rootDiskGB); - ct.setEphemeralDiskGB((int) ephemeralDiskGB); - - //Skip anything that doesn't match our filters - if (minimumVCPUs != null && minimumVCPUs > ct.getVcpus()) { - continue; - } else if (minimumRamMB != null && minimumRamMB > ct.getRamMB()) { - continue; - } else if (minimumRootDiskGB != null && minimumRootDiskGB > ct.getRootDiskGB()) { - continue; - } - - computeTypes.add(ct); - } - - return computeTypes.toArray(new ComputeType[computeTypes.size()]); - } - - /** - * Gets the set of zone names that should be skipped when attempting to find a zone to run a job at. - * - * @return - */ - public Set getSkippedZones() { - return skippedZones; - } - - /** - * Sets the set of zone names that should be skipped when attempting to find a zone to run a job at. - * - * @param skippedZones - */ - public void setSkippedZones(Set skippedZones) { - this.skippedZones = skippedZones; - } - - /** - * Will attempt to tail and return the last {@code numLines} from the given servers console. - * - * @param job - * the job which has been executed by this service - * @param numLines - * the number of console lines to return - * @return console output as string or null - * @return - */ - @Override - public String getConsoleLog(CloudJob job, int numLines) throws PortalServiceException { - String computeInstanceId = job.getComputeInstanceId(); - if (computeInstanceId == null) { - return null; - } - - try { - String[] accessParts = this.accessKey.split(":"); - String[] idParts = computeInstanceId.split("/"); - - //JClouds has no support (currently) for tailing server console output. Our current workaround - //is to offload this to openstack4j. - @SuppressWarnings("rawtypes") - OSClient os = null; - if(endpoint.contains("keystone") && endpoint.contains("v3")) { - os = OSFactory.builderV3() - .endpoint(endpoint) - .credentials(accessParts[1], secretKey, Identifier.byName("Default")) - .scopeToProject(Identifier.byName(accessParts[0]), Identifier.byName("Default")) - .authenticate(); - } else { - os = OSFactory.builderV2() - .endpoint(endpoint) - .credentials(accessParts[1], secretKey) - .tenantName(accessParts[0]) - .authenticate(); - } - return os.compute().servers().getConsoleOutput(idParts[1], numLines); - } catch (Exception ex) { - logger.error("Unable to retrieve console logs for " + computeInstanceId, ex); - throw new PortalServiceException("Unable to retrieve console logs for " + computeInstanceId, ex); - } - } - - /** - * Attempts to lookup low level status information about this job's compute instance from the remote cloud. - * - * Having no computeInstanceId set will result in an exception being thrown. - * - * @param job - * @return - * @throws PortalServiceException - */ - @Override - public InstanceStatus getJobStatus(CloudJob job) throws PortalServiceException { - if (StringUtils.isEmpty(job.getComputeInstanceId())) { - throw new PortalServiceException("No compute instance ID has been set"); - } - - try { - NodeMetadata md = computeService.getNodeMetadata(job.getComputeInstanceId()); - if (md == null) { - return InstanceStatus.Missing; - } - - Status status = md.getStatus(); - switch (status) { - case PENDING: - return InstanceStatus.Pending; - case RUNNING: - return InstanceStatus.Running; - default: - return InstanceStatus.Missing; - } - } catch (Exception ex) { - throw new PortalServiceException("Error fetching node metadata", ex); - } - } -} diff --git a/src/main/java/org/auscope/portal/core/services/cloud/CloudStorageService.java b/src/main/java/org/auscope/portal/core/services/cloud/CloudStorageService.java deleted file mode 100644 index 16e563853..000000000 --- a/src/main/java/org/auscope/portal/core/services/cloud/CloudStorageService.java +++ /dev/null @@ -1,368 +0,0 @@ -/** - * - */ -package org.auscope.portal.core.services.cloud; - -import java.io.File; -import java.io.InputStream; -import java.net.InetAddress; -import java.net.UnknownHostException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.auscope.portal.core.cloud.CloudDirectoryInformation; -import org.auscope.portal.core.cloud.CloudFileInformation; -import org.auscope.portal.core.cloud.CloudFileOwner; -import org.auscope.portal.core.services.PortalServiceException; -import org.auscope.portal.core.util.TextUtil; - -/** - * @author fri096 - * - */ -public abstract class CloudStorageService { - private final Log log = LogFactory.getLog(getClass()); - - abstract public InputStream getJobFile(CloudFileOwner job, String logFile) throws PortalServiceException; - - abstract public CloudFileInformation[] listJobFiles(CloudFileOwner job) throws PortalServiceException; - - abstract public CloudDirectoryInformation listJobDirectoriesAndFiles(CloudFileOwner job, CloudDirectoryInformation cloudDirectory) throws PortalServiceException; - - abstract public void deleteJobFiles(CloudFileOwner job) throws PortalServiceException; - - abstract public void uploadJobFiles(CloudFileOwner curJob, File[] files) throws PortalServiceException; - - abstract public void uploadJobFile(CloudFileOwner curJob, String fileName, InputStream data) throws PortalServiceException; - - abstract public CloudFileInformation getJobFileMetadata(CloudFileOwner job, String fileName) throws PortalServiceException; - - /** - * The region identifier string for this service (if any). Can be - * null/empty. Currently this field is NON functional, it is only for - * descriptive purposes due to limitations in JClouds. - */ - private String regionName; - /** - * @return the regionName - */ - - public String getRegionName() { - return regionName; - } - - private String defaultBucket = DEFAULT_BUCKET; - - private String adminEmail = "cg-admin@csiro.au"; - - /** - * @param regionName the regionName to set - */ - public void setRegionName(String regionName) { - this.regionName = regionName; - } - - /** - * @return the accessKey - */ - public String getAccessKey() { - return accessKey; - } - - /** - * @param accessKey the accessKey to set - */ - public void setAccessKey(String accessKey) { - this.accessKey = accessKey; - } - - /** - * @return the secretKey - */ - public String getSecretKey() { - return secretKey; - } - - /** - * @param secretKey the secretKey to set - */ - public void setSecretKey(String secretKey) { - this.secretKey = secretKey; - } - - /** - * @param provider the provider to set - */ - public void setProvider(String provider) { - this.provider = provider; - } - - /** - * @param endpoint the endpoint to set - */ - public void setEndpoint(String endpoint) { - this.endpoint = endpoint; - } - - /** Username credential for accessing the storage service */ - private String accessKey; - /** Password credentials for accessing the storage service */ - private String secretKey; - - /** Session token for accessing the storage service */ - private String sessionKey; - - public String getSessionKey() { - return sessionKey; - } - - public void setSessionKey(String sessionKey) { - this.sessionKey = sessionKey; - } - - /** The bucket name used when no bucket is specified */ - public static final String DEFAULT_BUCKET = "vgl"; - - /** - * Prefix to apply to any job files stored (will be appended with job id) - - * defaults to hostname - */ - protected String jobPrefix; - - /** - * The unique ID for this service - use it for distinguishing this service - * from other instances of this class - can be null or empty - */ - private String id; - /** A short descriptive name for human identification of this service */ - private String name; - /** - * The authentication version to use when connecting to this object store - - * can be null or empty - */ - private String authVersion; - - /** - * A unique identifier identifying the type of storage API used to store - * this job's files - eg 'swift' - */ - private String provider; - /** The URL endpoint for the cloud storage service */ - private String endpoint; - - public CloudStorageService(String endpoint, String provider, String regionName) { - this.endpoint = endpoint; - this.provider = provider; - this.regionName= regionName; - - try { - this.jobPrefix = "job-" + InetAddress.getLocalHost().getHostName() + "-"; - } catch (UnknownHostException e) { - this.jobPrefix = "job-"; - log.error("Unable to lookup hostname. Defaulting prefix to " + this.jobPrefix, e); - } - } - - - - /** - * @return the adminEmail - */ - public String getAdminEmail() { - return adminEmail; - } - - /** - * @param adminEmail - * the adminEmail to set - */ - public void setAdminEmail(String adminEmail) { - this.adminEmail = adminEmail; - } - - /** - * A unique identifier identifying the type of storage API used to store - * this job's files - eg 'swift' - * - * @return - */ - public String getProvider() { - return provider; - } - - /** - * The URL endpoint for the cloud storage service - * - * @return - */ - public String getEndpoint() { - return endpoint; - } - - /** - * Prefix to apply to any job files stored (will be appended with job id) - * - * @return - */ - public String getJobPrefix() { - return jobPrefix; - } - - /** - * Prefix to apply to any job files stored (will be appended with job id) - * - * @param jobPrefix - */ - public void setJobPrefix(String jobPrefix) { - this.jobPrefix = jobPrefix; - } - - /** - * The unique ID for this service - use it for distinguishing this service - * from other instances of this class - can be null or empty - * - * @return - */ - public String getId() { - return id; - } - - /** - * The unique ID for this service - use it for distinguishing this service - * from other instances of this class - can be null or empty - * - * @param id - */ - public void setId(String id) { - this.id = id; - } - - /** - * Utility for accessing the correct bucket based on owner's configuration - * - * @param owner - * @return - */ - String getBucket(CloudFileOwner owner) { - String ownerBucket = owner.getStorageBucket(); - if (TextUtil.isNullOrEmpty(ownerBucket)) { - return defaultBucket; - } - - return ownerBucket; - } - - /** - * The default bucket where the data will be stored (if the CloudFileOwners - * don't define a storage bucket) - * - * @return - */ - public String getBucket() { - return defaultBucket; - } - - /** - * The default bucket where the data will be stored (if the CloudFileOwners - * don't define a storage bucket) - * - * @param bucket - */ - public void setBucket(String bucket) { - this.defaultBucket = bucket; - } - - /** - * The authentication version to use when connecting to this object store - - * can be null or empty - * - * @return - */ - public String getAuthVersion() { - return authVersion; - } - - /** - * The authentication version to use when connecting to this object store - - * can be null or empty - * - * @param authVersion - */ - public void setAuthVersion(String authVersion) { - this.authVersion = authVersion; - } - - /** - * A short descriptive name for human identification of this service - * - * @return - */ - public String getName() { - return name; - } - - /** - * A short descriptive name for human identification of this service - * - * @param name - */ - public void setName(String name) { - this.name = name; - } - - /** - * Utility for allowing only whitelisted characters - * - * @param s - * @return - */ - private static String sanitise(String s, boolean allowDot) { - if (allowDot) { - return s.replaceAll("[^a-zA-Z0-9_\\-\\.]", "_"); - } else { - return s.replaceAll("[^a-zA-Z0-9_\\-]", "_"); - } - - } - - /** - * Utility for calculating an appropriate base cloud key for storing this - * jobs files - * - * @param job - * @return - */ - public String generateBaseKey(CloudFileOwner job) { - String baseKey = String.format("%1$s%2$s-%3$010d", jobPrefix, job.getUser(), job.getId()); - return sanitise(baseKey, false); - } - - /** - * Utility for generating the full path for a specific job file - * - * @param job - * The job whose storage space will be queried for - * @param key - * The key of the file (local to job). - * @return - */ - public String keyForJobFile(CloudFileOwner job, String key) { - return String.format("%1$s/%2$s", jobToBaseKey(job), sanitise(key, true)); - } - - /** - * Gets the preconfigured base key for a job. If the job doesn't have a base - * key, one will be generated. - * - * @param job - * Will have its baseKey parameter set if it's null - * @return - */ - protected String jobToBaseKey(CloudFileOwner job) { - if (job.getStorageBaseKey() == null) { - job.setStorageBaseKey(generateBaseKey(job)); - } - - return job.getStorageBaseKey(); - } -} diff --git a/src/main/java/org/auscope/portal/core/services/cloud/CloudStorageServiceJClouds.java b/src/main/java/org/auscope/portal/core/services/cloud/CloudStorageServiceJClouds.java deleted file mode 100644 index f13baf8e9..000000000 --- a/src/main/java/org/auscope/portal/core/services/cloud/CloudStorageServiceJClouds.java +++ /dev/null @@ -1,611 +0,0 @@ -package org.auscope.portal.core.services.cloud; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.auscope.portal.core.cloud.CloudDirectoryInformation; -import org.auscope.portal.core.cloud.CloudFileInformation; -import org.auscope.portal.core.cloud.CloudFileOwner; -import org.auscope.portal.core.cloud.CloudJob; -import org.auscope.portal.core.services.PortalServiceException; -import org.auscope.portal.core.util.TextUtil; -import org.jclouds.ContextBuilder; -import org.jclouds.aws.domain.SessionCredentials; -import org.jclouds.blobstore.BlobStore; -import org.jclouds.blobstore.BlobStoreContext; -import org.jclouds.blobstore.KeyNotFoundException; -import org.jclouds.blobstore.domain.Blob; -import org.jclouds.blobstore.domain.PageSet; -import org.jclouds.blobstore.domain.StorageMetadata; -import org.jclouds.blobstore.domain.StorageType; -import org.jclouds.blobstore.domain.internal.BlobMetadataImpl; -import org.jclouds.blobstore.domain.internal.MutableBlobMetadataImpl; -import org.jclouds.blobstore.options.ListContainerOptions; -import org.jclouds.domain.Credentials; -import org.jclouds.io.ContentMetadata; -import org.jclouds.openstack.keystone.config.KeystoneProperties; -import org.jclouds.openstack.swift.v1.blobstore.RegionScopedBlobStoreContext; -import org.jclouds.rest.AuthorizationException; -import org.jclouds.sts.STSApi; -import org.jclouds.sts.domain.UserAndSessionCredentials; -import org.jclouds.sts.options.AssumeRoleOptions; - -import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; -import com.google.common.io.Files; - - -/** - * Service for providing storage of objects (blobs) in a cloud using the JClouds library - * - * @author Josh Vote - * - */ -public class CloudStorageServiceJClouds extends CloudStorageService { - - private final Log log = LogFactory.getLog(getClass()); - - private boolean relaxHostName; - - private boolean stripExpectHeader; - - private STSRequirement stsRequirement=STSRequirement.Permissable; - - - - /** - * Returns whether AWS cross account authorization is mandatory, optional or forced off - * @return - */ - public STSRequirement getStsRequirement() { - return stsRequirement; - } - - /** - * Sets whether AWS cross account authorization is mandatory, optional or forced off - * @param stsRequirement - */ - public void setStsRequirement(STSRequirement stsRequirement) { - this.stsRequirement = stsRequirement; - } - - /** - * Creates a new instance for connecting to the specified parameters - * - * @param endpoint - * The URL endpoint for the cloud storage service - * @param provider - * A unique identifier identifying the type of storage API used to store this job's files - eg 'swift' - * @param accessKey - * Username credential for accessing the storage service - * @param secretKey - * Password credentials for accessing the storage service - */ - public CloudStorageServiceJClouds(String provider, String accessKey, String secretKey, String sessionKey) { - this(null, provider, accessKey, secretKey, sessionKey, null, false); - } - - /** - * Creates a new instance for connecting to the specified parameters - * - * @param endpoint - * The URL endpoint for the cloud storage service - * @param provider - * A unique identifier identifying the type of storage API used to store this job's files - eg 'swift' - * @param accessKey - * Username credential for accessing the storage service - * @param secretKey - * Password credentials for accessing the storage service - */ - public CloudStorageServiceJClouds(String endpoint, String provider, String accessKey, String secretKey, String sessionKey) { - this(endpoint, provider, accessKey, secretKey, sessionKey, null, false); - } - - /** - * Creates a new instance for connecting to the specified parameters - * - * @param endpoint - * The URL endpoint for the cloud storage service - * @param provider - * A unique identifier identifying the type of storage API used to store this job's files - eg 'swift' - * @param accessKey - * Username credential for accessing the storage service - * @param secretKey - * Password credentials for accessing the storage service - * @param relaxHostName - * Whether security certs are required to strictly match the host - */ - public CloudStorageServiceJClouds(String endpoint, String provider, String accessKey, String secretKey, String sessionKey, - boolean relaxHostName) { - this(endpoint, provider, accessKey, secretKey, sessionKey, null, relaxHostName); - } - - /** - * Creates a new instance for connecting to the specified parameters - * - * @param endpoint - * The URL endpoint for the cloud storage service - * @param provider - * A unique identifier identifying the type of storage API used to store this job's files - eg 'swift' - * @param accessKey - * Username credential for accessing the storage service - * @param secretKey - * Password credentials for accessing the storage service - * @param regionName - * The region identifier string for this service (if any). Can be null/empty. - */ - public CloudStorageServiceJClouds(String endpoint, String provider, String accessKey, String secretKey, String sessionKey, String regionName) { - this(endpoint, provider, accessKey, secretKey, sessionKey, regionName, false); - } - - /** - * Creates a new instance for connecting to the specified parameters - * - * @param endpoint - * The URL endpoint for the cloud storage service - * @param provider - * A unique identifier identifying the type of storage API used to store this job's files - eg 'swift' - * @param accessKey - * Username credential for accessing the storage service - * @param secretKey - * Password credentials for accessing the storage service - * @param regionName - * The region identifier string for this service (if any). Can be null/empty. - * @param relaxHostName - * Whether security certs are required to strictly match the host - */ - public CloudStorageServiceJClouds(String endpoint, String provider, String accessKey, String secretKey, String sessionKey, String regionName, - boolean relaxHostName) { - this(endpoint, provider, accessKey, secretKey, sessionKey, regionName, relaxHostName, false); - } - - /** - * Creates a new instance for connecting to the specified parameters - * - * @param endpoint - * The URL endpoint for the cloud storage service - * @param provider - * A unique identifier identifying the type of storage API used to store this job's files - eg 'swift' - * @param accessKey - * Username credential for accessing the storage service - * @param secretKey - * Password credentials for accessing the storage service - * @param regionName - * The region identifier string for this service (if any). Can be null/empty. - * @param relaxHostName - * Whether security certs are required to strictly match the host - * @param stripExpectHeader - * Whether to remove HTTP Expect header from requests; set to true for blobstores that do not support 100-Continue - */ - public CloudStorageServiceJClouds(String endpoint, String provider, String accessKey, String secretKey, String sessionKey, String regionName, - boolean relaxHostName, boolean stripExpectHeader) { - super(endpoint, provider, regionName); - - this.relaxHostName=relaxHostName; - this.stripExpectHeader=stripExpectHeader; - - this.setAccessKey(accessKey); - this.setSecretKey(secretKey); - this.setSessionKey(sessionKey); - } - - /*** - * For unit testing only - */ - public CloudStorageServiceJClouds() { - super(null, null, null); - } - - private BlobStore getBlobStore(String arn, String clientSecret) throws PortalServiceException { - try (BlobStoreContext ctx = getBlobStoreContext(arn, clientSecret)) { - if (getRegionName() != null && ctx instanceof RegionScopedBlobStoreContext) { - return ((RegionScopedBlobStoreContext) ctx).getBlobStore(getRegionName()); - } else { - return ctx.getBlobStore(); - } - } - } - - public BlobStoreContext getBlobStoreContext(String arn, String clientSecret) throws PortalServiceException { - if (stsRequirement == STSRequirement.ForceNone) { - arn = null; - clientSecret = null; - } - - Properties properties = new Properties(); - properties.setProperty("jclouds.relax-hostname", relaxHostName ? "true" : "false"); - properties.setProperty("jclouds.strip-expect-header", stripExpectHeader ? "true" : "false"); - - // Keystone v3 will require a few extra properties - if(this.getEndpoint() != null && this.getEndpoint().contains("keystone") && this.getEndpoint().contains("v3")) { - String[] accessParts = this.getAccessKey().split(":"); - String projectName = accessParts[0]; - properties.put(KeystoneProperties.KEYSTONE_VERSION, "3"); - properties.put(KeystoneProperties.SCOPE, "project:" + projectName); - } - - Class targetClass = BlobStoreContext.class; - if (getRegionName() != null) { - if (getProvider().contains("openstack") || getProvider().contains("swift")) { - targetClass = RegionScopedBlobStoreContext.class; - } else { - properties.setProperty("jclouds.region", getRegionName()); - } - } - - if(! TextUtil.isNullOrEmpty(arn)) { - ContextBuilder builder = ContextBuilder.newBuilder("sts"); - - if( (! TextUtil.isNullOrEmpty(getAccessKey())) && - (! TextUtil.isNullOrEmpty(getSecretKey()))) { - if(! TextUtil.isNullOrEmpty(getSessionKey())) { - - SessionCredentials credentials = SessionCredentials.builder() - .accessKeyId(getAccessKey()) - .secretAccessKey(getSecretKey()) - .sessionToken(getSessionKey()) - .build(); - - builder.credentialsSupplier(Suppliers.ofInstance(credentials)); - } else { - builder.credentials(getAccessKey(), getSecretKey()); - } - } - - try (STSApi api = builder.buildApi(STSApi.class)) { - AssumeRoleOptions assumeRoleOptions = new AssumeRoleOptions().durationSeconds(3600) - .externalId(clientSecret); - final UserAndSessionCredentials credentials = api.assumeRole(arn, "vgl", assumeRoleOptions); - - Supplier credentialsSupplier = new Supplier() { - @Override - public Credentials get() { - return credentials.getCredentials(); - } - }; - - ContextBuilder builder2 = ContextBuilder.newBuilder("aws-s3").overrides(properties) - .credentialsSupplier(credentialsSupplier); - - if (this.getEndpoint() != null) { - builder2.endpoint(this.getEndpoint()); - } - - return builder2.buildView(targetClass); - } catch (IOException e) { - throw new PortalServiceException(e.getMessage(), e); - } - } else { - if (stsRequirement == STSRequirement.Mandatory) { - throw new PortalServiceException("AWS cross account access is required, but not configured"); - } - - ContextBuilder builder = ContextBuilder.newBuilder(getProvider()).overrides(properties); - - if( (! TextUtil.isNullOrEmpty(getAccessKey())) && - (! TextUtil.isNullOrEmpty(getSecretKey()))) { - if(! TextUtil.isNullOrEmpty(getSessionKey())) { - SessionCredentials credentials = SessionCredentials.builder() - .accessKeyId(getAccessKey()) - .secretAccessKey(getSecretKey()) - .sessionToken(getSessionKey()) - .build(); - - builder.credentialsSupplier(Suppliers.ofInstance(credentials)); - } else { - String accessKey = getAccessKey(); - String secretKey = getSecretKey(); - if(this.getEndpoint() != null && this.getEndpoint().contains("keystone") && this.getEndpoint().contains("v3")) { - properties.put(KeystoneProperties.KEYSTONE_VERSION, "3"); - String[] accessParts = this.getAccessKey().split(":"); - accessKey = "default:" + accessParts[1]; - } - builder.credentials(accessKey, secretKey); - } - } - - if (this.getEndpoint() != null) { - builder.endpoint(this.getEndpoint()); - } - - return builder.build(targetClass); - } - } - - - /** - * Utility to extract file size from a StorageMetadata interface - * - * @param smd - * @return - */ - protected Long getFileSize(StorageMetadata smd) { - if (smd instanceof BlobMetadataImpl) { - ContentMetadata cmd = ((BlobMetadataImpl) smd).getContentMetadata(); - return cmd.getContentLength(); - } else if (smd instanceof MutableBlobMetadataImpl) { - ContentMetadata cmd = ((MutableBlobMetadataImpl) smd).getContentMetadata(); - return cmd.getContentLength(); - } else { - return 1L; - } - } - - - /** - * Gets the input stream for a job file identified by key. - * - * Ensure the resulting InputStream is closed - * - * @param job - * The job whose storage space will be queried - * @param key - * The file name (no prefixes) - * @return - * @throws PortalServiceException - */ - @Override - public InputStream getJobFile(CloudFileOwner job, String myKey) throws PortalServiceException { - String arn = job.getProperty(CloudJob.PROPERTY_STS_ARN); - String clientSecret = job.getProperty(CloudJob.PROPERTY_CLIENT_SECRET); - - try { - BlobStore bs = getBlobStore(arn, clientSecret); - Blob blob = bs.getBlob(getBucket(job), keyForJobFile(job, myKey)); - return blob.getPayload().openStream(); - } catch (Exception ex) { - log.error(String.format("Unable to get job file '%1$s' for job %2$s:", myKey, job)); - log.debug("error:", ex); - throw new PortalServiceException("Error retriving output file details", ex); - } - } - - /** - * Converts JClouds StorageMetadata into simpler CloudFileInformation. - * Returns null if the conversion is not possible - * @param md - * @return - */ - private static CloudFileInformation metadataToCloudFile(StorageMetadata md) { - //Skip objects that are not files - if (md.getType() != StorageType.BLOB) { - return null; - } - - long fileSize = 1L; - if (md instanceof BlobMetadataImpl) { - ContentMetadata cmd = ((BlobMetadataImpl) md).getContentMetadata(); - fileSize = cmd.getContentLength(); - } else if (md instanceof MutableBlobMetadataImpl) { - ContentMetadata cmd = ((MutableBlobMetadataImpl) md).getContentMetadata(); - fileSize = cmd.getContentLength(); - } - - return new CloudFileInformation(md.getName(), fileSize, md.getUri().toString(), md.getETag()); - } - - /** - * Gets the metadata for a job file identified by key. - * - * @param job - * The job whose storage space will be queried - * @param key - * The file name (no prefixes) - * @return - * @throws PortalServiceException - */ - @Override - public CloudFileInformation getJobFileMetadata(CloudFileOwner job, String myKey) throws PortalServiceException { - String arn = job.getProperty(CloudJob.PROPERTY_STS_ARN); - String clientSecret = job.getProperty(CloudJob.PROPERTY_CLIENT_SECRET); - - try { - BlobStore bs = getBlobStore(arn, clientSecret); - StorageMetadata md = bs.blobMetadata(getBucket(job), keyForJobFile(job, myKey)); - return metadataToCloudFile(md); - } catch (Exception ex) { - log.error(String.format("Unable to get job file metadata '%1$s' for job %2$s:", myKey, job)); - log.debug("error:", ex); - throw new PortalServiceException("Error retriving output file details", ex); - } - - } - - /** - * Gets information about every file in the job's cloud storage space - * - * @param job - * The job whose storage space will be queried - * @return - * @throws PortalServiceException - */ - @Override - public CloudFileInformation[] listJobFiles(CloudFileOwner job) throws PortalServiceException { - String arn = job.getProperty(CloudJob.PROPERTY_STS_ARN); - String clientSecret = job.getProperty(CloudJob.PROPERTY_CLIENT_SECRET); - - try { - BlobStore bs = getBlobStore(arn, clientSecret); - String baseKey = jobToBaseKey(job); - - String bucketName = getBucket(job); - - //Paging is a little awkward - this list method may return an incomplete list requiring followup queries - PageSet currentMetadataPage = bs.list(bucketName, ListContainerOptions.Builder.inDirectory(baseKey)); - String nextMarker = null; - List jobFiles = new ArrayList<>(); - do { - if (nextMarker != null) { - currentMetadataPage = bs.list(bucketName, ListContainerOptions.Builder - .inDirectory(baseKey) - .afterMarker(nextMarker)); - } - - //Turn our StorageMetadata objects into simpler CloudFileInformation objects - for (StorageMetadata md : currentMetadataPage) { - CloudFileInformation info = metadataToCloudFile(md); - if (info != null) { - jobFiles.add(info); - } - } - - nextMarker = currentMetadataPage.getNextMarker(); - } while (nextMarker != null); - - return jobFiles.toArray(new CloudFileInformation[jobFiles.size()]); - } catch (Exception ex) { - log.error("Unable to list files for job:" + job.toString()); - log.debug("error:", ex); - throw new PortalServiceException("Error retriving output file details", ex); - } - } - - /** - * - */ - @Override - public CloudDirectoryInformation listJobDirectoriesAndFiles(CloudFileOwner job, CloudDirectoryInformation cloudDirectory) throws PortalServiceException { - String arn = job.getProperty(CloudJob.PROPERTY_STS_ARN); - String clientSecret = job.getProperty(CloudJob.PROPERTY_CLIENT_SECRET); - - try { - BlobStore bs = getBlobStore(arn, clientSecret); - String baseKey = jobToBaseKey(job); - - String bucketName = getBucket(job); - - //Paging is a little awkward - this list method may return an incomplete list requiring followup queries - PageSet currentMetadataPage = bs.list(bucketName, ListContainerOptions.Builder.inDirectory(baseKey)); - String nextMarker = null; - List jobFiles = new ArrayList<>(); - do { - if (nextMarker != null) { - currentMetadataPage = bs.list(bucketName, ListContainerOptions.Builder - .inDirectory(baseKey) - .afterMarker(nextMarker)); - } - - //Turn our StorageMetadata objects into simpler CloudFileInformation objects - for (StorageMetadata md : currentMetadataPage) { - CloudFileInformation info = metadataToCloudFile(md); - if (info != null) { - jobFiles.add(info); - } - } - - nextMarker = currentMetadataPage.getNextMarker(); - } while (nextMarker != null); - - CloudDirectoryInformation cloudDir = new CloudDirectoryInformation("", null); - cloudDir.setFiles((ArrayList)jobFiles); - return cloudDir; - } catch (Exception ex) { - log.error("Unable to list files for job:" + job.toString()); - log.debug("error:", ex); - throw new PortalServiceException("Error retriving output file details", ex); - } - } - - /** - * Uploads an array of local files into the specified job's storage space - * - * @param job - * The job whose storage space will be used - * @param files - * The local files to upload - * @throws PortalServiceException - */ - @Override - public void uploadJobFiles(CloudFileOwner job, File[] files) throws PortalServiceException { - String arn = job.getProperty(CloudJob.PROPERTY_STS_ARN); - String clientSecret = job.getProperty(CloudJob.PROPERTY_CLIENT_SECRET); - - try { - BlobStore bs = getBlobStore(arn, clientSecret); - - String bucketName = getBucket(job); - bs.createContainerInLocation(null, bucketName); - for (File file : files) { - Blob newBlob = bs.blobBuilder(keyForJobFile(job, file.getName())) - .payload(Files.asByteSource(file)) - .contentLength(file.length()) - .build(); - bs.putBlob(bucketName, newBlob); - } - } catch (AuthorizationException ex) { - log.error("Storage credentials are not valid for job: " + job, ex); - throw new PortalServiceException("Storage credentials are not valid.", - "Please provide valid storage credentials."); - } catch (KeyNotFoundException ex) { - log.error("Storage container does not exist for job: " + job, ex); - throw new PortalServiceException("Storage container does not exist.", - "Please provide a valid storage container."); - } catch (Exception ex) { - log.error("Unable to upload files for job: " + job, ex); - throw new PortalServiceException("An unexpected error has occurred while uploading file(s) to storage.", - "Please report it to " + getAdminEmail()+"."); - } - } - - @Override - public void uploadJobFile(CloudFileOwner job, String fileName, InputStream data) throws PortalServiceException { - String arn = job.getProperty(CloudJob.PROPERTY_STS_ARN); - String clientSecret = job.getProperty(CloudJob.PROPERTY_CLIENT_SECRET); - - try { - BlobStore bs = getBlobStore(arn, clientSecret); - String bucketName = getBucket(job); - bs.createContainerInLocation(null, bucketName); - - Blob newBlob = bs.blobBuilder(keyForJobFile(job, fileName)) - .payload(data) - .build(); - bs.putBlob(bucketName, newBlob); - - log.debug(fileName + " uploaded to '" + bucketName + "' container"); - - } catch (AuthorizationException ex) { - log.error("Storage credentials are not valid for job: " + job, ex); - throw new PortalServiceException("Storage credentials are not valid.", - "Please provide valid storage credentials."); - } catch (KeyNotFoundException ex) { - log.error("Storage container does not exist for job: " + job, ex); - throw new PortalServiceException("Storage container does not exist.", - "Please provide a valid storage container."); - } catch (Exception ex) { - log.error("Unable to upload files for job: " + job, ex); - throw new PortalServiceException("An unexpected error has occurred while uploading file(s) to storage.", - "Please report it to " + getAdminEmail()+"."); - } - } - - /** - * Deletes all files including the container or directory for the specified job - * - * @param job - * The whose storage space will be deleted - * @throws PortalServiceException - */ - @Override - public void deleteJobFiles(CloudFileOwner job) throws PortalServiceException { - String arn = job.getProperty(CloudJob.PROPERTY_STS_ARN); - String clientSecret = job.getProperty(CloudJob.PROPERTY_CLIENT_SECRET); - try { - BlobStore bs = getBlobStore(arn, clientSecret); - String bucket = getBucket(job); - String baseKey = jobToBaseKey(job); - if(bs.blobExists(bucket, baseKey)) - bs.deleteDirectory(bucket, baseKey); - } catch (Exception ex) { - log.error("Error in removing job files or storage key.", ex); - throw new PortalServiceException( - "An unexpected error has occurred while removing job files from S3 storage", ex); - } - } -} diff --git a/src/main/java/org/auscope/portal/core/services/cloud/FileStagingService.java b/src/main/java/org/auscope/portal/core/services/cloud/FileStagingService.java deleted file mode 100644 index 1ac6ceb2a..000000000 --- a/src/main/java/org/auscope/portal/core/services/cloud/FileStagingService.java +++ /dev/null @@ -1,458 +0,0 @@ -package org.auscope.portal.core.services.cloud; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.List; - -import jakarta.servlet.http.HttpServletResponse; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.auscope.portal.core.cloud.StagedFile; -import org.auscope.portal.core.cloud.StagedFileOwner; -import org.auscope.portal.core.cloud.StagingInformation; -import org.auscope.portal.core.services.PortalServiceException; -import org.auscope.portal.core.util.FileIOUtil; -import org.springframework.web.multipart.MultipartFile; -import org.springframework.web.multipart.MultipartHttpServletRequest; - -/** - * A service class for handling file uploads and storing them in a local staging directory - * - * @author Josh Vote - * - */ -public class FileStagingService { - private final Log logger = LogFactory.getLog(getClass()); - protected StagingInformation stagingInformation; - - public FileStagingService(StagingInformation stagingInformation) { - this.stagingInformation = stagingInformation; - } - - /** - * Utility for returning a File handle to the actual file on HDD for a given job + fileName - * - * @param job - * @param fileName - * @return - */ - private File getFile(StagedFileOwner job, String fileName) { - if (fileName.contains(File.pathSeparator) || fileName.contains(File.separator)) { - throw new IllegalArgumentException("fileName cannot include " + File.pathSeparator + " or " - + File.separator); - } - - String directory = pathConcat(stagingInformation.getStageInDirectory(), getBaseFolderForJob(job)); - return new File(pathConcat(directory, fileName)); - } - - /** - * Given 2 path components this function will concatenate them together. - * - * The result will ensure that you will not end up with 2 consecutive File.pathSeperator characters at the place of the concatenation. - * - * @param p1 - * @param p2 - * @return - */ - public static String pathConcat(String p1, String p2) { - if (p1.endsWith(File.separator) && p2.startsWith(File.separator)) { - return p1 + p2.substring(1); - } else if ((!p1.endsWith(File.separator) && p2.startsWith(File.separator)) || - (p1.endsWith(File.separator) && !p2.startsWith(File.separator))) { - return p1 + p2; - } else { - return p1 + File.separator + p2; - } - } - - /** - * Generates a base folder name based on job ID. Basic attempt is made to sanitise input, No guarantees are made for security. - * - * @param job - * @return - */ - public static String getBaseFolderForJob(StagedFileOwner job) { - return String.format("job-%1$s", job.getId()); - } - - /** - * Deletes the entire job stage in directory, returns true on success. It silently fails and log the failure message to error log if the operation failed. - * - * @param job - * Must have its fileStorageId parameter set - */ - public boolean deleteStageInDirectory(StagedFileOwner job) { - boolean status = false; - try { - File jobInputDir = new File(pathConcat(stagingInformation.getStageInDirectory(), getBaseFolderForJob(job))); - logger.debug("Recursively deleting " + jobInputDir.getPath()); - if (!jobInputDir.exists()) { - status = true; - } - status = FileIOUtil.deleteFilesRecursive(jobInputDir); - } catch (Exception ex) { - logger.warn("There was a problem wiping the stage in directory for job: " + job, ex); - } - return status; - } - - /** - * Deletes a specific file from the job stage in directory. - * - * @param job - * Must have its fileStorageId parameter set - * @param fileName - * @return - */ - public boolean deleteStageInFile(StagedFileOwner job, String fileName) { - File file = getFile(job, fileName); - logger.debug("deleting " + file.getPath()); - if (!file.exists()) { - return true; - } - - return file.delete(); - } - - /** - * Renames a specific file in the job stage in directory. If currentFileName DNE, this will return false. If newFileName exists, it will be deleted first. - * if currentFileName equals newFileName this will return true and perform no operations. - * - * @param job - * Must have its fileStorageId parameter set - * @param currentFileName - * The file to be renamed - * @param newFileName - * The new file name (if it exists, it will be overwritten) - * @return - */ - public boolean renameStageInFile(StagedFileOwner job, String currentFileName, String newFileName) { - File currentFile = getFile(job, currentFileName); - File newFile = getFile(job, newFileName); - - //In case we rename something into itself - if (newFile.getAbsolutePath().equals(currentFile.getAbsolutePath())) { - return true; - } - - if (!currentFile.exists()) { - return false; - } - - if (newFile.exists()) { - newFile.delete(); - } - - logger.debug("renaming " + currentFile.getPath() + " to " + newFile.getPath()); - return currentFile.renameTo(newFile); - } - - /** - * Returns true if the specified file in the job staging area exists. False otherwise. - * - * @param job - * Must have its fileStorageId parameter set - * @param currentFileName - * The file to be checked - * @return - */ - public boolean stageInFileExists(StagedFileOwner job, String fileName) { - return getFile(job, fileName).exists(); - } - - /** - * Given a job see if a folder for the internal staging area has been created. - * - * @param job - * Must have its fileStorageId parameter set - * @return - * @throws IOException - * If the directory creation fails - */ - public boolean stageInDirectoryExists(StagedFileOwner job) { - String jobInputDir = pathConcat(stagingInformation.getStageInDirectory(), getBaseFolderForJob(job)); - return new File(jobInputDir).exists(); - } - - /** - * Given a job create a folder that is unique to that job in the internal staging area. - * - * @param job - * Must have its fileStorageId parameter set - * @return - * @throws IOException - * If the directory creation fails - */ - public void generateStageInDirectory(StagedFileOwner job) throws PortalServiceException { - String jobInputDir = pathConcat(stagingInformation.getStageInDirectory(), getBaseFolderForJob(job)); - - logger.debug("Attempting to generate job input dir " + jobInputDir); - - boolean success = new File(jobInputDir).mkdirs(); - if (!success) { - throw new PortalServiceException("Failed to create stage in directory: " + jobInputDir); - } - } - - /** - * Lists every file in the specified job's stage in directory - * - * @param job - * Must have its fileStorageId parameter set - * @return - * @throws IOException - */ - public StagedFile[] listStageInDirectoryFiles(StagedFileOwner job) throws PortalServiceException { - //List files in directory, add them to array - File directory = new File(pathConcat(stagingInformation.getStageInDirectory(), getBaseFolderForJob(job))); - logger.debug("Attempting to list files at " + directory.getPath()); - if (!directory.isDirectory()) { - throw new PortalServiceException("Not a directory: " + directory.getPath()); - } - File[] files = directory.listFiles(); - if (files == null) { - throw new PortalServiceException("Unable to list files in: " + directory.getPath(), ""); - } - - StagedFile[] stagedFiles = new StagedFile[files.length]; - for (int i = 0; i < stagedFiles.length; i++) { - stagedFiles[i] = new StagedFile(job, files[i].getName(), files[i]); - } - - return stagedFiles; - } - - /** - * Opens the specified staging file for reading - * - * The returned stream must be closed when finished with - * - * @param stagedFile - * @return - * @throws PortalServiceException - */ - public InputStream readFile(StagedFile stagedFile) throws PortalServiceException { - return this.readFile(stagedFile.getOwner(), stagedFile.getName()); - } - - /** - * Opens the specified staging file for reading - * - * The returned stream must be closed when finished with - * - * @param job - * Must have its fileStorageId parameter set - * @param fileName - * @return the staging file input stream if the file exists otherwise null - */ - public InputStream readFile(StagedFileOwner job, String fileName) throws PortalServiceException { - try { - File f = getFile(job, fileName); - FileInputStream fis = null; - if (f.exists()) { - fis = new FileInputStream(f); - } - return fis; - } catch (Exception e) { - throw new PortalServiceException(e.getMessage(), e); - } - } - - /** - * Opens the specified staging file for writing. If it DNE, it will be created - * - * The returned stream must be closed when finished with - * - * @param stagedFile - * @return - * @throws PortalServiceException - */ - public OutputStream writeFile(StagedFile stagedFile) throws PortalServiceException { - return writeFile(stagedFile.getOwner(), stagedFile.getName(), false); - } - - /** - * Opens the specified staging file for writing, If it DNE, it will be created - * - * The returned stream must be closed when finished with - * - * @param job - * @param fileName - * @return - */ - public OutputStream writeFile(StagedFileOwner job, String fileName) throws PortalServiceException { - return writeFile(job, fileName, false); - } - - /** - * Opens the specified staging file for writing or appending. If it DNE, it will be created. - * - * The returned stream must be closed when finished with - * - * @param job - * Must have its fileStorageId parameter set - * @param fileName - * @param append - * Should the file be overwritten or appended to. true to append, false to overwrite - * @return - */ - public OutputStream writeFile(StagedFileOwner job, String fileName, boolean append) throws PortalServiceException { - File f = getFile(job, fileName); - try { - return new FileOutputStream(f, append); - } catch (Exception e) { - throw new PortalServiceException(e.getMessage(), e); - } - } - - /** - * Upload files to the staging directory of the specified job. Helper function for code deduplication - * - * @param job - * @param f - * @return - * @throws PortalServiceException - */ - private StagedFile fileUploadHelper(StagedFileOwner job, MultipartFile f) - throws PortalServiceException { - String originalFileName = f.getOriginalFilename(); - String directory = pathConcat(stagingInformation.getStageInDirectory(), getBaseFolderForJob(job)); - String destinationPath = pathConcat(directory, originalFileName); - logger.debug("Saving uploaded file to " + destinationPath); - - File destination = new File(destinationPath); - if (destination.exists()) { - logger.debug("Will overwrite existing file."); - } - - try { - f.transferTo(destination); - } catch (Exception ex) { - logger.error("Failure during transfer:" + ex.getMessage()); - logger.debug("error:", ex); - throw new PortalServiceException("Failure during transfer", ex); - } - - return new StagedFile(job, originalFileName, destination); - } - - /** - * Given a MultipartHttpServletRequest with an internal file parameter, write that file to the staging directory of the specified job - * - * returns a FileInfo object describing the file on the file system - * - * @param job - * Must have its fileStorageId parameter set - * @param request - * @throws PortalServiceException - */ - public StagedFile handleFileUpload(StagedFileOwner job, MultipartHttpServletRequest request) - throws PortalServiceException { - MultipartFile f = request.getFile("file"); - if (f == null) { - throw new PortalServiceException("No file parameter provided."); - } - - return fileUploadHelper(job, f); - } - - /** - * Given a MultipartHttpServletRequest with an internal file parameter for several files, write that files to the staging directory of the specified job - * - * returns a List of FileInfo objects describing the file on the file system - * - * @param job - * Must have its fileStorageId parameter set - * @param request - * @throws IOException - */ - public List handleMultiFileUpload(StagedFileOwner job, MultipartHttpServletRequest request) - throws PortalServiceException { - - List result = new ArrayList<>(); - - List files = request.getFiles("file"); - if (files == null) { - throw new PortalServiceException("No file parameter provided."); - } - - if (files.size() > 0) { - for (MultipartFile multipartFile : files) { - result.add(fileUploadHelper(job, multipartFile)); - } - } else { - throw new PortalServiceException("No file parameter provided."); - } - - return result; - } - - /** - * This function will attempt to download fileName from job's staging directory by writing directly to the output stream of response. - * - * response will have its internal outputStream directly accessed and written to (if the internal file request is successful). - * - * @param stagedFile - * Must have owner and name set - * @throws IOException - */ - public void handleFileDownload(StagedFile stagedFile, HttpServletResponse response) throws PortalServiceException { - handleFileDownload(stagedFile.getOwner(), stagedFile.getName(), response); - } - - /** - * This function will attempt to download fileName from job's staging directory by writing directly to the output stream of response. - * - * response will have its internal outputStream directly accessed and written to (if the internal file request is successful). - * - * @param job - * Must have its fileStorageId parameter set - * @throws IOException - */ - @SuppressWarnings("resource") - public void handleFileDownload(StagedFileOwner job, String fileName, HttpServletResponse response) - throws PortalServiceException { - String directory = pathConcat(stagingInformation.getStageInDirectory(), getBaseFolderForJob(job)); - String filePath = pathConcat(directory, fileName); - - logger.debug("Downloading: " + filePath); - - //Simple sanity check - File f = new File(filePath); - if (!f.canRead()) { - throw new PortalServiceException("File " + f.getPath() + " not readable!"); - } - - //Start configuring our response for a download stream - response.setContentType("application/octet-stream"); - response.setHeader("Content-Disposition", - "attachment; filename=\"" + fileName + "\""); - - //Then push all data down - byte[] buffer = new byte[4096]; - int count = 0; - OutputStream out = null; - FileInputStream fin = null; - try { - out = response.getOutputStream(); - fin = new FileInputStream(f); - while ((count = fin.read(buffer)) != -1) { - out.write(buffer, 0, count); - } - out.flush(); - } catch (IOException ex) { - logger.error("Failure during download:" + ex.getMessage()); - logger.debug("error:", ex); - throw new PortalServiceException("Failure during transfer", ex); - } finally { - FileIOUtil.closeQuietly(fin); - } - } -} diff --git a/src/main/java/org/auscope/portal/core/services/cloud/STSRequirement.java b/src/main/java/org/auscope/portal/core/services/cloud/STSRequirement.java deleted file mode 100644 index 83372cab8..000000000 --- a/src/main/java/org/auscope/portal/core/services/cloud/STSRequirement.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.auscope.portal.core.services.cloud; - -/** - * The various levels of STS support that an instance of CloudStorageService/CloudComputeService can implement - * @author Josh Vote (CSIRO) - * - */ -public enum STSRequirement { - /** - * Only STS enabled jobs will be allowed to read/write data using this instance. Non STS - * enabled jobs will error when using this class - */ - Mandatory, - /** - * STS jobs will use STS as per normal, non STS jobs will use the inbuilt credentials - */ - Permissable, - /** - * ALL jobs (STS or otherwise) will be forced to use the inbuilt credentials - */ - ForceNone -} diff --git a/src/main/java/org/auscope/portal/core/services/cloud/monitor/JobStatusChangeListener.java b/src/main/java/org/auscope/portal/core/services/cloud/monitor/JobStatusChangeListener.java deleted file mode 100644 index 7f9f84fb2..000000000 --- a/src/main/java/org/auscope/portal/core/services/cloud/monitor/JobStatusChangeListener.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.auscope.portal.core.services.cloud.monitor; - -import java.util.EventListener; - -import org.auscope.portal.core.cloud.CloudJob; - -/** - * All job status change listener or handler should implement this interface. - * - * @author Richard Goh - */ -public interface JobStatusChangeListener extends EventListener { - /** - * This is a synchronous event to alert that a particular CloudJob status has changed - * - * Implementors should ensure that this method is thread safe - * - * @param job - * @param newStatus - * @param oldStatus - */ - public void handleStatusChange(CloudJob job, String newStatus, String oldStatus); -} \ No newline at end of file diff --git a/src/main/java/org/auscope/portal/core/services/cloud/monitor/JobStatusException.java b/src/main/java/org/auscope/portal/core/services/cloud/monitor/JobStatusException.java deleted file mode 100644 index 888236175..000000000 --- a/src/main/java/org/auscope/portal/core/services/cloud/monitor/JobStatusException.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.auscope.portal.core.services.cloud.monitor; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; - -import org.auscope.portal.core.cloud.CloudJob; - -/** - * General exception thrown when there is an error checking/updating job status. This class can be used as a general purpose exception or as a collection of - * exceptions for a batch status update - * - * @author Josh Vote - * - */ -public class JobStatusException extends Exception { - private static final long serialVersionUID = -7545429028086868801L; - - List exceptions; - List cloudJobs; - - public JobStatusException(Throwable t, CloudJob job) { - super(t); - this.exceptions = Arrays.asList(t); - this.cloudJobs = Arrays.asList(job); - } - - public JobStatusException(Collection t, Collection jobs) { - super(t.iterator().next()); - this.exceptions = new ArrayList<>(t); - this.cloudJobs = new ArrayList<>(jobs); - } - - /** - * Gets a list of all exceptions encapsulated by this job status exception - * - * @return - */ - public List getExceptions() { - return exceptions; - } - - /** - * Gets the cloud jobs associated with the throwables in exceptions. There should be a 1-1 correspondance with getExceptions - * - * @return - */ - public List getCloudJobs() { - return cloudJobs; - } - - /** - * Creates a message concatenated from all contained throwables - */ - @Override - public String getMessage() { - StringBuilder sb = new StringBuilder(); - - for (Throwable t : exceptions) { - sb.append(t.getClass().toString()); - sb.append(": [\""); - sb.append(t.getMessage()); - sb.append("\"] "); - } - - return sb.toString(); - } -} diff --git a/src/main/java/org/auscope/portal/core/services/cloud/monitor/JobStatusMonitor.java b/src/main/java/org/auscope/portal/core/services/cloud/monitor/JobStatusMonitor.java deleted file mode 100644 index 723762ebb..000000000 --- a/src/main/java/org/auscope/portal/core/services/cloud/monitor/JobStatusMonitor.java +++ /dev/null @@ -1,104 +0,0 @@ -package org.auscope.portal.core.services.cloud.monitor; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.auscope.portal.core.cloud.CloudJob; - -/** - * A simple class containing event listeners and the ability to poll CloudJob instances for information about their current status - * - * @author Josh Vote - * - */ -public class JobStatusMonitor { - - protected final Log log = LogFactory.getLog(getClass()); - - /** An object for reading status information about a particular job */ - private JobStatusReader jobStatusReader; - /** Event listeners to be notified whenever a job's status is changed (and detected by this class) */ - private JobStatusChangeListener[] jobStatusChangeListeners; - - /** - * Creates a new instance of this class - * - * @param jobStatusReader - * @param jobStatusChangeListeners - */ - public JobStatusMonitor(JobStatusReader jobStatusReader, JobStatusChangeListener[] jobStatusChangeListeners) { - super(); - this.jobStatusReader = jobStatusReader; - this.jobStatusChangeListeners = jobStatusChangeListeners; - } - - private void statusChanged(CloudJob job, String newStatus, String oldStatus) { - for (JobStatusChangeListener l : jobStatusChangeListeners) { - try { - l.handleStatusChange(job, newStatus, oldStatus); - } catch (Exception ex) { - //Simply log it if the event handler fails and move on - log.error("An error has occurred while handling status change event: " + ex.getMessage()); - log.debug("Exception: ", ex); - } - } - } - - /** - * Force a status update of a particular job. This is a blocking method that will not return until all status change listeners have finished their updates. - * - * @param job - * The job to update - may have its fields modified by status change listeners - * @throws JobStatusException - */ - public void statusUpdate(CloudJob job) throws JobStatusException { - String oldStatus = job.getStatus(); - String newStatus; - - try { - newStatus = jobStatusReader.getJobStatus(job); - } catch (Exception ex) { - throw new JobStatusException(ex, job); - } - - if (newStatus != null && !newStatus.equals(oldStatus)) { - statusChanged(job, newStatus, oldStatus); - } else { - log.trace("Skip bad or status quo job. Job id: " + job.getId()); - } - } - - /** - * Force a status update of a particular collection of jobs. This is a blocking method that will not return until all status change listeners have finished - * their updates for all jobs whose status has changed. - * - * If any job throws an exception, the exception will be stored and subsequent jobs will continue to be updated. At the end of all updates, all exceptions - * thrown will be wrapped in a single JobStatusException and rethrown - * - * @param jobs - * The job collection to update - may have its member fields modified by status change listeners - * @throws JobStatusException - * If and only if one or more job status updates fail - */ - public void statusUpdate(Collection jobs) throws JobStatusException { - List exceptions = new ArrayList<>(); - List failedUpdates = new ArrayList<>(); - - for (CloudJob job : jobs) { - //Do all updates before throwing exceptions - try { - statusUpdate(job); - } catch (Throwable t) { - failedUpdates.add(job); - exceptions.add(t); - } - } - - if (!exceptions.isEmpty()) { - throw new JobStatusException(exceptions, failedUpdates); - } - } -} diff --git a/src/main/java/org/auscope/portal/core/services/cloud/monitor/JobStatusReader.java b/src/main/java/org/auscope/portal/core/services/cloud/monitor/JobStatusReader.java deleted file mode 100644 index a6ab98b78..000000000 --- a/src/main/java/org/auscope/portal/core/services/cloud/monitor/JobStatusReader.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.auscope.portal.core.services.cloud.monitor; - -import org.auscope.portal.core.cloud.CloudJob; - -/** - * An interface that represents functionality for reading the latest details about a specific CloudJob. - * - * @author Josh Vote - * - */ -public interface JobStatusReader { - /** - * Implementors may either just return cloudJob.getStatus() or may calculate the latest status in a manner specific to the CloudJob implementation. - * - * @param cloudJob - * a CloudJob whose status will be calculated - * @return - */ - public String getJobStatus(CloudJob cloudJob); -} diff --git a/src/main/java/org/auscope/portal/core/services/namespaces/AWSEC2NamespaceContext.java b/src/main/java/org/auscope/portal/core/services/namespaces/AWSEC2NamespaceContext.java deleted file mode 100644 index f454f94fd..000000000 --- a/src/main/java/org/auscope/portal/core/services/namespaces/AWSEC2NamespaceContext.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.auscope.portal.core.services.namespaces; - -/** - * A simple implementation of NamespaceContext . Instances are - * immutable. - * - * This is suitable for reading AWS EC2 response XML - * - * @version $Id$ - */ -public class AWSEC2NamespaceContext extends IterableNamespace { - - public AWSEC2NamespaceContext() { - map.put("aws", "http://ec2.amazonaws.com/doc/2015-10-01/"); - } -} diff --git a/src/main/java/org/auscope/portal/core/uifilter/GenericFilter.java b/src/main/java/org/auscope/portal/core/uifilter/GenericFilter.java index aa00d102a..f60941a4f 100644 --- a/src/main/java/org/auscope/portal/core/uifilter/GenericFilter.java +++ b/src/main/java/org/auscope/portal/core/uifilter/GenericFilter.java @@ -14,8 +14,6 @@ import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; -import com.amazonaws.services.sqs.model.UnsupportedOperationException; - /** * @author Victor Tey diff --git a/src/main/java/org/auscope/portal/core/view/knownlayer/VMFSelector.java b/src/main/java/org/auscope/portal/core/view/knownlayer/VMFSelector.java index 1cfa08891..e29451c57 100644 --- a/src/main/java/org/auscope/portal/core/view/knownlayer/VMFSelector.java +++ b/src/main/java/org/auscope/portal/core/view/knownlayer/VMFSelector.java @@ -2,13 +2,9 @@ import java.net.MalformedURLException; import java.net.URL; -import java.util.ArrayList; -import org.auscope.portal.core.services.responses.csw.AbstractCSWOnlineResource; import org.auscope.portal.core.services.responses.csw.CSWRecord; import org.json.JSONArray; -import org.auscope.portal.core.services.responses.csw.AbstractCSWOnlineResource.OnlineResourceType; -import org.auscope.portal.core.view.knownlayer.KnownLayerSelector.RelationType; public class VMFSelector implements KnownLayerSelector { @@ -59,12 +55,9 @@ public void setLayerName(String layerName) { */ @Override public RelationType isRelatedRecord(CSWRecord record) { - - if (layerName.equals(record.getLayerName())) { return RelationType.Belongs; } - return RelationType.NotRelated; } diff --git a/src/test/java/org/auscope/portal/core/cloud/TestStagedFile.java b/src/test/java/org/auscope/portal/core/cloud/TestStagedFile.java deleted file mode 100644 index a202684d7..000000000 --- a/src/test/java/org/auscope/portal/core/cloud/TestStagedFile.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.auscope.portal.core.cloud; - -import java.io.File; - -import org.auscope.portal.core.test.PortalTestClass; -import org.junit.Assert; -import org.junit.Test; - -/** - * Unit tests for StagedFile - * - * @author Joshua - * - */ -public class TestStagedFile extends PortalTestClass { - /** - * Tests equals and hashcode align - */ - @Test - public void testEquality() { - CloudJob c1 = new CloudJob(123); - CloudJob c2 = new CloudJob(456); - - File mockFile = new File("xxx");//context.mock(File.class); - - StagedFile f1 = new StagedFile(c1, "f1", mockFile); - StagedFile f2 = new StagedFile(c1, "f1", mockFile); - StagedFile f3 = new StagedFile(c2, "f1", mockFile); - StagedFile f4 = new StagedFile(c1, "f2", mockFile); - - Assert.assertTrue(equalsWithHashcode(f1, f1)); - Assert.assertTrue(equalsWithHashcode(f1, f2)); - Assert.assertFalse(f1.equals(f3)); - Assert.assertFalse(f1.equals(f4)); - } -} diff --git a/src/test/java/org/auscope/portal/core/services/cloud/MockCloudStorageService.java b/src/test/java/org/auscope/portal/core/services/cloud/MockCloudStorageService.java deleted file mode 100644 index 2fb22ab4c..000000000 --- a/src/test/java/org/auscope/portal/core/services/cloud/MockCloudStorageService.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * - */ -package org.auscope.portal.core.services.cloud; - -import org.jclouds.blobstore.BlobStoreContext; - -/** - * @author fri096 - * - */ -public class MockCloudStorageService extends CloudStorageServiceJClouds { - - private BlobStoreContext mockBlobStoreContext; - - public MockCloudStorageService(BlobStoreContext mockBlobStoreContext) { - super(); - this.mockBlobStoreContext=mockBlobStoreContext; - } - - @Override - public BlobStoreContext getBlobStoreContext(String arn, String clientSecret) { - return mockBlobStoreContext; - } - -} diff --git a/src/test/java/org/auscope/portal/core/services/cloud/TestCloudComputeService.java b/src/test/java/org/auscope/portal/core/services/cloud/TestCloudComputeService.java deleted file mode 100644 index b117a0bbd..000000000 --- a/src/test/java/org/auscope/portal/core/services/cloud/TestCloudComputeService.java +++ /dev/null @@ -1,236 +0,0 @@ -package org.auscope.portal.core.services.cloud; - -import java.io.IOException; -import java.nio.charset.Charset; - -import org.auscope.portal.core.cloud.CloudJob; -import org.auscope.portal.core.services.PortalServiceException; -import org.auscope.portal.core.services.cloud.CloudComputeService.InstanceStatus; -import org.auscope.portal.core.test.PortalTestClass; -import org.jclouds.compute.ComputeService; -import org.jclouds.compute.RunNodesException; -import org.jclouds.compute.domain.NodeMetadata; -import org.jclouds.compute.domain.NodeMetadata.Status; -import org.jclouds.compute.domain.Template; -import org.jclouds.compute.domain.TemplateBuilder; -import org.jclouds.openstack.nova.v2_0.NovaApi; -import org.jclouds.openstack.nova.v2_0.compute.options.NovaTemplateOptions; -import org.jmock.Expectations; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import com.google.common.base.Predicate; -import com.google.common.collect.ImmutableSet; - -public class TestCloudComputeService extends PortalTestClass { - - private final ComputeService mockComputeService = context.mock(ComputeService.class); - private final NovaTemplateOptions mockTemplateOptions = context.mock(NovaTemplateOptions.class); - private final TemplateBuilder mockTemplateBuilder = context.mock(TemplateBuilder.class); - private final Template mockTemplate = context.mock(Template.class); - private final NodeMetadata mockMetadata = context.mock(NodeMetadata.class); - private final NovaApi mockNovaApi = context.mock(NovaApi.class); - private final Predicate mockFilter = (Predicate)context.mock(Predicate.class); - private final RunNodesException mockException = context.mock(RunNodesException.class); - - private CloudComputeServiceNectar service; - - private CloudJob job; - - @Before - public void initJobObject() { - job = new CloudJob(13); - job.setComputeVmId("image-id"); - job.setComputeInstanceType("type"); - service = new CloudComputeServiceNectar(mockComputeService, mockNovaApi, mockFilter); - service.setGroupName("group-name"); - service.setKeypair("vgl-developers"); - - } - - /** - * Tests that job execution correctly calls and parses a response from AmazonEC2 - * @throws RunNodesException - * @throws PortalServiceException - */ - @Test - public void testExecuteJob() throws RunNodesException, PortalServiceException { - final String userDataString = "user-data-string"; - final String expectedInstanceId = "instance-id"; - - context.checking(new Expectations() { - { - oneOf(mockComputeService).templateOptions(); - will(returnValue(mockTemplateOptions)); - oneOf(mockComputeService).templateBuilder(); - will(returnValue(mockTemplateBuilder)); - - oneOf(mockTemplateOptions).keyPairName("vgl-developers"); - will(returnValue(mockTemplateOptions)); - oneOf(mockTemplateOptions).userData(userDataString.getBytes(Charset.forName("UTF-8"))); - will(returnValue(mockTemplateOptions)); - - oneOf(mockTemplateBuilder).imageId(job.getComputeVmId()); - will(returnValue(mockTemplateBuilder)); - oneOf(mockTemplateBuilder).hardwareId(job.getComputeInstanceType()); - will(returnValue(mockTemplateBuilder)); - oneOf(mockTemplateBuilder).options(mockTemplateOptions); - will(returnValue(mockTemplateBuilder)); - oneOf(mockTemplateBuilder).build(); - will(returnValue(mockTemplate)); - - oneOf(mockComputeService).createNodesInGroup("group-name", 1, mockTemplate); - will(returnValue(ImmutableSet. of(mockMetadata))); - - oneOf(mockMetadata).getId(); - will(returnValue(expectedInstanceId)); - } - }); - - String actualInstanceId = service.executeJob(job, userDataString); - - Assert.assertEquals(expectedInstanceId, actualInstanceId); - } - - /** - * Tests that job execution correctly calls and parses a response from AmazonEC2 when EC2 reports failure by returning 0 running instances. - * @throws RunNodesException - * @throws PortalServiceException - */ - @Test(expected = PortalServiceException.class) - public void testExecuteJobFailure() throws RunNodesException, PortalServiceException { - final String userDataString = "user-data-string"; - - context.checking(new Expectations() { - { - oneOf(mockComputeService).templateOptions(); - will(returnValue(mockTemplateOptions)); - oneOf(mockComputeService).templateBuilder(); - will(returnValue(mockTemplateBuilder)); - - oneOf(mockTemplateOptions).keyPairName("vgl-developers"); - will(returnValue(mockTemplateOptions)); - oneOf(mockTemplateOptions).userData(userDataString.getBytes(Charset.forName("UTF-8"))); - will(returnValue(mockTemplateOptions)); - - oneOf(mockTemplateBuilder).imageId(job.getComputeVmId()); - will(returnValue(mockTemplateBuilder)); - oneOf(mockTemplateBuilder).hardwareId(job.getComputeInstanceType()); - will(returnValue(mockTemplateBuilder)); - oneOf(mockTemplateBuilder).options(mockTemplateOptions); - will(returnValue(mockTemplateBuilder)); - oneOf(mockTemplateBuilder).build(); - will(returnValue(mockTemplate)); - - oneOf(mockComputeService).createNodesInGroup("group-name", 1, mockTemplate); - will(throwException(mockException)); - - allowing(mockComputeService).destroyNodesMatching(mockFilter); - - allowing(mockException).fillInStackTrace(); - will(returnValue(mockException)); - allowing(mockException).getMessage(); - will(returnValue("mock-message")); - } - }); - - service.executeJob(job, userDataString); - } - - /** - * Tests that job execution correctly calls and parses a response from AmazonEC2 when EC2 reports failure by returning 0 running instances. - * @throws RunNodesException - * @throws PortalServiceException - */ - @Test(expected = PortalServiceException.class) - public void testExecuteJobFailure_EmptyResults() throws RunNodesException, PortalServiceException { - final String userDataString = "user-data-string"; - - context.checking(new Expectations() { - { - - oneOf(mockComputeService).templateOptions(); - will(returnValue(mockTemplateOptions)); - oneOf(mockComputeService).templateBuilder(); - will(returnValue(mockTemplateBuilder)); - - oneOf(mockTemplateOptions).keyPairName("vgl-developers"); - will(returnValue(mockTemplateOptions)); - oneOf(mockTemplateOptions).userData(userDataString.getBytes(Charset.forName("UTF-8"))); - will(returnValue(mockTemplateOptions)); - - oneOf(mockTemplateBuilder).imageId(job.getComputeVmId()); - will(returnValue(mockTemplateBuilder)); - oneOf(mockTemplateBuilder).hardwareId(job.getComputeInstanceType()); - will(returnValue(mockTemplateBuilder)); - oneOf(mockTemplateBuilder).options(mockTemplateOptions); - will(returnValue(mockTemplateBuilder)); - oneOf(mockTemplateBuilder).build(); - will(returnValue(mockTemplate)); - - oneOf(mockComputeService).createNodesInGroup("group-name", 1, mockTemplate); - will(returnValue(ImmutableSet. of())); - - } - }); - - service.executeJob(job, userDataString); - } - - /** - * Tests that job terminate correctly calls AmazonEC2 - */ - @Test - public void testTerminateJob() { - - job.setComputeInstanceId("running-id"); - - context.checking(new Expectations() { - { - oneOf(mockComputeService).destroyNode(job.getComputeInstanceId()); - } - }); - - service.terminateJob(job); - } - - @Test - public void testGetJobStatus() throws PortalServiceException { - job.setComputeInstanceId("i-running"); - - context.checking(new Expectations() {{ - oneOf(mockComputeService).getNodeMetadata("i-running"); - will(returnValue(mockMetadata)); - - allowing(mockMetadata).getStatus(); - will(returnValue(Status.PENDING)); - }}); - - Assert.assertEquals(InstanceStatus.Pending, service.getJobStatus(job)); - } - - @Test(expected=PortalServiceException.class) - public void testGetJobStatus_IOError() throws PortalServiceException { - job.setComputeInstanceId("i-running"); - - context.checking(new Expectations() {{ - oneOf(mockComputeService).getNodeMetadata("i-running"); - will(throwException(new IOException())); - }}); - - service.getJobStatus(job); - } - - @Test - public void testGetJobStatus_ReturnNull() throws PortalServiceException { - job.setComputeInstanceId("i-running"); - - context.checking(new Expectations() {{ - oneOf(mockComputeService).getNodeMetadata("i-running"); - will(returnValue(null)); - }}); - - Assert.assertEquals(InstanceStatus.Missing, service.getJobStatus(job)); - } -} diff --git a/src/test/java/org/auscope/portal/core/services/cloud/TestCloudComputeServiceAws.java b/src/test/java/org/auscope/portal/core/services/cloud/TestCloudComputeServiceAws.java deleted file mode 100644 index 8ac43d0e3..000000000 --- a/src/test/java/org/auscope/portal/core/services/cloud/TestCloudComputeServiceAws.java +++ /dev/null @@ -1,287 +0,0 @@ -package org.auscope.portal.core.services.cloud; - -import java.util.Arrays; -import java.util.Date; -import java.util.concurrent.TimeUnit; - -import org.auscope.portal.core.cloud.CloudJob; -import org.auscope.portal.core.services.PortalServiceException; -import org.auscope.portal.core.services.cloud.CloudComputeService.InstanceStatus; -import org.auscope.portal.core.test.PortalTestClass; -import org.jmock.Expectations; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import com.amazonaws.AmazonServiceException; -import com.amazonaws.services.ec2.AmazonEC2; -import com.amazonaws.services.ec2.AmazonEC2Client; -import com.amazonaws.services.ec2.model.BlockDeviceMapping; -import com.amazonaws.services.ec2.model.DescribeImagesRequest; -import com.amazonaws.services.ec2.model.DescribeImagesResult; -import com.amazonaws.services.ec2.model.DescribeInstanceStatusRequest; -import com.amazonaws.services.ec2.model.DescribeInstanceStatusResult; -import com.amazonaws.services.ec2.model.EbsBlockDevice; -import com.amazonaws.services.ec2.model.Image; -import com.amazonaws.services.ec2.model.InstanceState; - -public class TestCloudComputeServiceAws extends PortalTestClass{ - - @SuppressWarnings("serial") - private class TestableJob extends CloudJob { - // empty - } - - private class TestableCCS extends CloudComputeServiceAws { - - private AmazonEC2Client testableClient; - - public TestableCCS(AmazonEC2Client client) { - super("", ""); - testableClient = client; - } - - @Override - protected AmazonEC2 getEc2Client(CloudJob job) throws PortalServiceException { - return testableClient; - } - } - - private AmazonEC2Client mockClient = context.mock(AmazonEC2Client.class); - private DescribeInstanceStatusResult mockDescribeResult = context.mock(DescribeInstanceStatusResult.class); - private com.amazonaws.services.ec2.model.InstanceStatus mockStatus = context.mock(com.amazonaws.services.ec2.model.InstanceStatus.class); - private InstanceState mockState = context.mock(InstanceState.class); - private DescribeImagesResult mockDescribeImageResult = context.mock(DescribeImagesResult.class); - private CloudComputeServiceAws service; - - @Before - public void setup() { - service = new TestableCCS(mockClient); - } - - @Test - public void testJobStatus_ParseRunning() throws PortalServiceException { - CloudJob job = new TestableJob(); - - job.setComputeInstanceId("testable-id"); - job.setProperty(CloudJob.PROPERTY_STS_ARN, "sts-arn"); - job.setProperty(CloudJob.PROPERTY_CLIENT_SECRET, "client-secret"); - - context.checking(new Expectations() {{ - oneOf(mockClient).describeInstanceStatus(with(any(DescribeInstanceStatusRequest.class))); - will(returnValue(mockDescribeResult)); - - allowing(mockDescribeResult).getInstanceStatuses(); - will(returnValue(Arrays.asList(mockStatus))); - - allowing(mockStatus).getInstanceState(); - will(returnValue(mockState)); - - allowing(mockState).getName(); - will(returnValue("running")); - }}); - - Assert.assertEquals(InstanceStatus.Running, service.getJobStatus(job)); - } - - @Test - public void testJobStatus_ParsePending() throws PortalServiceException { - CloudJob job = new TestableJob(); - - // Date now = new Date(); - // Date submitTime = new Date(now.getTime() - (CloudComputeServiceAws.STATUS_PENDING_SECONDS * 1000) - 1000); - - job.setComputeInstanceId("testable-id"); - job.setProperty(CloudJob.PROPERTY_STS_ARN, "sts-arn"); - job.setProperty(CloudJob.PROPERTY_CLIENT_SECRET, "client-secret"); - - context.checking(new Expectations() {{ - oneOf(mockClient).describeInstanceStatus(with(any(DescribeInstanceStatusRequest.class))); - will(returnValue(mockDescribeResult)); - - allowing(mockDescribeResult).getInstanceStatuses(); - will(returnValue(Arrays.asList(mockStatus))); - - allowing(mockStatus).getInstanceState(); - will(returnValue(mockState)); - - allowing(mockState).getName(); - will(returnValue("pending")); - }}); - - Assert.assertEquals(InstanceStatus.Pending, service.getJobStatus(job)); - } - - @Test - public void testJobStatus_ParseTerminated() throws PortalServiceException { - CloudJob job = new TestableJob(); - - job.setComputeInstanceId("testable-id"); - job.setProperty(CloudJob.PROPERTY_STS_ARN, "sts-arn"); - job.setProperty(CloudJob.PROPERTY_CLIENT_SECRET, "client-secret"); - - context.checking(new Expectations() {{ - oneOf(mockClient).describeInstanceStatus(with(any(DescribeInstanceStatusRequest.class))); - will(returnValue(mockDescribeResult)); - - allowing(mockDescribeResult).getInstanceStatuses(); - will(returnValue(Arrays.asList(mockStatus))); - - allowing(mockStatus).getInstanceState(); - will(returnValue(mockState)); - - allowing(mockState).getName(); - will(returnValue("terminated")); - }}); - - Assert.assertEquals(InstanceStatus.Missing, service.getJobStatus(job)); - } - - @Test - public void testJobStatus_ParseMissingException() throws PortalServiceException { - CloudJob job = new TestableJob(); - - job.setComputeInstanceId("testable-id"); - job.setProperty(CloudJob.PROPERTY_STS_ARN, "sts-arn"); - job.setProperty(CloudJob.PROPERTY_CLIENT_SECRET, "client-secret"); - - final AmazonServiceException ex = new AmazonServiceException("Testing Exception"); - ex.setErrorCode("InvalidInstanceID.NotFound"); - - context.checking(new Expectations() {{ - oneOf(mockClient).describeInstanceStatus(with(any(DescribeInstanceStatusRequest.class))); - will(throwException(ex)); - }}); - - Assert.assertEquals(InstanceStatus.Missing, service.getJobStatus(job)); - } - - @Test - public void testJobStatus_NewJobPending() throws PortalServiceException { - CloudJob job = new TestableJob(); - - Date now = new Date(); - Date submitTime = new Date(now.getTime() - (CloudComputeServiceAws.STATUS_PENDING_SECONDS * 1000) + 1000); - - job.setComputeInstanceId("testable-id"); - job.setProperty(CloudJob.PROPERTY_STS_ARN, "sts-arn"); - job.setProperty(CloudJob.PROPERTY_CLIENT_SECRET, "client-secret"); - job.setSubmitDate(submitTime); - - final AmazonServiceException ex = new AmazonServiceException("Testing Exception"); - ex.setErrorCode("InvalidInstanceID.NotFound"); - - context.checking(new Expectations()); - - Assert.assertEquals(InstanceStatus.Pending, service.getJobStatus(job)); - } - - /** - * In case we have some time issues and a job's submit date is in the future - * - * In this case we expect it to return pending. If it's only a few seconds in the future then it's probably just a minor date/time - * shifting error (or a daylight savings time shift). If it's a LONG time in the future, what can we expect? It's probably overengineering - * the checks if we start accounting for the latter. - * @throws PortalServiceException - */ - @Test - public void testJobStatus_FutureJob() throws PortalServiceException { - CloudJob job = new TestableJob(); - - Date now = new Date(); - Date submitTime = new Date(now.getTime() + TimeUnit.DAYS.toMillis(2)); //Throw the submit 2 days into the future to simulate some weird clock behavior - - job.setComputeInstanceId("testable-id"); - job.setProperty(CloudJob.PROPERTY_STS_ARN, "sts-arn"); - job.setProperty(CloudJob.PROPERTY_CLIENT_SECRET, "client-secret"); - job.setSubmitDate(submitTime); - - context.checking(new Expectations()); - - Assert.assertEquals(InstanceStatus.Pending, service.getJobStatus(job)); - } - - @Test(expected=PortalServiceException.class) - public void testStsRequired() throws PortalServiceException { - final TestableCCS stsService = new TestableCCS(mockClient); - stsService.setStsRequirement(STSRequirement.Mandatory); - stsService.getCredentials(null, null); - } - - @Test(expected=PortalServiceException.class) - public void testJobStatus_BadResponse() throws PortalServiceException { - CloudJob job = new TestableJob(); - - job.setComputeInstanceId("testable-id"); - job.setProperty(CloudJob.PROPERTY_STS_ARN, "sts-arn"); - job.setProperty(CloudJob.PROPERTY_CLIENT_SECRET, "client-secret"); - - final AmazonServiceException ex = new AmazonServiceException("Testing Exception"); - ex.setErrorCode("unrecognized-ID"); - ex.setStatusCode(503); - - context.checking(new Expectations() {{ - oneOf(mockClient).describeInstanceStatus(with(any(DescribeInstanceStatusRequest.class))); - will(throwException(ex)); - }}); - - service.getJobStatus(job); - } - - @Test - public void testContainsPersistentVolumes_HandleMultipleVolumes() throws PortalServiceException { - CloudJob job = new TestableJob(); - - job.setComputeVmId("computeid"); - job.setComputeInstanceId("testable-id"); - job.setProperty(CloudJob.PROPERTY_STS_ARN, "sts-arn"); - job.setProperty(CloudJob.PROPERTY_CLIENT_SECRET, "client-secret"); - - final Image mockImage = context.mock(Image.class); - final BlockDeviceMapping bdm1 = context.mock(BlockDeviceMapping.class, "bdm1"); - final BlockDeviceMapping bdm2 = context.mock(BlockDeviceMapping.class, "bdm2"); - - final EbsBlockDevice ebd1 = context.mock(EbsBlockDevice.class, "ebd1"); - final EbsBlockDevice ebd2 = context.mock(EbsBlockDevice.class, "ebd2"); - - context.checking(new Expectations() {{ - oneOf(mockClient).describeImages(with(any(DescribeImagesRequest.class))); - will(returnValue(mockDescribeImageResult)); - - allowing(mockDescribeImageResult).getImages(); - will(returnValue(Arrays.asList(mockImage))); - - allowing(mockImage).getBlockDeviceMappings(); - will(returnValue(Arrays.asList(bdm1, bdm2))); - - allowing(bdm1).getEbs(); - will(returnValue(ebd1)); - allowing(bdm2).getEbs(); - will(returnValue(ebd2)); - - allowing(ebd1).getDeleteOnTermination(); - will(returnValue(true)); - allowing(ebd2).getDeleteOnTermination(); - will(returnValue(false)); - }}); - - Assert.assertTrue(service.containsPersistentVolumes(job)); - } - - @Test(expected=PortalServiceException.class) - public void testContainsPersistentVolumes_AwsError() throws PortalServiceException { - CloudJob job = new TestableJob(); - - job.setComputeVmId("computeid"); - job.setComputeInstanceId("testable-id"); - job.setProperty(CloudJob.PROPERTY_STS_ARN, "sts-arn"); - job.setProperty(CloudJob.PROPERTY_CLIENT_SECRET, "client-secret"); - - context.checking(new Expectations() {{ - oneOf(mockClient).describeImages(with(any(DescribeImagesRequest.class))); - will(throwException(new AmazonServiceException("error"))); - }}); - - service.containsPersistentVolumes(job); - } -} diff --git a/src/test/java/org/auscope/portal/core/services/cloud/TestCloudStorageService.java b/src/test/java/org/auscope/portal/core/services/cloud/TestCloudStorageService.java deleted file mode 100644 index 9f876cda8..000000000 --- a/src/test/java/org/auscope/portal/core/services/cloud/TestCloudStorageService.java +++ /dev/null @@ -1,436 +0,0 @@ -package org.auscope.portal.core.services.cloud; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.Map; - -import org.auscope.portal.core.cloud.CloudFileInformation; -import org.auscope.portal.core.cloud.CloudJob; -import org.auscope.portal.core.services.PortalServiceException; -import org.auscope.portal.core.test.PortalTestClass; -import org.jclouds.blobstore.BlobStore; -import org.jclouds.blobstore.BlobStoreContext; -import org.jclouds.blobstore.domain.Blob; -import org.jclouds.blobstore.domain.BlobBuilder.PayloadBlobBuilder; -import org.jclouds.blobstore.domain.PageSet; -import org.jclouds.blobstore.domain.StorageMetadata; -import org.jclouds.blobstore.domain.StorageType; -import org.jclouds.blobstore.domain.internal.BlobMetadataImpl; -import org.jclouds.blobstore.domain.internal.MutableBlobMetadataImpl; -import org.jclouds.blobstore.options.ListContainerOptions; -import org.jclouds.io.MutableContentMetadata; -import org.jclouds.io.Payload; -import org.jclouds.io.payloads.BaseImmutableContentMetadata; -import org.jmock.Expectations; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import com.google.common.io.ByteSource; - - -public class TestCloudStorageService extends PortalTestClass { - private final String bucket = "bucket-name"; - - private BlobStoreContext mockBlobStoreContext = context.mock(BlobStoreContext.class); - private CloudJob job; - private CloudStorageService service; - - private final String jobStorageBaseKey = "job/base/key"; - - @Before - public void initJobObject() { - job = new CloudJob(13); - job.setStorageBaseKey(jobStorageBaseKey); - service = new MockCloudStorageService(mockBlobStoreContext); - service.setBucket(bucket); - } - - @Test - public void testGetJobFileData() throws IOException, PortalServiceException { - final String myKey = "my.key"; - final BlobStore mockBlobStore = context.mock(BlobStore.class); - final Blob mockBlob = context.mock(Blob.class); - final ByteArrayInputStream is = new ByteArrayInputStream(new byte[0]); - - try (final Payload mockPayload = context.mock(Payload.class)) { - context.checking(new Expectations() { - { - oneOf(mockBlobStoreContext).close(); - oneOf(mockBlobStoreContext).getBlobStore(); - will(returnValue(mockBlobStore)); - - oneOf(mockBlobStore).getBlob(bucket, jobStorageBaseKey + "/" + myKey); - will(returnValue(mockBlob)); - - oneOf(mockBlob).getPayload(); - will(returnValue(mockPayload)); - - oneOf(mockPayload).openStream(); - will(returnValue(is)); - - allowing(mockPayload).close(); - } - }); - - try (InputStream actualInputStream = service.getJobFile(job, myKey)) { - Assert.assertSame(is, actualInputStream); - } - } - } - - @Test - public void testGetJobFileMetaData() throws URISyntaxException, PortalServiceException { - final String myKey = "my.key"; - final BlobStore mockBlobStore = context.mock(BlobStore.class); - - final Map userMetadata = new HashMap<>(); - final BaseImmutableContentMetadata contentMetadata = new BaseImmutableContentMetadata("mime/type", 24L, null, null, null, null, null); - final StorageMetadata metadata = new BlobMetadataImpl("id", "name", null, new URI("http://example.cloud/file"), "asdsadsasd", new Date(), new Date(), userMetadata, new URI("http://example.cloud/publicfile"), null, contentMetadata); - - - context.checking(new Expectations() { - { - oneOf(mockBlobStoreContext).close(); - oneOf(mockBlobStoreContext).getBlobStore(); - will(returnValue(mockBlobStore)); - - oneOf(mockBlobStore).blobMetadata(bucket, jobStorageBaseKey + "/" + myKey); - will(returnValue(metadata)); - } - }); - - CloudFileInformation result = service.getJobFileMetadata(job, myKey); - - Assert.assertEquals("asdsadsasd", result.getFileHash()); - Assert.assertEquals("name", result.getName()); - Assert.assertEquals(24L, result.getSize()); - } - - @Test - public void testGetJobFileSanitation() throws URISyntaxException, PortalServiceException { - final String myKey = "my.key"; //will have . preserved - final BlobStore mockBlobStore = context.mock(BlobStore.class); - - final Map userMetadata = new HashMap<>(); - final BaseImmutableContentMetadata contentMetadata = new BaseImmutableContentMetadata("mime/type", 24L, null, null, null, null, null); - final StorageMetadata metadata = new BlobMetadataImpl("id", "name", null, new URI("http://example.cloud/file"), "asdsadsasd", new Date(), new Date(), userMetadata, new URI("http://example.cloud/publicfile"), null, contentMetadata); - - job.setStorageBaseKey(null); - job.setUser("user"); - service.setJobPrefix("prefix.san-"); //will get the . removed - - context.checking(new Expectations() { - { - oneOf(mockBlobStoreContext).close(); - oneOf(mockBlobStoreContext).getBlobStore(); - will(returnValue(mockBlobStore)); - - oneOf(mockBlobStore).blobMetadata(bucket, "prefix_san-user-0000000013/my.key"); - will(returnValue(metadata)); - } - }); - - CloudFileInformation result = service.getJobFileMetadata(job, myKey); - - Assert.assertEquals("asdsadsasd", result.getFileHash()); - Assert.assertEquals("name", result.getName()); - Assert.assertEquals(24L, result.getSize()); - } - - /** - * Tests that requests for listing files successfully call all dependencies - * @throws URISyntaxException - * @throws PortalServiceException - */ - @Test - public void testListOutputJobFiles() throws URISyntaxException, PortalServiceException { - final BlobStore mockBlobStore = context.mock(BlobStore.class); - - final BlobMetadataImpl mockStorageMetadata1 = context.mock(BlobMetadataImpl.class, "mockStorageMetadata1"); - final BlobMetadataImpl mockStorageMetadata2 = context.mock(BlobMetadataImpl.class, "mockStorageMetadata2"); - final BlobMetadataImpl mockStorageMetadata3 = context.mock(BlobMetadataImpl.class, "mockStorageMetadata3"); - - final MutableContentMetadata mockObj1ContentMetadata = context.mock(MutableContentMetadata.class, "mockObj1Md"); - final MutableContentMetadata mockObj2ContentMetadata = context.mock(MutableContentMetadata.class, "mockObj2Md"); - final MutableContentMetadata mockObj3ContentMetadata = context.mock(MutableContentMetadata.class, "mockObj3Md"); - final PageSet mockPageSet = context.mock(PageSet.class); - - LinkedList ls = new LinkedList<>(); - ls.add(context.mock(MutableBlobMetadataImpl.class, "mockObj1")); - ls.add(context.mock(MutableBlobMetadataImpl.class, "mockObj2")); - - final String obj1Key = "key/obj1"; - final String obj1Bucket = "bucket1"; - final long obj1Length = 1234L; - final String obj2Key = "key/obj2"; - final String obj2Bucket = "bucket2"; - final long obj2Length = 4567L; - - context.checking(new Expectations() { - { - oneOf(mockBlobStoreContext).close(); - oneOf(mockBlobStoreContext).getBlobStore(); - will(returnValue(mockBlobStore)); - - oneOf(mockBlobStore).list(with(equal(bucket)), with(any(ListContainerOptions.class))); - will(returnValue(mockPageSet)); - - allowing(mockPageSet).getNextMarker(); - will(returnValue(null)); - - allowing(mockPageSet).iterator(); - will(returnValue(Arrays.asList(mockStorageMetadata1, mockStorageMetadata2, mockStorageMetadata3).iterator())); - - allowing(mockStorageMetadata1).getName(); - will(returnValue(obj1Key)); - allowing(mockStorageMetadata1).getUri(); - will(returnValue(new URI(obj1Bucket))); - allowing(mockStorageMetadata1).getContentMetadata(); - will(returnValue(mockObj1ContentMetadata)); - allowing(mockStorageMetadata1).getType(); - will(returnValue(StorageType.BLOB)); - allowing(mockObj1ContentMetadata).getContentLength(); - will(returnValue(obj1Length)); - allowing(mockStorageMetadata1).getETag(); - will(returnValue("sadgafsadfa")); - - - allowing(mockStorageMetadata2).getName(); - will(returnValue(obj2Key)); - allowing(mockStorageMetadata2).getUri(); - will(returnValue(new URI(obj2Bucket))); - allowing(mockStorageMetadata2).getContentMetadata(); - will(returnValue(mockObj2ContentMetadata)); - allowing(mockStorageMetadata2).getType(); - will(returnValue(StorageType.BLOB)); - allowing(mockObj2ContentMetadata).getContentLength(); - will(returnValue(obj2Length)); - allowing(mockStorageMetadata2).getETag(); - will(returnValue("mocoqqwiiluhqw")); - - allowing(mockStorageMetadata3).getContentMetadata(); - will(returnValue(mockObj3ContentMetadata)); - allowing(mockStorageMetadata3).getType(); - will(returnValue(StorageType.FOLDER)); - } - }); - - CloudFileInformation[] fileInfo = service.listJobFiles(job); - Assert.assertNotNull(fileInfo); - Assert.assertEquals(ls.size(), fileInfo.length); - Assert.assertEquals(obj1Key, fileInfo[0].getCloudKey()); - Assert.assertEquals(obj1Length, fileInfo[0].getSize()); - Assert.assertEquals(obj2Key, fileInfo[1].getCloudKey()); - Assert.assertEquals(obj2Length, fileInfo[1].getSize()); - } - - /** - * Tests that requests for uploading files successfully call all dependencies - * @throws PortalServiceException - */ - @Test - public void testUploadJobFiles() throws PortalServiceException { - final BlobStore mockBlobStore = context.mock(BlobStore.class); - - final Blob mockBlob1 = context.mock(Blob.class, "mockBlob1"); - final Blob mockBlob2 = context.mock(Blob.class, "mockBlob2"); - - final PayloadBlobBuilder mockBuilder1 = context.mock(PayloadBlobBuilder.class, "mockBuilder1"); - final PayloadBlobBuilder mockBuilder2 = context.mock(PayloadBlobBuilder.class, "mockBuilder2"); - - @SuppressWarnings("serial") - final File[] mockFiles = new File[] { - new File("mockFile1") { - @Override - public String getName() { - return "file1Name"; - } - - @Override - public long length() { - return 1L; - } - - }, - new File("mockFile2") { - @Override - public String getName() { - return "file2Name"; - } - - @Override - public long length() { - return 1L; - } - - }, - }; - - context.checking(new Expectations() { - { - oneOf(mockBlobStoreContext).getBlobStore(); - will(returnValue(mockBlobStore)); - - oneOf(mockBlobStore).createContainerInLocation(null, bucket); - will(returnValue(true)); - - oneOf(mockBlobStore).blobBuilder(jobStorageBaseKey + "/file1Name"); - will(returnValue(mockBuilder1)); - oneOf(mockBlobStore).blobBuilder(jobStorageBaseKey + "/file2Name"); - will(returnValue(mockBuilder2)); - - allowing(mockBuilder1).contentLength(1L); - will(returnValue(mockBuilder1)); - allowing(mockBuilder1).payload(with(any(ByteSource.class))); - will(returnValue(mockBuilder1)); - oneOf(mockBuilder1).build(); - will(returnValue(mockBlob1)); - - allowing(mockBuilder2).contentLength(1L); - will(returnValue(mockBuilder2)); - allowing(mockBuilder2).payload(with(any(ByteSource.class))); - will(returnValue(mockBuilder2)); - oneOf(mockBuilder2).build(); - will(returnValue(mockBlob2)); - - oneOf(mockBlobStore).putBlob(bucket, mockBlob1); - oneOf(mockBlobStore).putBlob(bucket, mockBlob2); - oneOf(mockBlobStoreContext).close(); - } - }); - - service.uploadJobFiles(job, mockFiles); - } - - /** - * Tests that requests for uploading single files successfully call all dependencies - * @throws PortalServiceException - */ - @Test - public void testUploadSingleJobFile() throws PortalServiceException { - final BlobStore mockBlobStore = context.mock(BlobStore.class); - - final Blob mockBlob1 = context.mock(Blob.class, "mockBlob1"); - - final PayloadBlobBuilder mockBuilder1 = context.mock(PayloadBlobBuilder.class, "mockBuilder1"); - - final InputStream is = new ByteArrayInputStream(new byte[0]); - - final String fileName = "file.name"; - - context.checking(new Expectations() { - { - oneOf(mockBlobStoreContext).getBlobStore(); - will(returnValue(mockBlobStore)); - - oneOf(mockBlobStore).createContainerInLocation(null, bucket); - will(returnValue(true)); - - oneOf(mockBlobStore).blobBuilder(jobStorageBaseKey + "/" + fileName); - will(returnValue(mockBuilder1)); - - allowing(mockBuilder1).contentLength(1L); - will(returnValue(mockBuilder1)); - allowing(mockBuilder1).payload(with(is)); - will(returnValue(mockBuilder1)); - oneOf(mockBuilder1).build(); - will(returnValue(mockBlob1)); - - oneOf(mockBlobStore).putBlob(bucket, mockBlob1); - oneOf(mockBlobStoreContext).close(); - } - }); - - service.uploadJobFile(job, fileName, is); - } - - /** - * Tests that requests for deleting files successfully call all dependencies - * @throws PortalServiceException - */ - @Test - public void testDeleteJobFiles() throws PortalServiceException { - final BlobStore mockBlobStore = context.mock(BlobStore.class); - - context.checking(new Expectations() { - { - allowing(mockBlobStoreContext).getBlobStore(); - will(returnValue(mockBlobStore)); - oneOf(mockBlobStore).blobExists(bucket, jobStorageBaseKey); - will(returnValue(true)); - oneOf(mockBlobStore).deleteDirectory(bucket, jobStorageBaseKey); - oneOf(mockBlobStoreContext).close(); - } - }); - - service.deleteJobFiles(job); - } - - /** - * Tests that no exceptions occur during base key generation edge cases - */ - @Test - public void testBaseKeyGeneration() { - CloudJob emptyJob = new CloudJob(null); - - String emptyJobBaseKey = service.generateBaseKey(emptyJob); - Assert.assertNotNull(emptyJobBaseKey); - Assert.assertFalse(emptyJobBaseKey.isEmpty()); - - CloudJob nonEmptyJob = new CloudJob(42); - String nonEmptyJobBaseKey = service.generateBaseKey(nonEmptyJob); - Assert.assertNotNull(nonEmptyJobBaseKey); - Assert.assertFalse(nonEmptyJobBaseKey.isEmpty()); - - //Base keys should differ based soley on ID - Assert.assertFalse(nonEmptyJobBaseKey.equals(emptyJobBaseKey)); - } - - /** - * Tests that generateBaseKey doesn't generate keys that are a substring of eachother. - * - * Cloud storage treats files as being in a 'directory' based solely on its prefix. If you have the following files: - * - * job5/aFile.txt job52/anotherFile.txt - * - * And searched for all files whose prefix begins 'job5' (i.e. to list all files in the job5 directory), you would get BOTH of the above files returned. We - * need to manage this edge case by ensuring our prefixes don't overlap like that. - */ - @Test - public void testBaseKeyNoSubstrings() { - CloudJob jobBase = new CloudJob(Integer.valueOf(5)); - CloudJob[] jobsToTest = new CloudJob[] { - new CloudJob(Integer.valueOf(50)), - new CloudJob(Integer.valueOf(52)), - new CloudJob(Integer.valueOf(500)), - new CloudJob(Integer.valueOf(500000000)), - }; - - for (int i = 0; i < jobsToTest.length; i++) { - String base = service.generateBaseKey(jobBase); - String test = service.generateBaseKey(jobsToTest[i]); - - Assert.assertFalse(base.startsWith(test)); - Assert.assertFalse(test.startsWith(base)); - } - } - - @Test(expected=PortalServiceException.class) - public void testStsRequired() throws PortalServiceException { - CloudStorageServiceJClouds stsService = new CloudStorageServiceJClouds(); - stsService.setStsRequirement(STSRequirement.Mandatory); - stsService.setEndpoint("https://keystone.rc.nectar.org.au:5000/v3"); - stsService.setAccessKey("Project:User"); - stsService.getBlobStoreContext(null, null); - } - -} diff --git a/src/test/java/org/auscope/portal/core/services/cloud/TestFileStagingService.java b/src/test/java/org/auscope/portal/core/services/cloud/TestFileStagingService.java deleted file mode 100644 index 9626d17d0..000000000 --- a/src/test/java/org/auscope/portal/core/services/cloud/TestFileStagingService.java +++ /dev/null @@ -1,546 +0,0 @@ -package org.auscope.portal.core.services.cloud; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -import jakarta.servlet.http.HttpServletResponse; - -import org.apache.commons.io.IOUtils; -import org.auscope.portal.core.cloud.CloudJob; -import org.auscope.portal.core.cloud.StagedFile; -import org.auscope.portal.core.cloud.StagingInformation; -import org.auscope.portal.core.services.PortalServiceException; -import org.auscope.portal.core.test.PortalTestClass; -import org.auscope.portal.core.test.jmock.ReadableServletOutputStream; -import org.auscope.portal.core.util.FileIOUtil; -import org.jmock.Expectations; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.springframework.web.multipart.MultipartFile; -import org.springframework.web.multipart.MultipartHttpServletRequest; - -/** - * Unit tests for JobFileService - * - * @author Josh Vote - * - */ -public class TestFileStagingService extends PortalTestClass { - - private static StagingInformation testStagingInfo; - private static int testCounter = 0; - - private FileStagingService service; - private CloudJob job; - - /** - * This sets up a temporary directory in the target directory for the JobFileService to utilise as a staging area - */ - @BeforeClass - public static void setup() { - testStagingInfo = new StagingInformation(String.format( - "target%1$sTestJobFileService-%2$s%1$s", - File.separator, new Date().getTime())); - - File dir = new File(testStagingInfo.getStageInDirectory()); - Assert.assertTrue("Failed setting up staging directory", dir.mkdirs()); - } - - /** - * This tears down the staging area used by the tests - */ - @AfterClass - public static void tearDown() { - File dir = new File(testStagingInfo.getStageInDirectory()); - FileIOUtil.deleteFilesRecursive(dir); - - //Ensure cleanup succeeded - Assert.assertFalse(dir.exists()); - } - - /** - * Creates a fresh job object for each unit test (with a unique fileStorageID). - */ - @Before - public void setupJobObj() { - job = new CloudJob(testCounter++); - - service = new FileStagingService(testStagingInfo); - } - - /** - * Asserts that the shared staging directory (not the job staging directory) still exists after a unit test is run - */ - @After - public void ensureStagingDirectoryPreserved() { - File dir = new File(testStagingInfo.getStageInDirectory()); - Assert.assertTrue(dir.exists()); - Assert.assertTrue(dir.isDirectory()); - } - - /** - * Tests for the pathConcat utility method - */ - @Test - public void testPathConcat() { - Assert.assertEquals("p1" + File.separator + "p2", FileStagingService.pathConcat("p1", "p2")); - Assert.assertEquals("p1" + File.separator + "p2", FileStagingService.pathConcat("p1" + File.separator, "p2")); - Assert.assertEquals("p1" + File.separator + "p2", FileStagingService.pathConcat("p1", File.separator + "p2")); - Assert.assertEquals("p1" + File.separator + "p2", - FileStagingService.pathConcat("p1" + File.separator, File.separator + "p2")); - } - - /** - * Tests the existence/nonexistence of job's stage in directory - * - * @param job - * @param exists - */ - private static void assertStagedDirectory(CloudJob job, boolean exists) { - File stageInDir = new File(FileStagingService.pathConcat(testStagingInfo.getStageInDirectory(), - FileStagingService.getBaseFolderForJob(job))); - Assert.assertEquals(exists, stageInDir.exists()); - if (exists) { - Assert.assertEquals(true, stageInDir.isDirectory()); - } - } - - /** - * Tests the existence/nonexistence of job's stage in file - * - * @param job - * @param fileName - * @param exists - * @throws IOException - */ - private static void assertStagedFile(CloudJob job, String fileName, boolean exists) throws IOException { - assertStagedFile(job, fileName, exists, null); - } - - /** - * Tests the existence/nonexistence of job's stage in file as well as its contents - * - * @param job - * @param exists - * @param expectedData - * if not null and file exists, file data will be tested against this value - * @throws IOException - */ - private static void assertStagedFile(CloudJob job, String fileName, boolean exists, byte[] expectedData) throws IOException { - String stageInDir = FileStagingService.pathConcat(testStagingInfo.getStageInDirectory(), - FileStagingService.getBaseFolderForJob(job)); - File stageInFile = new File(FileStagingService.pathConcat(stageInDir, fileName)); - - Assert.assertEquals(exists, stageInFile.exists()); - if (exists) { - Assert.assertEquals(true, stageInFile.isFile()); - - //Test file contents - if (expectedData != null) { - try (FileInputStream fis = new FileInputStream(stageInFile)) { - byte[] actualData = IOUtils.toByteArray(fis); - Assert.assertArrayEquals(expectedData, actualData); - } - } - - } - } - - /** - * Tests that creating/deleting an empty job staging area works - * @throws PortalServiceException - */ - @Test - public void testEmptyStartupTeardown() throws PortalServiceException { - service.generateStageInDirectory(job); - - assertStagedDirectory(job, true); - - service.deleteStageInDirectory(job); - - assertStagedDirectory(job, false); - } - - /** - * Tests that creating and listing files in a job staging area works - * @throws PortalServiceException - * - * @throws IOException - */ - @Test - public void testFileCreationAndListing() throws PortalServiceException, IOException { - service.generateStageInDirectory(job); - - final byte[] file1Data = new byte[] {1, 2, 3}; - final byte[] file2Data = new byte[] {4, 3, 1}; - - OutputStream file1 = service.writeFile(job, "testFile1"); - OutputStream file2 = service.writeFile(job, "testFile2"); - - file1.write(file1Data); - file2.write(file2Data); - file1.close(); - file2.close(); - - assertStagedDirectory(job, true); - assertStagedFile(job, "testFile1", true, file1Data); - assertStagedFile(job, "testFile2", true, file2Data); - - //Ensure that listing returns all the files (in no particular order) - StagedFile[] expectedFiles = new StagedFile[] {new StagedFile(job, "testFile1", null), - new StagedFile(job, "testFile2", null)}; - StagedFile[] listedFiles = service.listStageInDirectoryFiles(job); - Assert.assertNotNull(listedFiles); - Assert.assertEquals(expectedFiles.length, listedFiles.length); - for (StagedFile expectedFile : expectedFiles) { - boolean foundFile = false; - for (StagedFile listedFile : listedFiles) { - if (listedFile.equals(expectedFile)) { - Assert.assertNotNull("File reference in StagedFile not set!", listedFile.getFile()); - Assert.assertEquals(job, listedFile.getOwner()); - foundFile = true; - break; - } - } - - Assert.assertTrue(String.format("File '%1$s' not listed", expectedFile), foundFile); - } - - service.deleteStageInDirectory(job); - assertStagedDirectory(job, false); - assertStagedFile(job, "testFile1", false); - assertStagedFile(job, "testFile2", false); - } - - /** - * Tests that using relative paths in a filename will generate exceptions - * @throws PortalServiceException - * - * @throws IOException - */ - @Test - public void testBadFilenames() throws PortalServiceException { - service.generateStageInDirectory(job); - - //Should either return null or throw exception - try (OutputStream file = service.writeFile(job, FileStagingService.pathConcat("..", "testFile1"))) { - Assert.assertNull(file); - } catch (Exception ex) { - // empty - } - try (OutputStream file = service.writeFile(job, "testFile1" + File.pathSeparator + "testFile2")) { - Assert.assertNull(file); - } catch (Exception ex) { - // empty - } - try (InputStream file = service.readFile(job, FileStagingService.pathConcat("..", "testFile1"))) { - Assert.assertNull(file); - } catch (Exception ex) { - // empty - } - try (InputStream file = service.readFile(job, "testFile1" + File.pathSeparator + "testFile2")) { - Assert.assertNull(file); - } catch (Exception ex) { - // empty - } - - service.deleteStageInDirectory(job); - assertStagedDirectory(job, false); - } - - /** - * Tests that file uploads can be handled - * @throws IOException - * @throws PortalServiceException - */ - @Test - public void testFileUpload() throws IOException, PortalServiceException { - final MultipartHttpServletRequest request = context.mock(MultipartHttpServletRequest.class); - final MultipartFile file = context.mock(MultipartFile.class); - final String fileName = "myFileName"; - - context.checking(new Expectations() { - { - oneOf(request).getFile("file"); - will(returnValue(file)); - oneOf(file).getOriginalFilename(); - will(returnValue(fileName)); - oneOf(file).transferTo(with(aFileWithName(fileName))); - } - }); - - service.generateStageInDirectory(job); - - //"Upload" the file and check it gets created - StagedFile newlyStagedFile = service.handleFileUpload(job, request); - Assert.assertNotNull(newlyStagedFile); - Assert.assertNotNull("File reference not set!", newlyStagedFile.getFile()); - Assert.assertEquals(job, newlyStagedFile.getOwner()); - Assert.assertEquals(fileName, newlyStagedFile.getName()); - - service.deleteStageInDirectory(job); - assertStagedDirectory(job, false); - } - - /** - * Tests that multi-file uploads can be handled - * @throws IOException - * @throws PortalServiceException - */ - @Test - public void testMultiFileUpload() throws IOException, PortalServiceException { - final MultipartHttpServletRequest request = context.mock(MultipartHttpServletRequest.class); - final MultipartFile file = context.mock(MultipartFile.class); - final String fileName = "myFileName"; - final ArrayList files = new ArrayList<>(); - files.add(file); - files.add(file); - - context.checking(new Expectations() { - { - oneOf(request).getFiles("file"); - will(returnValue(files)); - exactly(2).of(file).getOriginalFilename(); - will(returnValue(fileName)); - exactly(2).of(file).transferTo(with(aFileWithName(fileName))); - } - }); - - service.generateStageInDirectory(job); - - //"Upload" the file and check it gets created - List newlyStagedFiles = service.handleMultiFileUpload(job, request); - Assert.assertNotNull(newlyStagedFiles); - Assert.assertEquals(files.size(), newlyStagedFiles.size()); - Assert.assertEquals(job, newlyStagedFiles.get(0).getOwner()); - - service.deleteStageInDirectory(job); - assertStagedDirectory(job, false); - } - - /** - * Tests that file downloads are handled correctly - * @throws IOException - * @throws PortalServiceException - */ - @Test - public void testFileDownload() throws IOException, PortalServiceException { - try (final ReadableServletOutputStream outStream = new ReadableServletOutputStream()) { - final byte[] data = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 8, 5, 8, 9, 9, 9, 91, 1, 1 }; - final HttpServletResponse mockResponse = context.mock(HttpServletResponse.class); - final String fileName = "myFileName"; - - // Start by creating our file that we want to download - service.generateStageInDirectory(job); - OutputStream fos = service.writeFile(job, fileName); - fos.write(data); - fos.close(); - - context.checking(new Expectations() { - { - // This is so we can inject our own fake output stream so we - // can inspect the result - oneOf(mockResponse).getOutputStream(); - will(returnValue(outStream)); - oneOf(mockResponse).setContentType("application/octet-stream"); - allowing(mockResponse).setHeader("Content-Disposition", - "attachment; filename=\"" + fileName + "\""); - } - }); - - // 'Download' the file - service.handleFileDownload(job, fileName, mockResponse); - - // Inspect the data we downloaded - Assert.assertArrayEquals(data, outStream.getDataWritten()); - } - - service.deleteStageInDirectory(job); - assertStagedDirectory(job, false); - } - - /** - * Tests that creating and renaming files in a job staging area works - * @throws PortalServiceException - * - * @throws IOException - */ - @Test - public void testFileRenaming() throws PortalServiceException, IOException { - service.generateStageInDirectory(job); - - final byte[] file1Data = new byte[] {1, 2, 3}; - - OutputStream file1 = service.writeFile(job, "testFile1"); - file1.write(file1Data); - file1.close(); - - service.renameStageInFile(job, "testFile1", "renamedTestFile1"); - - assertStagedDirectory(job, true); - assertStagedFile(job, "testFile1", false); - assertStagedFile(job, "renamedTestFile1", true, file1Data); - - //Ensure that listing returns all the files (in no particular order) - StagedFile[] expectedFiles = new StagedFile[] {new StagedFile(job, "renamedTestFile1", null)}; - StagedFile[] listedFiles = service.listStageInDirectoryFiles(job); - Assert.assertNotNull(listedFiles); - Assert.assertEquals(expectedFiles.length, listedFiles.length); - for (StagedFile expectedFile : expectedFiles) { - boolean foundFile = false; - for (StagedFile listedFile : listedFiles) { - if (listedFile.equals(expectedFile)) { - Assert.assertNotNull("File reference in StagedFile not set!", listedFile.getFile()); - Assert.assertEquals(job, listedFile.getOwner()); - foundFile = true; - break; - } - } - - Assert.assertTrue(String.format("File '%1$s' not listed", expectedFile), foundFile); - } - - service.deleteStageInDirectory(job); - assertStagedDirectory(job, false); - assertStagedFile(job, "testFile1", false); - assertStagedFile(job, "renamedTestFile1", false); - } - - /** - * Tests that creating and renaming files in a job staging area works when the target file already exists - * - * @throws IOException - * @throws PortalServiceException - */ - @Test - public void testFileRenamingOverwrite() throws IOException, PortalServiceException { - service.generateStageInDirectory(job); - - final byte[] file1Data = new byte[] {1, 2, 3}; - final byte[] file2Data = new byte[] {4, 3, 1}; - - OutputStream file1 = service.writeFile(job, "testFile1"); - OutputStream file2 = service.writeFile(job, "testFile2"); - - file1.write(file1Data); - file2.write(file2Data); - file1.close(); - file2.close(); - - assertStagedDirectory(job, true); - assertStagedFile(job, "testFile1", true, file1Data); - assertStagedFile(job, "testFile2", true, file2Data); - - Assert.assertTrue(service.renameStageInFile(job, "testFile1", "testFile2")); - - assertStagedFile(job, "testFile1", false); - assertStagedFile(job, "testFile2", true, file1Data); - - Assert.assertFalse(service.renameStageInFile(job, "testFile1", "testFile2")); - - assertStagedFile(job, "testFile1", false); - assertStagedFile(job, "testFile2", true, file1Data); - - service.deleteStageInDirectory(job); - assertStagedDirectory(job, false); - assertStagedFile(job, "testFile1", false); - assertStagedFile(job, "testFile2", false); - } - - /** - * Tests that renaming file to itself does nothing - * - * @throws IOException - * @throws PortalServiceException - */ - @Test - public void testFileRenamingSameFile() throws IOException, PortalServiceException { - service.generateStageInDirectory(job); - - final byte[] file1Data = new byte[] {1, 2, 3}; - - OutputStream file1 = service.writeFile(job, "testFile1"); - - file1.write(file1Data); - file1.close(); - - assertStagedDirectory(job, true); - assertStagedFile(job, "testFile1", true, file1Data); - - Assert.assertTrue(service.renameStageInFile(job, "testFile1", "testFile1")); - - assertStagedFile(job, "testFile1", true, file1Data); - - service.deleteStageInDirectory(job); - assertStagedDirectory(job, false); - assertStagedFile(job, "testFile1", false); - } - - /** - * Tests that creating and renaming files in a job staging area works when the target file already exists - * @throws PortalServiceException - * - * @throws IOException - */ - @Test - public void testFileExists() throws PortalServiceException, IOException { - service.generateStageInDirectory(job); - - final byte[] file1Data = new byte[] {1, 2, 3}; - final byte[] file2Data = new byte[] {4, 3, 1}; - - OutputStream file1 = service.writeFile(job, "testFile1"); - OutputStream file2 = service.writeFile(job, "testFile2"); - - file1.write(file1Data); - file2.write(file2Data); - file1.close(); - file2.close(); - - assertStagedDirectory(job, true); - assertStagedFile(job, "testFile1", true, file1Data); - assertStagedFile(job, "testFile2", true, file2Data); - - Assert.assertTrue(service.stageInFileExists(job, "testFile1")); - Assert.assertTrue(service.stageInFileExists(job, "testFile2")); - Assert.assertFalse(service.stageInFileExists(job, "fileDNE")); - - service.deleteStageInDirectory(job); - assertStagedDirectory(job, false); - assertStagedFile(job, "testFile1", false); - assertStagedFile(job, "testFile2", false); - - Assert.assertFalse(service.stageInFileExists(job, "testFile1")); - Assert.assertFalse(service.stageInFileExists(job, "testFile2")); - Assert.assertFalse(service.stageInFileExists(job, "fileDNE")); - } - - /** - * Tests that creating a job staging works and that it can be detected - * @throws PortalServiceException - * - * @throws IOException - */ - @Test - public void testStagingDirectoryExists() throws PortalServiceException { - service.generateStageInDirectory(job); - - assertStagedDirectory(job, true); - Assert.assertTrue(service.stageInDirectoryExists(job)); - service.deleteStageInDirectory(job); - - assertStagedDirectory(job, false); - Assert.assertFalse(service.stageInDirectoryExists(job)); - } - -} diff --git a/src/test/java/org/auscope/portal/core/services/cloud/monitor/TestJobStatusException.java b/src/test/java/org/auscope/portal/core/services/cloud/monitor/TestJobStatusException.java deleted file mode 100644 index 6cdafd0d3..000000000 --- a/src/test/java/org/auscope/portal/core/services/cloud/monitor/TestJobStatusException.java +++ /dev/null @@ -1,64 +0,0 @@ -package org.auscope.portal.core.services.cloud.monitor; - -import java.io.FileNotFoundException; -import java.util.Arrays; - -import org.auscope.portal.core.cloud.CloudJob; -import org.auscope.portal.core.test.PortalTestClass; -import org.junit.Assert; -import org.junit.Test; - -public class TestJobStatusException extends PortalTestClass { - - /** - * Ensures that getMessage correctly concatenates messages - */ - @Test - public void testGetMessage() { - final Throwable ex1 = new Exception("msg1"); - final Throwable ex2 = new FileNotFoundException("msg2"); - final Throwable ex3 = new NullPointerException("msg3"); - - final CloudJob cj1 = new CloudJob(1); - final CloudJob cj2 = new CloudJob(2); - final CloudJob cj3 = new CloudJob(3); - - JobStatusException jse = new JobStatusException(Arrays.asList(ex1, ex2, ex3), Arrays.asList(cj1, cj2, cj3)); - - String msg = jse.getMessage(); - Assert.assertNotNull(msg); - Assert.assertTrue(msg.contains("Exception")); - Assert.assertTrue(msg.contains("FileNotFoundException")); - Assert.assertTrue(msg.contains("NullPointerException")); - - Assert.assertTrue(msg.contains("msg1")); - Assert.assertTrue(msg.contains("msg2")); - Assert.assertTrue(msg.contains("msg3")); - } - - /** - * Ensures that toString correctly concatenates messages - */ - @Test - public void testToString() { - final Throwable ex1 = new Exception("msg1"); - final Throwable ex2 = new FileNotFoundException("msg2"); - final Throwable ex3 = new NullPointerException("msg3"); - - final CloudJob cj1 = new CloudJob(1); - final CloudJob cj2 = new CloudJob(2); - final CloudJob cj3 = new CloudJob(3); - - JobStatusException jse = new JobStatusException(Arrays.asList(ex1, ex2, ex3), Arrays.asList(cj1, cj2, cj3)); - - String s = jse.toString(); - Assert.assertNotNull(s); - Assert.assertTrue(s.contains("Exception")); - Assert.assertTrue(s.contains("FileNotFoundException")); - Assert.assertTrue(s.contains("NullPointerException")); - - Assert.assertTrue(s.contains("msg1")); - Assert.assertTrue(s.contains("msg2")); - Assert.assertTrue(s.contains("msg3")); - } -} diff --git a/src/test/java/org/auscope/portal/core/services/cloud/monitor/TestJobStatusMonitor.java b/src/test/java/org/auscope/portal/core/services/cloud/monitor/TestJobStatusMonitor.java deleted file mode 100644 index 93a377040..000000000 --- a/src/test/java/org/auscope/portal/core/services/cloud/monitor/TestJobStatusMonitor.java +++ /dev/null @@ -1,89 +0,0 @@ -package org.auscope.portal.core.services.cloud.monitor; - -import java.util.Arrays; -import java.util.List; - -import org.auscope.portal.core.cloud.CloudJob; -import org.auscope.portal.core.test.PortalTestClass; -import org.jmock.Expectations; -import org.junit.Before; -import org.junit.Test; - -public class TestJobStatusMonitor extends PortalTestClass { - private JobStatusMonitor monitor; - private JobStatusReader mockJobStatusReader; - private JobStatusChangeListener[] mockJobStatusChangeListeners; - - @Before - public void init() { - //Mock objects required for the unit tests - mockJobStatusReader = context.mock(JobStatusReader.class); - mockJobStatusChangeListeners = new JobStatusChangeListener[] {context.mock(JobStatusChangeListener.class)}; - - //Component under test - monitor = new JobStatusMonitor(mockJobStatusReader, mockJobStatusChangeListeners); - } - - /** - * Tests that the execution of VGLJobStatusMonitor task run as expected. - * @throws JobStatusException - */ - @Test - public void testExecuteInternal() throws JobStatusException { - final CloudJob job1 = new CloudJob(1); - job1.setStatus("s1"); - - final CloudJob job2 = new CloudJob(2); - job2.setStatus("s2"); - - final List jobs = Arrays.asList(job1, job2); - - final String job1NewStat = "ns1"; //change - final String job2NewStat = job2.getStatus(); //no change - - context.checking(new Expectations() { - { - - oneOf(mockJobStatusReader).getJobStatus(job1); - will(returnValue(job1NewStat)); - oneOf(mockJobStatusReader).getJobStatus(job2); - will(returnValue(job2NewStat)); - - oneOf(mockJobStatusChangeListeners[0]).handleStatusChange(job1, job1NewStat, job1.getStatus()); - } - }); - - monitor.statusUpdate(jobs); - } - - /** - * Tests that exception caused by job status change handler won't impact the status change handling for other job(s) being processed. - * @throws JobStatusException - */ - @Test(expected = JobStatusException.class) - public void testExecuteInternal_Exception() throws JobStatusException { - final CloudJob job1 = new CloudJob(1); - job1.setStatus("s1"); - - final CloudJob job2 = new CloudJob(2); - job2.setStatus("s2"); - - final List jobs = Arrays.asList(job1, job2); - - final String job2NewStat = "ns2"; //change - - context.checking(new Expectations() { - { - - oneOf(mockJobStatusReader).getJobStatus(job1); - will(throwException(new Exception())); - oneOf(mockJobStatusReader).getJobStatus(job2); - will(returnValue(job2NewStat)); - - oneOf(mockJobStatusChangeListeners[0]).handleStatusChange(job2, job2NewStat, job2.getStatus()); - } - }); - - monitor.statusUpdate(jobs); - } -}