diff --git a/server/pom.xml b/server/pom.xml index 2982daf..183b578 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -404,6 +404,19 @@ jetty-maven-plugin 8.1.16.v20140903 + + org.apache.maven.plugins + maven-source-plugin + 2.4 + + + attach-sources + + jar + + + + diff --git a/server/src/main/java/org/diskproject/server/adapters/AirFlowAdapter.java b/server/src/main/java/org/diskproject/server/adapters/AirFlowAdapter.java index be00745..20b3a71 100644 --- a/server/src/main/java/org/diskproject/server/adapters/AirFlowAdapter.java +++ b/server/src/main/java/org/diskproject/server/adapters/AirFlowAdapter.java @@ -105,7 +105,7 @@ public Map getWorkflowInputs(String id) { } @Override - public String runWorkflow(String wfId, List vBindings, Map inputVariables) { + public String runWorkflow(String wfId, String workflowName, List vBindings, Map inputVariables) { // Auto-generated method stub return null; } @@ -129,8 +129,21 @@ public Map getRunVariableBindings(String runId) { } @Override - public String getDataUri(String id) { + public String getDataUri(String dataId, String runId) { // Auto-generated method stub return null; } + + @Override + public String addData(String url, String workSpaceId, String name, String dType) throws Exception { + // TODO Auto-generated method stub + return null; + } + + + @Override + public String duplicateWorkflow(String workflowId, String newName) { + // TODO Auto-generated method stub + return null; + } } diff --git a/server/src/main/java/org/diskproject/server/adapters/Reana/ApiClient.java b/server/src/main/java/org/diskproject/server/adapters/Reana/ApiClient.java new file mode 100644 index 0000000..94ae6f9 --- /dev/null +++ b/server/src/main/java/org/diskproject/server/adapters/Reana/ApiClient.java @@ -0,0 +1,389 @@ +package org.diskproject.server.adapters.Reana; + +import java.io.File; +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpHeaders; +import org.apache.http.NameValuePair; +import org.apache.http.client.entity.EntityBuilder; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpRequestBase; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.util.EntityUtils; +import org.diskproject.server.adapters.Reana.ApiSchema.ReanaSpecification; +import org.diskproject.server.adapters.Reana.ApiSchema.ReanaWorkflow; +import org.diskproject.server.adapters.Reana.ApiSchema.ResponseGetSpecification; +import org.diskproject.server.adapters.Reana.ApiSchema.ResponseRunStatus; + +import com.google.gson.Gson; + +public class ApiClient { + public String url; + String token; + CloseableHttpClient httpClient; + String API_WORKFLOW_URL = "%s/api/workflows"; + String API_WORKFLOW_STATUS_URL = "%s/api/workflows/%s/status"; + String API_WORKFLOW_START = "%s/api/workflows/%s/start"; + String API_WORKFLOW_ADD_FILE = "%s/api/workflows/%s/workspace"; + String API_WORKFLOW_SPECIFICATION = "%s/api/workflows/%s/specification"; + public String API_WORKFLOW_DOWNLOAD_FILE = "%s/api/workflows/%s/workspace/%s"; + + String API_LAUNCH_URL = "/api/launch"; + + public ApiClient(String url, String token, String username) { + this.url = url; + this.token = token; + httpClient = HttpClients.custom() + .build(); + + } + + public String createWorkflow(ReanaSpecification specification, String name) throws Exception { + String workflowSpecification = new Gson().toJson(specification); + String workflowName = name; + String workflowId = null; + String requestUrl = String.format(API_WORKFLOW_URL, this.url); + List requestParameters = new ArrayList(); + requestParameters.add(new BasicNameValuePair("workflow_name", workflowName)); + try { + String responseString = this.performHTTPPost(requestUrl, workflowSpecification, requestParameters); + Gson gson = new Gson(); + ResponseCreateWorkflow responseCreateWorkflow = gson.fromJson(responseString, ResponseCreateWorkflow.class); + workflowId = responseCreateWorkflow.workflow_id; + return workflowId; + } catch (Exception e) { + System.out.println("Error creating workflow: " + e.getMessage()); + throw e; + } + } + + /** + * Get the workflow specification for a workflow + * + * @param id The id of the workflow (not url) + * @return The workflow specification + * @throws Exception + */ + public ResponseGetSpecification getSpecification(String id) throws Exception { + String requestUrl = String.format(API_WORKFLOW_SPECIFICATION, this.url, id); + String requestAccept = "application/json"; + try { + String response = this.performHTTPGet(requestUrl, requestAccept); + Gson gson = new Gson(); + return gson.fromJson(response, ResponseGetSpecification.class); + } catch (Exception e) { + System.err.println("Could not get the workflow specification for workflow " + id); + System.err.println(e.getMessage()); + throw e; + } + } + + /** + * List all workflows filter by status (created) + * + * @return List of workflows + * @throws Exception + */ + public ResponseListWorkflow getWorkflows() throws Exception { + String requestUrl = String.format(API_WORKFLOW_URL, this.url); + String requestAccept = "application/json"; + List requestParameters = new ArrayList(); + requestParameters.add(new BasicNameValuePair("status", "created")); + try { + String response = this.performHTTPGet(requestUrl, requestAccept, requestParameters); + Gson gson = new Gson(); + return gson.fromJson(response, ResponseListWorkflow.class); + } catch (Exception e) { + System.err.println("Could not list methods"); + System.err.println(e.getMessage()); + throw e; + } + } + + public ResponseRunStatus getRunStatus(String id) throws Exception { + String requestUrl = String.format(API_WORKFLOW_STATUS_URL, this.url, id); + String requestAccept = "application/json"; + try { + String response = this.performHTTPGet(requestUrl, requestAccept); + Gson gson = new Gson(); + return gson.fromJson(response, ResponseRunStatus.class); + } catch (Exception e) { + System.err.println("Unable to get status"); + System.err.println(e.getMessage()); + throw e; + } + } + + public byte[] downloadData(String url) throws Exception { + String requestAccept = "application/octet-stream"; + try { + return this.performHTTPGetByte(url, requestAccept); + } catch (Exception e) { + System.err.println("Unable to fetch file"); + System.err.println(e.getMessage()); + throw e; + } + } + public void fetchFile(String workflowId, String filename) throws Exception { + String requestUrl = String.format(API_WORKFLOW_DOWNLOAD_FILE, this.url, workflowId, filename); + String requestAccept = "application/octet-stream"; + try { + this.performHTTPGet(requestUrl, requestAccept); + } catch (Exception e) { + System.err.println("Unable to fetch file"); + System.err.println(e.getMessage()); + throw e; + } + } + public Object startWorkflow(String workflowId, Map inputParameters, + Map operationParameters) throws Exception { + String requestUrl = String.format(API_WORKFLOW_START, this.url, workflowId); + String requestAccept = "application/json"; + String requestContentType = "application/json"; + RequestApiStart requestApiStart = new RequestApiStart(inputParameters, + operationParameters); + String requestBody = new Gson().toJson(requestApiStart); + try { + return this.performHTTPPost(requestUrl, requestBody, requestAccept, requestContentType); + } catch (Exception e) { + System.err.println("Could start the workflow"); + System.err.println(e.getMessage()); + throw new RuntimeException(e); + } + } + + public String addFileWorkspace(String workflowId, String localFilePath, String remoteFilePath) throws Exception { + File file = new File(localFilePath); + String fileRemoteUrl = String.format(API_WORKFLOW_ADD_FILE, this.url, workflowId); + String requestUrl = fileRemoteUrl + "?file_name=" + remoteFilePath; + try { + this.performHTTPPostFile(requestUrl, file); + } catch (Exception e) { + System.err.println("Could add file to workspace"); + System.err.println(e.getMessage()); + throw e; + } + return fileRemoteUrl; + } + + public String performHTTPGet(String url, String accept, List parameters) throws Exception { + /** + * Perform an HTTP GET request + */ + HttpGet httpGet = new HttpGet(url); + parameters.add(new BasicNameValuePair("access_token", this.token)); + httpGet.setHeader(HttpHeaders.ACCEPT, accept); + URI uri = new URIBuilder(httpGet.getURI()) + .addParameters(parameters) + .build(); + ((HttpRequestBase) httpGet).setURI(uri); + CloseableHttpResponse response = httpClient.execute(httpGet); + if (response.getStatusLine().getStatusCode() != 200) { + throw new Exception("HTTP GET request failed: " + response.getStatusLine().getStatusCode()); + } + HttpEntity entity = response.getEntity(); + String responseString = EntityUtils.toString(entity); + return responseString; + } + + public String performHTTPGet(String url, String accept) throws Exception { + /** + * Perform an HTTP GET request + */ + HttpGet httpGet = new HttpGet(url); + ArrayList parameters = new ArrayList(); + parameters.add(new BasicNameValuePair("access_token", this.token)); + httpGet.setHeader(HttpHeaders.ACCEPT, accept); + URI uri = new URIBuilder(httpGet.getURI()) + .addParameters(parameters) + .build(); + ((HttpRequestBase) httpGet).setURI(uri); + CloseableHttpResponse response = httpClient.execute(httpGet); + if (response.getStatusLine().getStatusCode() != 200) { + throw new Exception("HTTP GET request failed: " + response.getStatusLine().getStatusCode()); + } + HttpEntity entity = response.getEntity(); + String responseString = EntityUtils.toString(entity); + return responseString; + } + + public byte[] performHTTPGetByte(String url, String accept) throws Exception { + /** + * Perform an HTTP GET request + */ + HttpGet httpGet = new HttpGet(url); + ArrayList parameters = new ArrayList(); + parameters.add(new BasicNameValuePair("access_token", this.token)); + httpGet.setHeader(HttpHeaders.ACCEPT, accept); + URI uri = new URIBuilder(httpGet.getURI()) + .addParameters(parameters) + .build(); + ((HttpRequestBase) httpGet).setURI(uri); + CloseableHttpResponse response = httpClient.execute(httpGet); + if (response.getStatusLine().getStatusCode() != 200) { + throw new Exception("HTTP GET request failed: " + response.getStatusLine().getStatusCode()); + } + HttpEntity entity = response.getEntity(); + return EntityUtils.toByteArray(entity); + } + /** + * Perform an HTTP POST request + * + * @param url + * @param data + * @return + */ + public String performHTTPPost(String url, String data, String contentType, String acceptContent) throws Exception { + HttpPost httpPost = new HttpPost(url); + ArrayList parameters = new ArrayList(); + parameters.add(new BasicNameValuePair("access_token", this.token)); + URI uri = new URIBuilder(httpPost.getURI()) + .addParameters(parameters) + .build(); + ((HttpRequestBase) httpPost).setURI(uri); + httpPost.setEntity(new StringEntity(data)); + httpPost.addHeader("Content-type", contentType); + httpPost.addHeader("Accept", acceptContent); + CloseableHttpResponse httpResponse = httpClient.execute(httpPost); + try { + HttpEntity responseEntity = httpResponse.getEntity(); + String strResponse = EntityUtils.toString(responseEntity); + EntityUtils.consume(responseEntity); + httpResponse.close(); + return strResponse; + } finally { + httpResponse.close(); + } + } + + private String performHTTPPost(String url, String data, List parameters) throws Exception { + HttpPost httpPost = new HttpPost(url); + httpPost.setEntity(new StringEntity(data)); + httpPost.addHeader("Content-type", "application/json"); + httpPost.addHeader("Accept", "application/json"); + parameters.add(new BasicNameValuePair("access_token", this.token)); + URI uri = new URIBuilder(httpPost.getURI()) + .addParameters(parameters) + .build(); + ((HttpRequestBase) httpPost).setURI(uri); + CloseableHttpResponse httpResponse = this.httpClient.execute(httpPost); + if (httpResponse.getStatusLine().getStatusCode() != 201 + && httpResponse.getStatusLine().getStatusCode() != 200) { + throw new Exception("HTTP GET request failed: " + httpResponse.getStatusLine().getStatusCode()); + } + try { + HttpEntity responseEntity = httpResponse.getEntity(); + String strResponse = EntityUtils.toString(responseEntity); + EntityUtils.consume(responseEntity); + httpResponse.close(); + + return strResponse; + } finally { + httpResponse.close(); + } + } + + private String performHTTPPostFile(String url, File file) throws Exception { + HttpPost httpPost = new HttpPost(url); + List parameters = new ArrayList<>(); + parameters.add(new BasicNameValuePair("access_token", this.token)); + URI uri = new URIBuilder(httpPost.getURI()) + .addParameters(parameters) + .build(); + ((HttpRequestBase) httpPost).setURI(uri); + httpPost.setHeader("Accept", "application/json"); + httpPost.setHeader("Content-type", "application/octet-stream"); + EntityBuilder builder = EntityBuilder.create(); + builder.setFile(file); + HttpEntity entity = builder.build(); + httpPost.setEntity(entity); + CloseableHttpResponse httpResponse = this.httpClient.execute(httpPost); + + HttpEntity responseEntity = httpResponse.getEntity(); + String strResponse = EntityUtils.toString(responseEntity); + if (httpResponse.getStatusLine().getStatusCode() != 201 + && httpResponse.getStatusLine().getStatusCode() != 200) { + throw new Exception("HTTP GET request failed: " + httpResponse.getStatusLine().getStatusCode()); + } + try { + EntityUtils.consume(responseEntity); + httpResponse.close(); + return strResponse; + } finally { + httpResponse.close(); + } + } + + public class RequestApiStart { + public Map input_parameters; + public Map operational_parameters; + + public RequestApiStart(Map input_parameters, + Map operational_parameters) { + this.input_parameters = input_parameters; + this.operational_parameters = operational_parameters; + } + } + + public class RequestApiState { + public String name; + public String workflowSpecification; + + public RequestApiState(String name, String workflowSpecification) { + this.name = name; + this.workflowSpecification = workflowSpecification; + } + } + + public class ResponseBasic { + public String message; + } + + public class ResponseCreateWorkflow { + public String message; + public String workflow_id; + public String workflow_name; + } + + public class ResponseListWorkflow { + public Boolean has_next; + public String has_prev; + public Integer page; + public Integer total; + public ReanaWorkflow[] items; + } + + public class ResponseGetWorkspace { + public Boolean has_next; + public String has_prev; + public Integer page; + public Integer total; + public Workspace[] items; + } + + public class Workspace { + String last_modified; + String name; + Object size; + } + + public class ResponseStartWorkflow { + public String message; + public String workflow_id; + public String workflow_name; + public Integer run_number; + public String status; + public String user; + } + +} diff --git a/server/src/main/java/org/diskproject/server/adapters/Reana/ApiSchema.java b/server/src/main/java/org/diskproject/server/adapters/Reana/ApiSchema.java new file mode 100644 index 0000000..6e6b39e --- /dev/null +++ b/server/src/main/java/org/diskproject/server/adapters/Reana/ApiSchema.java @@ -0,0 +1,228 @@ +package org.diskproject.server.adapters.Reana; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +public class ApiSchema { + public class ResponseGetSpecification { + public Map parameters; + public ReanaSpecification specification; + } + + public class ReanaSpecification { + public Inputs inputs; + public Outputs outputs; + String version; + Workflow workflow; + + public class Workflow { + public WorkflowSpecification specification; + public String type; + + public class WorkflowSpecification { + public Inputs inputs; + public Outputs outputs; + public String version; + public Workflow workflow; + public ArrayList steps; + } + + public class Step { + public ArrayList commands; + public String environment; + public String kubernetes_memory_limit; + public String name; + } + } + + public class Outputs { + ArrayList files = new ArrayList(); + + public ArrayList getFiles() { + return files; + } + + // Getter Methods + + // Setter Methods + + } + + public class Inputs { + ArrayList directories = new ArrayList(); + ArrayList files = new ArrayList(); + Map parameters = new HashMap(); + + public Inputs(ArrayList directories, ArrayList files, Map parameters) { + this.directories = directories; + this.files = files; + this.parameters = parameters; + } + + public ArrayList getDirectories() { + return directories; + } + + public ArrayList getFiles() { + return files; + } + + public Map getParameters() { + return parameters; + } + + } + + public class Parameters { + private String input; + + // Getter Methods + + public String getInput() { + return input; + } + + // Setter Methods + + public void setInput(String input) { + this.input = input; + } + } + } + + public class RequestApiLaunch { + String url; + String name; + String specification; + String parameters; + + public RequestApiLaunch(String url, String name, String specification, String parameters) { + this.url = url; + this.name = name; + this.specification = specification; + this.parameters = parameters; + } + } + + public class ReanaWorkflow { + public String name; + public String created; + public String id; + public String launcher_url; + public Object progress; + public Object size; + public String status; + public String user; + + public ReanaWorkflow(String name, String created, String id, String launcher_url, Object progress, Object size, + String status, String user) { + this.name = name; + this.created = created; + this.id = id; + this.launcher_url = launcher_url; + this.progress = progress; + this.size = size; + this.status = status; + this.user = user; + } + } + + public class ResponseRunStatus { + public class Progress { + String current_command; + String current_step_name; + Object failed; + Object finished; + String run_finished_at; + String run_started_at; + Object running; + Object total; + + public String getCurrent_command() { + return current_command; + } + + public String getCurrent_step_name() { + return current_step_name; + } + + public Object getFailed() { + return failed; + } + + public Object getFinished() { + return finished; + } + + public String getRun_finished_at() { + return run_finished_at; + } + + public String getRun_started_at() { + return run_started_at; + } + + public Object getRunning() { + return running; + } + + public Object getTotal() { + return total; + } + } + + public String created; + public String id; + public String logs; + public String name; + public Progress progress; + public String status; + public String user; + + public String getCreated() { + return created; + } + + public String getId() { + return id; + } + + public String getLogs() { + return logs; + } + + public String getName() { + return name; + } + + public Object getProgress() { + return progress; + } + + public String getStatus() { + return status; + } + + public String getUser() { + return user; + } + } + + public class WorkflowInventory { + String url; + String name; + String specification; + String rawUrl; + String id; + + public WorkflowInventory(String url, String name, String specification, String rawUrl, String id) { + this.url = url; + this.name = name; + this.specification = specification; + this.rawUrl = rawUrl; + this.id = id; + } + + } +} diff --git a/server/src/main/java/org/diskproject/server/adapters/Reana/ReanaSpecification.java b/server/src/main/java/org/diskproject/server/adapters/Reana/ReanaSpecification.java new file mode 100644 index 0000000..2ddd95d --- /dev/null +++ b/server/src/main/java/org/diskproject/server/adapters/Reana/ReanaSpecification.java @@ -0,0 +1,115 @@ +// package org.diskproject.server.adapters.Reana; + +// import java.util.ArrayList; +// import java.util.HashMap; +// import java.util.Map; + +// public class ReanaSpecification { +// Map parameters = new HashMap(); +// Specification specification; + +// public void setSpecification(Specification specificationObject) { +// this.specification = specificationObject; +// } + +// public class Specification { +// Inputs inputs; +// Outputs outputs; +// private String version; +// Workflow workflow; + +// // Getter Methods + +// public Inputs getInputs() { +// return inputs; +// } + +// public Outputs getOutputs() { +// return outputs; +// } + +// public String getVersion() { +// return version; +// } + +// public Workflow getWorkflow() { +// return workflow; +// } + +// } + +// public class Workflow { +// private String file; +// private String type; + +// // Getter Methods + +// public String getFile() { +// return file; +// } + +// public String getType() { +// return type; +// } + +// // Setter Methods + +// public void setFile(String file) { +// this.file = file; +// } + +// public void setType(String type) { +// this.type = type; +// } +// } + +// public class Outputs { +// ArrayList files = new ArrayList(); + +// // Getter Methods + +// // Setter Methods + +// } + +// public class Inputs { +// ArrayList directories = new ArrayList(); +// ArrayList files = new ArrayList(); +// Map parameters = new HashMap(); + +// public Inputs(ArrayList directories, ArrayList files, Map parameters) { +// this.directories = directories; +// this.files = files; +// this.parameters = parameters; +// } + +// public ArrayList getDirectories() { +// return directories; +// } + +// public ArrayList getFiles() { +// return files; +// } + +// public Map getParameters() { +// return parameters; +// } + +// } + +// public class Parameters { +// private String input; + +// // Getter Methods + +// public String getInput() { +// return input; +// } + +// // Setter Methods + +// public void setInput(String input) { +// this.input = input; +// } +// } +// } \ No newline at end of file diff --git a/server/src/main/java/org/diskproject/server/adapters/ReanaAdapter.java b/server/src/main/java/org/diskproject/server/adapters/ReanaAdapter.java new file mode 100644 index 0000000..bd9e5cd --- /dev/null +++ b/server/src/main/java/org/diskproject/server/adapters/ReanaAdapter.java @@ -0,0 +1,444 @@ +package org.diskproject.server.adapters; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.io.FileUtils; +import org.diskproject.server.adapters.Reana.ApiClient; +import org.diskproject.server.adapters.Reana.ApiClient.ResponseListWorkflow; +import org.diskproject.server.adapters.Reana.ApiSchema.ReanaSpecification; +import org.diskproject.server.adapters.Reana.ApiSchema.ReanaSpecification.Inputs; +import org.diskproject.server.adapters.Reana.ApiSchema.ReanaSpecification.Outputs; +import org.diskproject.server.adapters.Reana.ApiSchema.ReanaWorkflow; +import org.diskproject.server.adapters.Reana.ApiSchema.ResponseGetSpecification; +import org.diskproject.server.adapters.Reana.ApiSchema.ResponseRunStatus; +import org.diskproject.server.adapters.Reana.ApiSchema.ResponseRunStatus.Progress; +import org.diskproject.shared.classes.adapters.MethodAdapter; +import org.diskproject.shared.classes.loi.TriggeredLOI.Status; +import org.diskproject.shared.classes.workflow.Variable; +import org.diskproject.shared.classes.workflow.VariableBinding; +import org.diskproject.shared.classes.workflow.Workflow; +import org.diskproject.shared.classes.workflow.WorkflowRun; + +public class ReanaAdapter extends MethodAdapter { + private static final int CONNECT_TIMEOUT = 1000 * 2; + private static final int READ_TIMEOUT = 1000 * 2; + private ApiClient apiClient; + private ResponseRunStatus runStatus; + + /** + * Method Adapter for Reana https://reanahub.io/ + * + * @param adapterName Name of the adapter: reana + * @param apiUrl REANA API URL + * @param username REANA username (leave blank for token access) + * @param password REANA password (if you are using token access, use the + * token) + */ + public ReanaAdapter(String adapterName, String apiUrl, String username, String password) { + super(adapterName, apiUrl, username, password); + this.status.put(Status.QUEUED, "queued"); + this.status.put(Status.FAILED, "failed"); + this.status.put(Status.RUNNING, "running"); + this.status.put(Status.SUCCESSFUL, "finished"); + this.apiClient = new ApiClient(apiUrl, password, username); + } + + public String handleCollectionParameter(String value) { + if (value.startsWith("[") && value.endsWith("]")) { + String delimiter = ""; + String array[] = value.replace("[", "").replace("]", "").split(","); + StringBuilder sb = new StringBuilder(); + for (String item : array) + sb.append(item.toString()).append(delimiter); + return sb.toString(); + } else { + return value; + } + } + + /** + * Create the parameters for the REANA workflow using DISK variables binding + * + * @param variableBindings + * @param inputVariables + * @return + */ + public Map createInputParameters(List variableBindings, + Map inputVariables) { + /** + * Convert variable bindings to json + */ + Map inputParameters = new HashMap(); + for (String variableKey : inputVariables.keySet()) { + Variable variable = inputVariables.get(variableKey); + for (VariableBinding variableBinding : variableBindings) { + if (variableBinding.getVariable().equals(variable.getName())) { + String value = handleCollectionParameter(variableBinding.getBinding()); + inputParameters.put(variableKey, value); + } + } + } + return inputParameters; + } + + /** + * @param url + * @param name + * @param dType + * @return String + * @throws Exception + */ + @Override + public String addData(String url, String name, String dType) throws Exception { + return null; + } + + /** + * @param url + * @param name + * @return String + * @throws IOException + */ + public String downloadData(String url, String name) throws IOException { + File directory = File.createTempFile("tmp", ""); + if (!directory.delete() || !directory.mkdirs()) { + System.err.println("Could not create temporary directory " + directory); + return null; + } + File file = new File(directory.getAbsolutePath() + "/" + name); + FileUtils.copyURLToFile( + new URL(url), + file, + CONNECT_TIMEOUT, + READ_TIMEOUT); + return file.getAbsolutePath(); + } + + /** + * @param workflowName + * @param workflowId + * @param variableBindings + * @param inputVariables + * @return String + * @throws Exception + */ + /* + * (non-Javadoc) + * + * @see + * org.diskproject.shared.classes.adapters.MethodAdapter#runWorkflow(java.lang. + * String, java.util.List, java.util.Map) + */ + @Override + public String runWorkflow(String workflowId, String workflowLink, List variableBindings, + Map inputVariables) throws Exception { + + Map inputParameters = createInputParameters(variableBindings, inputVariables); + this.apiClient.startWorkflow(workflowId, inputParameters, null); + return workflowId; + } + + /** + * @return boolean + */ + @Override + public boolean ping() { + String requestUrl = this.getEndpointUrl() + "/api/ping"; + try { + this.apiClient.performHTTPGet(requestUrl, "application/json"); + } catch (Exception e) { + return false; + } + return true; + } + + /** + * @return List + */ + /* + * (non-Javadoc) + * + * @see org.diskproject.shared.classes.adapters.MethodAdapter#getWorkflowList() + */ + @Override + public List getWorkflowList() { + List workflowDisk = new ArrayList(); + try { + ResponseListWorkflow workflowResponse = this.apiClient.getWorkflows(); + for (ReanaWorkflow workflowReana : workflowResponse.items) { + String url = this.getEndpointUrl() + "/api/workflows/" + workflowReana.id; + Workflow workflow = new Workflow(workflowReana.id, workflowReana.name, url, this.getName()); + workflowDisk.add(workflow); + } + } catch (Exception e) { + System.err.println(e.getMessage()); + } + return workflowDisk; + } + + /** + * @param id + * @return List + * @throws Exception + */ + public List getWorkflowVariables(String id) throws Exception { + ResponseGetSpecification responseSpecification = this.apiClient.getSpecification(id); + ReanaSpecification specification = responseSpecification.specification; + return getVariables(specification); + } + + /** + * @param id + * @return String + */ + @Override + public String getWorkflowId(String id) { + return this.getEndpointUrl() + "/api/workflows/" + id; + } + + /** + * @param id + * @return String + */ + @Override + public String getWorkflowUri(String id) { + return this.getEndpointUrl() + "/api/workflows/" + id; + } + + /** + * @param id + * @return String + */ + @Override + public String getWorkflowLink(String id) { + return this.getEndpointUrl() + "/api/workflows/" + id; + } + + /** + * @param runId + * @return WorkflowRun + * @throws Exception + */ + @Override + public WorkflowRun getRunStatus(String runId) throws Exception { + // Auto-generated method stub + ResponseRunStatus reanaRun = this.apiClient.getRunStatus(runId); + WorkflowRun run = new WorkflowRun(); + Progress progress = reanaRun.progress; + if (reanaRun.id == null) { + run.setStatus("failed"); + throw new Exception("Run not found"); + } + run.setId(reanaRun.id); + run.setStatus(reanaRun.status); + run.setStartDate(progress.getRun_started_at()); + run.setEndDate(progress.getRun_finished_at()); + + ReanaSpecification specification = this.apiClient.getSpecification(runId).specification; + if (run.getStatus().equals("finished")) { + + Map outputs = convertReanaOutputs(specification); + Map inputs = convertReanaInputs(specification); + run.setOutputs(outputs); + run.setFiles(inputs); + } + + return run; + } + + /** + * @param dataUrl + * @return String + * @throws Exception + */ + @Override + public byte[] fetchData(String dataUrl) throws Exception { + return this.apiClient.downloadData(dataUrl); + } + + /** + * @param runId + * @return Map + */ + @Override + public Map getRunVariableBindings(String runId) { + // Auto-generated method stub + return null; + } + + /** + * @param id + * @return String + */ + @Override + public String getDataUri(String dataId, String runId) { + String requestUrl = String.format(this.apiClient.API_WORKFLOW_DOWNLOAD_FILE, this.apiClient.url, runId, dataId); + return requestUrl; + } + + /** + * @param fileList + * @param dType + * @return List + */ + @Override + public List areFilesAvailable(Set fileList, String dType) { + return new ArrayList(); + } + + /** + * @param specification + * @return List + * @throws Exception + */ + public List getVariables(ReanaSpecification specification) throws Exception { + List variables = new ArrayList(); + Inputs reanaInputs = specification.inputs; + ArrayList reanaFiles = reanaInputs.getFiles(); + reanaInputs.getParameters().forEach((key, value) -> { + Variable variable = new Variable(); + variable.setName(key); + variable.setInput(true); + List type = new ArrayList(); + if (reanaFiles.contains(value)) { + type.add("File"); + variable.setParam(false); + } else { + variable.setName(key.toString()); + // TODO: the spec doesn't contain information about the type + type.add("Parameter"); + variable.setParam(true); + } + variable.setType(type); + variables.add(variable); + }); + return variables; + } + + /** + * @param id + * @return String + * @throws Exception + */ + public String duplicateWorkflow(String id) throws Exception { + ReanaSpecification oldSpecification = this.apiClient.getSpecification(id).specification; + runStatus = this.apiClient.getRunStatus(id); + String newWorkflowId = this.apiClient.createWorkflow(oldSpecification, runStatus.getName()); + return newWorkflowId; + } + + /** + * @param id + * @param name + * @return String + * @throws Exception + */ + public String duplicateWorkflow(String id, String name) throws Exception { + ReanaSpecification oldSpecification = this.apiClient.getSpecification(id).specification; + if (name == null) { + String[] nameArray = id.split("\\."); + name = nameArray[0]; + } + runStatus = this.apiClient.getRunStatus(id); + String newWorkflowId = this.apiClient.createWorkflow(oldSpecification, name); + return newWorkflowId; + } + + /** + * @param specification + * @return Map + * @throws Exception + */ + Map convertReanaOutputs(ReanaSpecification specification) + throws Exception { + Map outputs = new HashMap(); + Outputs reanaRunOutputs = specification.outputs; + for (String output : reanaRunOutputs.getFiles()) { + Path path = Paths.get(output); + path.getFileName(); + outputs.put(path.getFileName().toString(), output); + } + return outputs; + } + + /** + * @param specification + * @return Map + * @throws Exception + */ + private Map convertReanaInputs(ReanaSpecification specification) throws Exception { + Map inputs = new HashMap(); + Inputs reanaRunInputs = specification.inputs; + for (String input : reanaRunInputs.getFiles()) { + inputs.put(input, input); + } + return inputs; + } + + /** + * @param url + * @param workSpaceId + * @param name + * @param dType + * @return String + * @throws Exception + */ + @Override + public String addData(String url, String workSpaceId, String name, String dType) throws Exception { + String filePath = downloadData(url, name); + String remoteUrl = this.apiClient.addFileWorkspace(workSpaceId, filePath, name); + return remoteUrl; + } + + @Override + public Map getWorkflowInputs(String id) throws Exception { + // TODO Auto-generated method stub + ResponseGetSpecification specification = this.apiClient.getSpecification(id); + ReanaSpecification reanaSpecification = specification.specification; + Map variables = getVariablesDiskFormat(reanaSpecification); + return variables; + } + + /* + * Get the list of variables using the Disk format + * + * @param specification + * + * @return List + * + * @throws Exception + */ + public Map getVariablesDiskFormat(ReanaSpecification specification) throws Exception { + Map variables2 = new HashMap<>(); + Inputs reanaInputs = specification.inputs; + ArrayList reanaFiles = reanaInputs.getFiles(); + reanaInputs.getParameters().forEach((key, value) -> { + Variable variable = new Variable(); + variable.setName(key); + variable.setInput(true); + + List type = new ArrayList(); + if (reanaFiles.contains(value)) { + type.add("File"); + variable.setParam(false); + } else { + variable.setName(key.toString()); + // TODO: the spec doesn't contain information about the type + type.add("Parameter"); + variable.setParam(true); + } + variable.setType(type); + variables2.put(key, variable); + }); + return variables2; + } +} diff --git a/server/src/main/java/org/diskproject/server/api/impl/DiskResource.java b/server/src/main/java/org/diskproject/server/api/impl/DiskResource.java index 3dd8a18..b0a182e 100644 --- a/server/src/main/java/org/diskproject/server/api/impl/DiskResource.java +++ b/server/src/main/java/org/diskproject/server/api/impl/DiskResource.java @@ -377,7 +377,28 @@ public List listWorkflows() { public List getWorkflowVariables( @PathParam("source") String source, @PathParam("id") String id) { - return this.repo.getWorkflowVariables(source, id); + Gson response_error = new Gson(); + try { + return this.repo.getWorkflowVariables(source, id); + } catch (Exception e) { + try { + // Create Json error response + ErrorMessage error = new ErrorMessage(e.getMessage()); + String jsonData = response_error.toJson(error); + + // Prepare the response + response.setContentType("application/json"); + response.setCharacterEncoding("utf-8"); + response.setStatus(500); + + // Send the response + response.getWriter().print(jsonData.toString()); + response.getWriter().flush(); + } catch (IOException e1) { + e1.printStackTrace(); + } + } + return null; } @GET @@ -386,7 +407,29 @@ public List getWorkflowVariables( public WorkflowRun monitorWorkflow( @PathParam("source") String source, @PathParam("id") String id) { - return this.repo.getWorkflowRunStatus(source, id); + try { + // return WingsAdapter.get().getWorkflowList(); + return this.repo.getWorkflowRunStatus(source, id); + } catch (Exception e) { + try { + // Create Json error response + Gson gson = new Gson(); + ErrorMessage error = new ErrorMessage(e.getMessage()); + String jsonData = gson.toJson(error); + + // Prepare the response + response.setContentType("application/json"); + response.setCharacterEncoding("utf-8"); + response.setStatus(500); + + // Send the response + response.getWriter().print(jsonData.toString()); + response.getWriter().flush(); + } catch (IOException e1) { + e1.printStackTrace(); + } + } + return null; } @GET @@ -570,17 +613,35 @@ public Map getNarratives( @POST @Path("getData") - @Produces("text/html") @Override public String getOutputData( @JsonProperty("request") ExternalDataRequest r) { - byte[] result = this.repo.getOutputData(r.getSource(), r.getDataId()); - String tmp = ""; // FIXME: this should not be an string. - if (result == null) { - System.out.println("ERROR: " + r.getDataId() + " not available on " + r.getSource() + "."); - } else { - tmp = new String(result, StandardCharsets.UTF_8); + try { + byte[] result = this.repo.getOutputData(r.getSource(), r.getRunId(), r.getDataId()); + response.setContentType("application/json"); + if (result == null) { + throw new Exception("No data found"); + } + return new String(result, StandardCharsets.UTF_8); + } catch (Exception e) { + try { + // Create Json error response + Gson gson = new Gson(); + ErrorMessage error = new ErrorMessage(e.getMessage()); + String jsonData = gson.toJson(error); + + // Prepare the response + response.setContentType("application/json"); + response.setCharacterEncoding("utf-8"); + response.setStatus(500); + + // Send the response + response.getWriter().print(jsonData.toString()); + response.getWriter().flush(); + } catch (IOException e1) { + e1.printStackTrace(); + } } - return tmp; + return null; } } \ No newline at end of file diff --git a/server/src/main/java/org/diskproject/server/repository/DiskRepository.java b/server/src/main/java/org/diskproject/server/repository/DiskRepository.java index 756d4dd..a394223 100644 --- a/server/src/main/java/org/diskproject/server/repository/DiskRepository.java +++ b/server/src/main/java/org/diskproject/server/repository/DiskRepository.java @@ -26,6 +26,7 @@ import org.apache.commons.lang.SerializationUtils; import org.apache.jena.query.QueryParseException; import org.diskproject.server.adapters.AirFlowAdapter; +import org.diskproject.server.adapters.ReanaAdapter; import org.diskproject.server.adapters.GraphDBAdapter; import org.diskproject.server.adapters.SparqlAdapter; import org.diskproject.server.util.Config; @@ -73,7 +74,7 @@ public class DiskRepository extends WriteKBRepository { private static SimpleDateFormat dateformatter = new SimpleDateFormat("HH:mm:ss yyyy-MM-dd"); Pattern varPattern = Pattern.compile("\\?(.+?)\\b"); - Pattern varCollPattern = Pattern.compile("\\[\\s*\\?(.+?)\\s*\\]"); + Pattern varCollectionPattern = Pattern.compile("\\[\\s*\\?(.+?)\\s*\\]"); protected KBAPI questionKB, hypothesisVocabulary; protected KBCache SQOnt; @@ -86,11 +87,19 @@ public class DiskRepository extends WriteKBRepository { private Map>> optionsCache; private Map externalVocabularies; + + /** + * @param args + */ public static void main(String[] args) { get(); get().shutdownExecutors(); } + + /** + * @return DiskRepository + */ public static DiskRepository get() { if (!creatingKB && singleton == null) { creatingKB = true; @@ -154,6 +163,10 @@ public void initializeKB() throws Exception { this.initializeVocabularies(); } + + /** + * @throws Exception + */ private void loadKBFromConfig() throws Exception { this.externalVocabularies = new HashMap(); @@ -213,6 +226,10 @@ private void loadKBFromConfig() throws Exception { } } + + /** + * @throws Exception + */ public void reloadKBCaches() throws Exception { KBAPI[] kbs = { this.ontKB, this.questionKB, this.hypothesisVocabulary }; @@ -247,6 +264,10 @@ public void reloadKBCaches() throws Exception { this.initializeKB(); } + + /** + * @return PropertyListConfiguration + */ public PropertyListConfiguration getConfig() { return Config.get().getProperties(); } @@ -339,12 +360,21 @@ private void initializeDataAdapters() { } } + + /** + * @param url + * @return DataAdapter + */ private DataAdapter getDataAdapter(String url) { if (this.dataAdapters.containsKey(url)) return this.dataAdapters.get(url); return null; } + + /** + * @return List + */ public List getDataAdapters() { List adapters = new ArrayList(); for (DataAdapter da : this.dataAdapters.values()) { @@ -383,7 +413,8 @@ private void initializeMethodAdapters() { continue; } String curURI = cur.get(ConfigKeys.ENDPOINT), curType = cur.get(ConfigKeys.TYPE); - String curUser = null, curPass = null, curDomain = null, curInternalServer = null; + String curUser = null, curPass = null, curDomain = null, curInternalServer = null, inventory = null; + Float curVersion = null; if (cur.containsKey(ConfigKeys.USERNAME)) curUser = cur.get(ConfigKeys.USERNAME); @@ -404,6 +435,9 @@ private void initializeMethodAdapters() { case ConfigKeys.METHOD_TYPE_AIRFLOW: curAdapter = new AirFlowAdapter(name, curURI, curUser, curPass); break; + case ConfigKeys.METHOD_TYPE_REANA: + curAdapter = new ReanaAdapter(name, curURI, curUser, curPass); + break; default: System.out.println("Error: Method adapter type not found: '" + curType + "'"); break; @@ -426,12 +460,21 @@ private void initializeMethodAdapters() { } } + + /** + * @param url + * @return MethodAdapter + */ public MethodAdapter getMethodAdapter(String url) { if (this.methodAdapters.containsKey(url)) return this.methodAdapters.get(url); return null; } + + /** + * @return List + */ public List getWorkflowList() { List list = new ArrayList(); for (MethodAdapter adapter : this.methodAdapters.values()) { @@ -442,7 +485,14 @@ public List getWorkflowList() { return list; } - public List getWorkflowVariables(String source, String id) { + + /** + * @param source + * @param id + * @return List + * @throws Exception + */ + public List getWorkflowVariables(String source, String id) throws Exception { for (MethodAdapter adapter : this.methodAdapters.values()) { if (adapter.getName().equals(source)) { return adapter.getWorkflowVariables(id); @@ -481,10 +531,24 @@ private void initializeVocabularies() { } } + + /** + * @param uri + * @return Vocabulary + */ public Vocabulary getVocabulary(String uri) { return this.vocabularies.get(uri); } + + /** + * @param kb + * @param ns + * @param prefix + * @param title + * @param description + * @return Vocabulary + */ public Vocabulary initializeVocabularyFromKB(KBAPI kb, String ns, String prefix, String title, String description) { Vocabulary vocabulary = new Vocabulary(ns); vocabulary.setPrefix(prefix); @@ -496,6 +560,11 @@ public Vocabulary initializeVocabularyFromKB(KBAPI kb, String ns, String prefix, return vocabulary; } + + /** + * @param kb + * @param vocabulary + */ private void fetchPropertiesFromKB(KBAPI kb, Vocabulary vocabulary) { for (KBObject prop : kb.getAllProperties()) { if (!prop.getID().startsWith(vocabulary.getNamespace())) @@ -522,6 +591,11 @@ private void fetchPropertiesFromKB(KBAPI kb, Vocabulary vocabulary) { } } + + /** + * @param kb + * @param vocabulary + */ private void fetchTypesAndIndividualsFromKB(KBAPI kb, Vocabulary vocabulary) { KBObject typeprop = kb.getProperty(KBConstants.RDF_NS + "type"); for (KBTriple t : kb.genericTripleQuery(null, typeprop, null)) { @@ -600,6 +674,11 @@ private void fetchTypesAndIndividualsFromKB(KBAPI kb, Vocabulary vocabulary) { } } + + /** + * @param pname + * @return String + */ private String createPropertyLabel(String pname) { // Remove starting "has" pname = pname.replaceAll("^has", ""); @@ -618,6 +697,12 @@ public Map getVocabularies() { return this.vocabularies; } + + /** + * @param graph + * @param localDomain + * @return Graph + */ private Graph resolvePrefixesForGraph(Graph graph, String localDomain) { List triples = new ArrayList(); for (Triple triple : graph.getTriples()) { @@ -630,6 +715,12 @@ private Graph resolvePrefixesForGraph(Graph graph, String localDomain) { return newGraph; } + + /** + * @param value + * @param localDomain + * @return String + */ private String resolvePrefixes(String value, String localDomain) { if (localDomain != null && !localDomain.equals("") && value.charAt(0) == ':') { // replace ":" for local domain value = localDomain + value.substring(1); @@ -650,6 +741,12 @@ private String resolvePrefixes(String value, String localDomain) { return value; } + + /** + * @param username + * @param hypothesis + * @return Hypothesis + */ /* * Hypotheses */ @@ -681,14 +778,33 @@ public Hypothesis addHypothesis(String username, Hypothesis hypothesis) { return null; } + + /** + * @param username + * @param id + * @return boolean + */ public boolean removeHypothesis(String username, String id) { return this.deleteHypothesis(username, id); } + + /** + * @param username + * @param id + * @return Hypothesis + */ public Hypothesis getHypothesis(String username, String id) { return loadHypothesis(username, id); } + + /** + * @param username + * @param id + * @param hypothesis + * @return Hypothesis + */ public Hypothesis updateHypothesis(String username, String id, Hypothesis hypothesis) { String name = hypothesis.getName(); String desc = hypothesis.getDescription(); @@ -700,6 +816,11 @@ public Hypothesis updateHypothesis(String username, String id, Hypothesis hypoth return null; } + + /** + * @param username + * @return List + */ public List listHypotheses(String username) { List list = new ArrayList(); List hypothesisList = listHypothesesPreviews(username); @@ -716,6 +837,12 @@ public List listHypotheses(String username) { return list; } + + /** + * @param username + * @param loi + * @return LineOfInquiry + */ /* * Lines of Inquiry */ @@ -740,20 +867,44 @@ public LineOfInquiry addLOI(String username, LineOfInquiry loi) { return null; } + + /** + * @param username + * @param id + * @return boolean + */ public boolean removeLOI(String username, String id) { return deleteLOI(username, id); } + + /** + * @param username + * @param id + * @return LineOfInquiry + */ public LineOfInquiry getLOI(String username, String id) { return this.loadLOI(username, id); } + + /** + * @param username + * @param id + * @param loi + * @return LineOfInquiry + */ public LineOfInquiry updateLOI(String username, String id, LineOfInquiry loi) { if (loi.getId() != null && this.deleteLOI(username, id)) return this.addLOI(username, loi); return null; } + + /** + * @param username + * @return List + */ public List listLOIs(String username) { List list = new ArrayList(); List lois = this.listLOIPreviews(username); @@ -770,6 +921,12 @@ public List listLOIs(String username) { return list; } + + /** + * @param username + * @param tloi + * @return TriggeredLOI + */ /* * Triggered Lines of Inquiries (TLOIs) */ @@ -789,20 +946,46 @@ public TriggeredLOI addTriggeredLOI(String username, TriggeredLOI tloi) { return tloi; } + + /** + * @param username + * @param id + * @return boolean + */ public boolean removeTriggeredLOI(String username, String id) { return deleteTLOI(username, id); } + + /** + * @param username + * @param id + * @return TriggeredLOI + */ public TriggeredLOI getTriggeredLOI(String username, String id) { return loadTLOI(username, id); } + + /** + * @param username + * @param id + * @param tloi + * @return TriggeredLOI + */ private TriggeredLOI updateTriggeredLOI(String username, String id, TriggeredLOI tloi) { if (tloi.getId() != null && this.deleteTLOI(username, id) && this.writeTLOI(username, tloi)) return tloi; return null; } + + /** + * @param username + * @param id + * @param tloi + * @return TriggeredLOI + */ public TriggeredLOI updateTLOINotes(String username, String id, TriggeredLOI tloi) { TriggeredLOI updatedTLOI = getTriggeredLOI(username, id); if (updatedTLOI != null && tloi != null) { @@ -810,10 +993,16 @@ public TriggeredLOI updateTLOINotes(String username, String id, TriggeredLOI tlo if (this.deleteTLOI(username, id) && this.writeTLOI(username, updatedTLOI)) return tloi; } - //TODO: We return the request as default, check what when wrong and send apropiate error. + // TODO: We return the request as default, check what when wrong and send + // apropiate error. return tloi; } + + /** + * @param username + * @return List + */ public List listTriggeredLOIs(String username) { return listTLOIs(username); } @@ -858,6 +1047,11 @@ private void loadQuestionTemplates() { } } + + /** + * @param url + * @return List + */ public List loadQuestionsFromKB(String url) { System.out.println("Loading Question Templates: " + url); List questions = new ArrayList(); @@ -917,10 +1111,20 @@ public List loadQuestionsFromKB(String url) { return null; } + + /** + * @return List + */ public List listHypothesesQuestions() { return new ArrayList(this.allQuestions.values()); } + + /** + * @param sid + * @return List> + * @throws Exception + */ public List> listVariableOptions(String sid) throws Exception { if (!optionsCache.containsKey(sid)) { optionsCache.put(sid, this.loadVariableOptions(sid)); @@ -928,6 +1132,12 @@ public List> listVariableOptions(String sid) throws Exception { return optionsCache.get(sid); } + + /** + * @param sid + * @return List> + * @throws Exception + */ private List> loadVariableOptions(String sid) throws Exception { System.out.println("ID: " + sid); List> options = new ArrayList>(); @@ -1069,6 +1279,10 @@ public Map>> listDynamicOptions (QuestionOptionsRequest return all; } + + /** + * @return String + */ /* * Querying */ @@ -1081,6 +1295,15 @@ private String getAllPrefixes() { return prefixes; } + + /** + * @param endpoint + * @param sparqlQuery + * @param variables + * @return Map> + * @throws Exception + * @throws QueryParseException + */ public Map> queryExternalStore(String endpoint, String sparqlQuery, String variables) throws Exception, QueryParseException { // FIXME: change this to DataResults @@ -1132,6 +1355,13 @@ public Map> queryExternalStore(String endpoint, String spar return dataVarBindings; } + + /** + * @param queryPattern + * @param variablePattern + * @param variableBindings + * @return String + */ private String getQueryBindings(String queryPattern, Pattern variablePattern, Map variableBindings) { String pattern = ""; @@ -1157,6 +1387,12 @@ private String getQueryBindings(String queryPattern, Pattern variablePattern, return pattern; } + + /** + * @param queryA + * @param queryB + * @return Set + */ public Set interceptVariables(final String queryA, final String queryB) { Set A = new HashSet(); Matcher a = varPattern.matcher(queryA); @@ -1176,6 +1412,11 @@ public Set interceptVariables(final String queryA, final String queryB) return B; } + + /** + * @param loi + * @return Boolean + */ /* * Executing hypothesis */ @@ -1206,6 +1447,12 @@ private Boolean isValid(LineOfInquiry loi) { return true; } + + /** + * @param loi + * @param hypothesisBindings + * @return Boolean + */ private Boolean isValid(LineOfInquiry loi, Map hypothesisBindings) { // Check if the hypothesis bindings (values from the user) are valid in this LOI if (!isValid(loi)) @@ -1237,6 +1484,12 @@ private Boolean isValid(LineOfInquiry loi, Map hypothesisBinding return true; } + + /** + * @param username + * @param id + * @return Map>> + */ public Map>> getLOIByHypothesisId(String username, String id) { String hypuri = this.HYPURI(username) + "/" + id; // LOIID -> [{ variable -> value }] @@ -1324,6 +1577,14 @@ public Map>> getLOIByHypothesisId(String return results; } + + /** + * @param username + * @param id + * @return List + * @throws Exception + * @throws QueryParseException + */ public List queryHypothesis(String username, String id) throws Exception, QueryParseException { // Create TLOIs that match with a hypothesis and username List tlois = new ArrayList(); @@ -1343,8 +1604,8 @@ public List queryHypothesis(String username, String id) throws Exc continue; } else { boolean allOk = true; - for (WorkflowBindings wb: loi.getWorkflows()) { - String source = wb.getSource(); + for (WorkflowBindings wb : loi.getWorkflows()) { + String source = wb.getSource(); if (source == null || getMethodAdapterByName(source) == null) { allOk = false; System.out.println("Warning: " + loi.getId() + " uses an unknown method adapter: " + source); @@ -1352,11 +1613,12 @@ public List queryHypothesis(String username, String id) throws Exc } } if (allOk) - for (WorkflowBindings wb: loi.getMetaWorkflows()) { - String source = wb.getSource(); + for (WorkflowBindings wb : loi.getMetaWorkflows()) { + String source = wb.getSource(); if (source == null || getMethodAdapterByName(source) == null) { allOk = false; - System.out.println("Warning: " + loi.getId() + " uses an unknown method adapter: " + source); + System.out + .println("Warning: " + loi.getId() + " uses an unknown method adapter: " + source); break; } } @@ -1438,8 +1700,7 @@ public List queryHypothesis(String username, String id) throws Exc tloi.setDataQuery(dq); // Updated data query tloi.setDateCreated(dateformatter.format(new Date())); tlois.add(tloi); - } - else { + } else { System.out.println("LOI " + loi.getId() + " got no results"); } } @@ -1448,6 +1709,12 @@ public List queryHypothesis(String username, String id) throws Exc return checkExistingTLOIs(username, tlois); } + + /** + * @param username + * @param tlois + * @return List + */ // This replaces all triggered lines of inquiry already executed. private List checkExistingTLOIs(String username, List tlois) { List checked = new ArrayList(); @@ -1479,35 +1746,59 @@ private List checkExistingTLOIs(String username, List + * @throws Exception + */ @SuppressWarnings("unchecked") private List getTLOIBindings(String username, List wflowBindings, Map> dataVarBindings, DataAdapter dataAdapter) throws Exception { List tloiBindings = new ArrayList(); for (WorkflowBindings bindings : wflowBindings) { // FOR EACH WORKFLOW // For each Workflow, create an empty copy to set the values + MethodAdapter methodAdapter = getMethodAdapterByName(bindings.getSource()); + if (methodAdapter == null) + throw new Exception("No method adapter found for " + bindings.getSource()); + else if (methodAdapter instanceof ReanaAdapter) { + try { + String newWorkflowId = methodAdapter.duplicateWorkflow(bindings.getWorkflow(), null); + String newWorkflowLink = methodAdapter.getWorkflowLink(newWorkflowId); + bindings.setWorkflow(newWorkflowId); + bindings.setWorkflowLink(newWorkflowLink); + } catch (Exception e) { + System.out.println("Error duplicating workflow " + bindings.getWorkflowLink()); + System.out.println(e.getMessage()); + continue; + } + } + + String workflowId = bindings.getWorkflow(); WorkflowBindings tloiBinding = new WorkflowBindings( bindings.getWorkflow(), bindings.getWorkflowLink()); tloiBinding.setSource(bindings.getSource()); tloiBinding.setMeta(bindings.getMeta()); tloiBindings.add(tloiBinding); - MethodAdapter methodAdapter = getMethodAdapterByName(bindings.getSource()); List allVars = methodAdapter.getWorkflowVariables(bindings.getWorkflow()); - for (VariableBinding vbinding : bindings.getBindings()) { // Normal variable bindings. + for (VariableBinding variableBinding : bindings.getBindings()) { // Normal variable bindings. // For each Variable binding, check : // - If this variable expects a collection or single values // - Check the binding values on the data store - String binding = vbinding.getBinding(); - Matcher collmat = varCollPattern.matcher(binding); + String binding = variableBinding.getBinding(); + Matcher collectionMatch = varCollectionPattern.matcher(binding); Matcher mat = varPattern.matcher(binding); // Get the sparql variable boolean isCollection = false; String sparqlvar = null; - if (collmat.find() && dataVarBindings.containsKey(collmat.group(1))) { - sparqlvar = collmat.group(1); + if (collectionMatch.find() && dataVarBindings.containsKey(collectionMatch.group(1))) { + sparqlvar = collectionMatch.group(1); isCollection = true; } else if (mat.find() && dataVarBindings.containsKey(mat.group(1))) { sparqlvar = mat.group(1); @@ -1517,11 +1808,11 @@ private List getTLOIBindings(String username, List dsurls = dataVarBindings.get(sparqlvar); + List resourcesUrl = dataVarBindings.get(sparqlvar); // Checks if the bindings are input files. boolean bindingsAreFiles = true; - for (String candurl : dsurls) { + for (String candurl : resourcesUrl) { if (!candurl.startsWith("http")) { bindingsAreFiles = false; break; @@ -1529,35 +1820,36 @@ private List getTLOIBindings(String username, List dsnames = new ArrayList(); + List datasetNames = new ArrayList(); if (bindingsAreFiles) { - String varName = vbinding.getVariable(); - String dType = null; - for (Variable v: allVars) { + String varName = variableBinding.getVariable(); + String resourceDataType = null; + for (Variable v : allVars) { if (varName.equals(v.getName())) { List classes = v.getType(); if (classes != null && classes.size() > 0) { - dType = classes.contains(vbinding.getType()) ? vbinding.getType() : classes.get(0); + resourceDataType = classes.contains(variableBinding.getType()) ? variableBinding.getType() : classes.get(0); } } } // TODO: this should be async // Check hashes, create local name and upload data: - Map urlToName = addData(dsurls, methodAdapter, dataAdapter, dType); - for (String dsurl : dsurls) { + Map urlToName = addData(resourcesUrl, methodAdapter, dataAdapter, resourceDataType, + workflowId); + for (String dsurl : resourcesUrl) { String dsname = urlToName.containsKey(dsurl) ? urlToName.get(dsurl) : dsurl.replaceAll("^.*\\/", ""); - dsnames.add(dsname); + datasetNames.add(dsname); } } else { // If the binding is not a file, send the value with no quotes - for (String value : dsurls) { + for (String value : resourcesUrl) { // Remove quotes from parameters if (value.charAt(0) == '"' && value.charAt(value.length() - 1) == '"') { value = value.substring(1, value.length() - 1); } - dsnames.add(value); + datasetNames.add(value); } } @@ -1565,17 +1857,17 @@ private List getTLOIBindings(String username, List newTloiBindings = new ArrayList(); for (WorkflowBindings tmpBinding : tloiBindings) { // For all already processed workflow // bindings - for (String dsname : dsnames) { + for (String dsname : datasetNames) { ArrayList newBindings = (ArrayList) SerializationUtils .clone((Serializable) tmpBinding.getBindings()); @@ -1583,7 +1875,8 @@ private List getTLOIBindings(String username, List getTLOIBindings(String username, List URL - private Map addData(List dsurls, MethodAdapter methodAdapter, DataAdapter dataAdapter, String dType) + + /** + * @param resourceUrl + * @param methodAdapter + * @param dataAdapter + * @param resourceDataType + * @param workflowId + * @return Map + * @throws Exception + */ + // This adds dsUrls to the data-repository, returns filename -> URL + private Map addData(List resourceUrl, MethodAdapter methodAdapter, DataAdapter dataAdapter, + String resourceDataType, String workflowId) throws Exception { - // To add files to wings and not replace anything, we need to get the hash from the wiki. + // To add files to wings and not replace anything, we need to get the hash from + // the wiki. // TODO: here connect with minio. Map nameToUrl = new HashMap(); Map urlToName = new HashMap(); - Map filesETag = dataAdapter.getFileHashesByETag(dsurls); // File -> ETag + Map filesETag = dataAdapter.getFileHashesByETag(resourceUrl); // File -> ETag boolean allOk = true; // All is OK if we have all file ETags. - for (String fileUrl: dsurls) { + for (String fileUrl : resourceUrl) { if (filesETag.containsKey(fileUrl)) { String eTag = filesETag.get(fileUrl); // This name should be different now, this is not the SHA @@ -1622,8 +1927,8 @@ private Map addData(List dsurls, MethodAdapter methodAda } if (!allOk) { // Get hashes from the data-adapter (SPARQL) - Map hashes = dataAdapter.getFileHashes(dsurls); // File -> SHA1 - for (String fileUrl : dsurls) { + Map hashes = dataAdapter.getFileHashes(resourceUrl); // File -> SHA1 + for (String fileUrl : resourceUrl) { if (hashes.containsKey(fileUrl)) { if (!urlToName.containsKey(fileUrl)) { String hash = hashes.get(fileUrl); @@ -1638,28 +1943,34 @@ private Map addData(List dsurls, MethodAdapter methodAda } // Show files with no hash and throw a exception. - for (String file : dsurls) { + for (String file : resourceUrl) { if (!urlToName.containsKey(file)) { - //TODO: hadnle exception + // TODO: hadnle exception System.err.println("Warning: file " + file + " does not contain any hash on " + dataAdapter.getName()); } } // avoid to duplicate files Set names = nameToUrl.keySet(); - List availableFiles = methodAdapter.areFilesAvailable(names, dType); + List availableFiles = methodAdapter.areFilesAvailable(names, resourceDataType); names.removeAll(availableFiles); // upload the files for (String newFilename : names) { String newFile = nameToUrl.get(newFilename); - System.out.println("Uploading to " + methodAdapter.getName() + ": " + newFile + " as " + newFilename + "(" + dType + ")"); - methodAdapter.addData(newFile, newFilename, dType); + System.out.println("Uploading to " + methodAdapter.getName() + ": " + newFile + " as " + newFilename); + methodAdapter.addData(newFile, workflowId, newFilename, resourceDataType); } return urlToName; } + + /** + * @param username + * @return Boolean + * @throws Exception + */ public Boolean runAllHypotheses(String username) throws Exception { List hlist = new ArrayList(); String url = this.HYPURI(username); @@ -1703,6 +2014,12 @@ public Boolean runAllHypotheses(String username) throws Exception { return true; } + + /** + * @param username + * @param tloid + * @return Map + */ /* * Narratives */ @@ -1815,6 +2132,11 @@ public Map getNarratives(String username, String tloiId) { return narratives; } + + /** + * @param dataQuery + * @return String + */ private String dataQueryNarrative(String dataQuery) { // this is necessary to replace the new line characters in query String dataQuery1 = dataQuery.replaceAll("^(//)n${1}", ""); @@ -1883,6 +2205,13 @@ private String dataQueryNarrative(String dataQuery) { } + + /** + * @param username + * @param hypId + * @param loiId + * @return List + */ /* * Running */ @@ -1901,6 +2230,14 @@ public List getTLOIsForHypothesisAndLOI(String username, String hy return list; } + + /** + * @param username + * @param hypid + * @param loiid + * @return List + * @throws Exception + */ public List runHypothesisAndLOI(String username, String hypid, String loiid) throws Exception { List hyptlois = queryHypothesis(username, hypid); // TriggeredLOI match = null; @@ -1919,22 +2256,37 @@ public List runHypothesisAndLOI(String username, String hypid, Str return getTLOIsForHypothesisAndLOI(username, hypid, loiid); } + + /** + * @param source + * @param id + * @return WorkflowRun + * @throws Exception + */ /* * Threads helpers */ - public WorkflowRun getWorkflowRunStatus(String source, String id) { + public WorkflowRun getWorkflowRunStatus(String source, String id) throws Exception { MethodAdapter methodAdapter = getMethodAdapterByName(source); if (methodAdapter == null) return null; return methodAdapter.getRunStatus(id); } - public byte[] getOutputData(String source, String id) { + + /** + * @param source + * @param id + * @return String + * @throws Exception + */ + public byte[] getOutputData(String source, String dataId, String workflowId) throws Exception { MethodAdapter methodAdapter = getMethodAdapterByName(source); if (methodAdapter == null) return null; - return methodAdapter.fetchData(methodAdapter.getDataUri(id)); + String dataUrl = methodAdapter.getDataUri(workflowId, dataId); + return methodAdapter.fetchData(dataUrl); } /* @@ -1988,7 +2340,8 @@ public void run() { + (l == null ? v.getBinding() : l[0] + " (" + l.length + ")")); } - String runid = methodAdapter.runWorkflow(bindings.getWorkflow(), sendbindings, inputs); + String runid = methodAdapter.runWorkflow(bindings.getWorkflow(), bindings.getWorkflowLink(), + sendbindings, inputs); if (runid != null) { System.out.println("[R] Run ID: " + runid); @@ -2048,17 +2401,17 @@ public void run() { WorkflowRun wstatus = methodAdapter.getRunStatus(rname); bindings.setRun(wstatus); - if (wstatus.getStatus().equals("FAILURE")) { + if (wstatus.getStatus().equals(methodAdapter.status.get(Status.FAILED))) { overallStatus = Status.FAILED; numFinished++; continue; } - if (wstatus.getStatus().equals("RUNNING")) { + if (wstatus.getStatus().equals(methodAdapter.status.get(Status.RUNNING))) { if (overallStatus != Status.FAILED) overallStatus = Status.RUNNING; continue; } - if (wstatus.getStatus().equals("SUCCESS")) { + if (wstatus.getStatus().equals(methodAdapter.status.get(Status.SUCCESSFUL))) { numFinished++; numSuccessful++; diff --git a/server/src/main/java/org/diskproject/server/repository/WingsAdapter.java b/server/src/main/java/org/diskproject/server/repository/WingsAdapter.java index 7facba0..de84b2b 100644 --- a/server/src/main/java/org/diskproject/server/repository/WingsAdapter.java +++ b/server/src/main/java/org/diskproject/server/repository/WingsAdapter.java @@ -41,6 +41,7 @@ import org.apache.http.message.BasicNameValuePair; import org.apache.http.util.EntityUtils; import org.diskproject.shared.classes.adapters.MethodAdapter; +import org.diskproject.shared.classes.loi.TriggeredLOI.Status; import org.diskproject.shared.classes.util.KBConstants; import org.diskproject.shared.classes.workflow.Variable; import org.diskproject.shared.classes.workflow.VariableBinding; @@ -67,6 +68,10 @@ public class WingsAdapter extends MethodAdapter { private String domain; private String internal_server; + public enum STATUS { + FAILURE, SUCCESS, RUNNING, QUEUED, UNKNOWN + } + private CookieStore cookieStore; public String workflowNS = "http://www.wings-workflows.org/ontology/workflow.owl#"; @@ -82,6 +87,10 @@ public WingsAdapter(String name, String url, String username, String password, S this.cookieStore = new BasicCookieStore(); this.json = new Gson(); this.jsonParser = new JsonParser(); + this.status.put(Status.QUEUED, "QUEUED"); + this.status.put(Status.FAILED, "FAILED"); + this.status.put(Status.RUNNING, "RUNNING"); + this.status.put(Status.SUCCESSFUL, "SUCCESS"); } public String DOMURI() { @@ -128,7 +137,7 @@ public String getWorkflowUri(String id) { } @Override - public String getDataUri(String id) { + public String getDataUri(String id, String workflowId) { return this.DATAID(id); } @@ -542,12 +551,12 @@ private boolean isPartOfCollection(String key) { } @Override - public String runWorkflow(String wflowname, List vbindings, Map inputVariables) { - wflowname = WFLOWID(wflowname); + public String runWorkflow(String workflowName, String workflowLink, List vbindings, Map inputVariables) { + workflowName = WFLOWID(workflowName); String toPost = null, getData = null, getParams = null, getExpansions = null; JsonObject response = null; try { - toPost = toPlanAcceptableFormat(wflowname, vbindings, inputVariables); + toPost = toPlanAcceptableFormat(workflowName, vbindings, inputVariables); getData = postWithSpecifiedMediaType("users/" + getUsername() + "/" + domain + "/plan/getData", toPost, "application/json", "application/json"); @@ -572,7 +581,7 @@ public String runWorkflow(String wflowname, List vbindings, Map try { vbindings = addDataBindings(inputVariables, vbindings, getData, false); - toPost = toPlanAcceptableFormat(wflowname, vbindings, inputVariables); + toPost = toPlanAcceptableFormat(workflowName, vbindings, inputVariables); getParams = postWithSpecifiedMediaType( "users/" + getUsername() + "/" + domain + "/plan/getParameters", toPost, "application/json", "application/json"); @@ -593,13 +602,13 @@ public String runWorkflow(String wflowname, List vbindings, Map try { vbindings = addDataBindings(inputVariables, vbindings, getParams, true); - toPost = toPlanAcceptableFormat(wflowname, vbindings, inputVariables); + toPost = toPlanAcceptableFormat(workflowName, vbindings, inputVariables); // TODO: This should be called after getting expanded workflow. // - Create mapping data from expanded workflow, and then check. // - *NEEDED* to handle collections properly - String runid = getWorkflowRunWithSameBindings(wflowname, vbindings); + String runid = getWorkflowRunWithSameBindings(workflowName, vbindings); if (runid != null) { System.out.println("Found existing run : " + runid); return runid; @@ -648,7 +657,7 @@ public String runWorkflow(String wflowname, List vbindings, Map // Run the first Expanded workflow List formdata = new ArrayList(); - formdata.add(new BasicNameValuePair("template_id", wflowname)); + formdata.add(new BasicNameValuePair("template_id", workflowName)); formdata.add(new BasicNameValuePair("json", jsonTemplate)); formdata.add(new BasicNameValuePair("constraints_json", jsonConstraints)); formdata.add(new BasicNameValuePair("seed_json", jsonSeed)); @@ -1342,4 +1351,15 @@ public byte[] fetchData(String dataId) { return this.fetchDataFromWings(dataId); } + @Override + public String addData(String url, String workSpaceId, String name, String dType) throws Exception { + // TODO Auto-generated method stub + return null; + } + + @Override + public String duplicateWorkflow(String workflowId, String workflowName) throws Exception { + // TODO Auto-generated method stub + return null; + } } diff --git a/server/src/main/java/org/diskproject/server/repository/WriteKBRepository.java b/server/src/main/java/org/diskproject/server/repository/WriteKBRepository.java index d179894..1503052 100644 --- a/server/src/main/java/org/diskproject/server/repository/WriteKBRepository.java +++ b/server/src/main/java/org/diskproject/server/repository/WriteKBRepository.java @@ -1121,6 +1121,7 @@ private List loadBindings(String userDomain, String id, KBObje // Workflow details KBObject workflowobj = kb.getPropertyValue(wbObj, DISKOnt.getProperty(DISK.HAS_WORKFLOW)); if (workflowobj != null && methodAdapter != null) { + bindings.setWorkflow(workflowobj.getName()); String link = methodAdapter.getWorkflowLink(workflowobj.getName()); if (link != null) diff --git a/server/src/main/java/org/diskproject/server/util/ConfigKeys.java b/server/src/main/java/org/diskproject/server/util/ConfigKeys.java index 265609d..d2723e7 100644 --- a/server/src/main/java/org/diskproject/server/util/ConfigKeys.java +++ b/server/src/main/java/org/diskproject/server/util/ConfigKeys.java @@ -33,4 +33,5 @@ public class ConfigKeys { public static final String DATA_TYPE_GRAPH_DB = "graphdb"; public static final String METHOD_TYPE_WINGS = "wings"; public static final String METHOD_TYPE_AIRFLOW = "airflow"; + public static final String METHOD_TYPE_REANA = "reana"; } diff --git a/server/src/test/java/org/diskproject/ApiClientTest.java b/server/src/test/java/org/diskproject/ApiClientTest.java new file mode 100644 index 0000000..4acc3fc --- /dev/null +++ b/server/src/test/java/org/diskproject/ApiClientTest.java @@ -0,0 +1,84 @@ +package org.diskproject; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.Reader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; + +import org.diskproject.server.adapters.Reana.ApiClient; +import org.diskproject.server.adapters.Reana.ApiClient.ResponseListWorkflow; +import org.diskproject.server.adapters.Reana.ApiSchema.ReanaSpecification; +import org.diskproject.server.adapters.Reana.ApiSchema.ResponseGetSpecification; +import org.diskproject.server.adapters.Reana.ApiSchema.ResponseRunStatus; +import org.junit.jupiter.api.Test; + +import com.google.gson.Gson; + +public class ApiClientTest { + String apiUrl = "http://localhost:30080"; + String apiKey = "qTgcgo9iXWIWHxt3IfLHig"; + ApiClient apiClient = new ApiClient(apiUrl, apiKey, null); + + private ReanaSpecification getSpecificationFromFile() { + Gson gson = new Gson(); + Path path = Paths.get("src/test/resources/bikes/specification.json"); + try (Reader reader = Files.newBufferedReader(path)) { + return gson.fromJson(reader, ReanaSpecification.class); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + @Test + public void createWorkflowTest() throws Exception { + ReanaSpecification specification = getSpecificationFromFile(); + String workflowId = apiClient.createWorkflow(specification, "bikes-rental.1"); + System.out.print(workflowId); + } + + @Test + public void getSpecificationTest() throws Exception { + ResponseGetSpecification responseGetSpecification = apiClient.getSpecification("dev.1"); + assertEquals(responseGetSpecification.specification.inputs.getParameters().size(), 2); + } + + @Test + public void getWorkflowsTest() throws Exception { + ResponseListWorkflow workflows = apiClient.getWorkflows(); + System.out.println(workflows.total); + } + + @Test + public void getRunStatusTest() throws Exception { + ResponseRunStatus runStatus = apiClient.getRunStatus("020d40b9-911e-4bf0-8b40-2e6df92d21aa"); + assertEquals(runStatus.status, "finished"); + } + + @Test + public void startWorkflowTest() throws Exception { + ReanaSpecification specification = getSpecificationFromFile(); + Map parameters = new HashMap(); + parameters.put("databikes", "data/bikes-2019-2020-ny.csv"); + parameters.put("variables", "Temperature Humidity"); + String workflowId = apiClient.createWorkflow(specification, "new workflow"); + apiClient.startWorkflow(workflowId, parameters, null); + return; + } + + @Test + public void addFileWorkspaceTest() throws Exception { + ReanaSpecification specification = getSpecificationFromFile(); + Map parameters = new HashMap(); + parameters.put("databikes", "data/bikes-2019-2020-ny.csv"); + parameters.put("variables", "Temperature Humidity"); + String workflowId = apiClient.createWorkflow(specification, "new workflow"); + apiClient.addFileWorkspace(workflowId, "src/test/resources/bikes/data/bikes-2019-2020-ny.csv", "data/bikes-2019-2020-ny.csv"); + apiClient.startWorkflow(workflowId, parameters, null); + return; + } +} diff --git a/server/src/test/java/org/diskproject/server/adapters/ReanaAdapterTest.java b/server/src/test/java/org/diskproject/server/adapters/ReanaAdapterTest.java new file mode 100644 index 0000000..0b5dc25 --- /dev/null +++ b/server/src/test/java/org/diskproject/server/adapters/ReanaAdapterTest.java @@ -0,0 +1,163 @@ +package org.diskproject.server.adapters; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.IOException; +import java.io.Reader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.diskproject.server.adapters.Reana.ApiSchema.ReanaSpecification; +import org.diskproject.server.adapters.Reana.ApiSchema.ResponseGetSpecification; +import org.diskproject.shared.classes.workflow.Variable; +import org.diskproject.shared.classes.workflow.VariableBinding; +import org.diskproject.shared.classes.workflow.WorkflowRun; +import org.junit.jupiter.api.Test; + +import com.google.gson.Gson; + +public class ReanaAdapterTest { + ReanaAdapter adapter = new ReanaAdapter("reana", "http://localhost:30080", "null", "qTgcgo9iXWIWHxt3IfLHig"); + private Map convertReanaOutputs; + + @Test + public void test() { + System.out.println("Hello World!"); + } + + @Test + // TODO: fix this test + public void getDiskVariablesTest() throws Exception { + List variableBinding = new ArrayList(); + VariableBinding vb1 = new VariableBinding(); + String vb1Binding = "[SHAW86598_bikes-2019-2020-ny.csv, SHAW266eb_bikes-2021-ny.csv]"; + String vb1VariableName = "databikes"; + vb1.setBinding(vb1Binding); + vb1.setVariable(vb1VariableName); + String vb2Binding = "Temperature"; + String vb2VariableName = "variables"; + VariableBinding vb2 = new VariableBinding(); + vb2.setBinding(vb2Binding); + vb2.setVariable(vb2VariableName); + variableBinding.add(vb1); + variableBinding.add(vb2); + } + + @Test + public void getVariablesBindingDiskTest() { + + } + + @Test + public void addDataTest() throws Exception { + String path = this.adapter.addData(null, null, null); + assertEquals(null, path); + } + + @Test + public void addDataWorkspaceTest() throws Exception { + String url = "https://raw.githubusercontent.com/mosoriob/bikes_rent/master/data/bikes-2021-ny.csv"; + String name = "data/bikes-2021-ny.csv"; + String workspaceId = "85cbce19-0ce7-4a09-a80a-142a3fbbe301"; + String path = this.adapter.addData(url, workspaceId, name, null); + } + + @Test + public void downloadData() throws IOException { + String helloUrl = "https://gist.githubusercontent.com/mosoriob/7aa54dab98fb18e4bf861191f0a50032/raw/12cbf08067c97a9dbff3009e7e94693dfa6de045/hello.txt"; + String path = this.adapter.downloadData(helloUrl, "hello.txt"); + byte[] actualFileBytes = Files.readAllBytes(Paths.get(path)); + String actualFileStrings = new String(actualFileBytes, StandardCharsets.UTF_8); + assertEquals("hello", actualFileStrings); + } + + @Test + // TODO: fix this test + public void runWorkflowTest() { + return; + } + + @Test + public void pingTest() { + assertEquals(true, this.adapter.ping()); + } + + @Test + // TODO: mockup + public void getWorkflowList() { + return; + } + + @Test + public void getWorkflowVariablesBindingTest() throws IOException { + Path testResources = Paths.get("src/test/resources/root/specification.json"); + Reader reader = Files.newBufferedReader(testResources); + Gson gson = new Gson(); + ResponseGetSpecification specification = gson.fromJson(reader, ResponseGetSpecification.class); + return; + } + + @Test + public void handleCollectionParameterTest() throws Exception { + String parameter = "[SHAW86598_bikes-2019-2020-ny.csv, SHAW266eb_bikes-2021-ny.csv]"; + String expected = "SHAW86598_bikes-2019-2020-ny.csv SHAW266eb_bikes-2021-ny.csv"; + String path = this.adapter.handleCollectionParameter(parameter); + assertEquals(expected, path); + } + + @Test + public void handleCollectionParameterTest2() throws Exception { + String parameter = "SHAW86598_bikes-2019-2020-ny.csv"; + String expected = "SHAW86598_bikes-2019-2020-ny.csv"; + String path = this.adapter.handleCollectionParameter(parameter); + assertEquals(expected, path); + } + + @Test + public void getWorkflowInputsTest() throws Exception { + String workflowId = "85cbce19-0ce7-4a09-a80a-142a3fbbe301"; + Map workflowInputs = adapter.getWorkflowInputs(workflowId); + System.out.print("test"); + } + + @Test + public void getRunStatusTest() throws Exception { + WorkflowRun runStatus = this.adapter.getRunStatus("020d40b9-911e-4bf0-8b40-2e6df92d21aa"); + assertEquals(2, runStatus.getOutputs().size()); + assertEquals(1, runStatus.getFiles().size()); + return; + } + + @Test + public void getVariablesTest() throws Exception { + Path testResources = Paths.get("src/test/resources/bikes/specification.json"); + Reader reader = Files.newBufferedReader(testResources); + Gson gson = new Gson(); + ReanaSpecification specification = gson.fromJson(reader, ReanaSpecification.class); + List result; + } + + @Test + public void duplicateWorkflowTest() throws Exception { + adapter.duplicateWorkflow("bikes-prod.1", null); + } + + @Test + public void convertReanaOutputsTest() throws Exception { + Path testResources = Paths.get("src/test/resources/bikes/specification.json"); + Reader reader = Files.newBufferedReader(testResources); + Gson gson = new Gson(); + ReanaSpecification specification = gson.fromJson(reader, ReanaSpecification.class); + Map actual = adapter.convertReanaOutputs(specification); + Map expected = new HashMap(); + expected.put("r_squared.txt", "results/r_squared.txt"); + expected.put("summary.txt", "results/summary.txt"); + assertEquals(expected, actual); + } +} diff --git a/server/src/test/resources/bikes/data/bikes-2019-2020-ny.csv b/server/src/test/resources/bikes/data/bikes-2019-2020-ny.csv new file mode 100644 index 0000000..ea2a8df --- /dev/null +++ b/server/src/test/resources/bikes/data/bikes-2019-2020-ny.csv @@ -0,0 +1,512 @@ +season,mnth,holiday,weekday,workingday,weathersit,temp,atemp,hum,windspeed,cnt +spring,4,0,4,1,mist,15.421651,15.916477999999998,75.66669999999999,11.833875,5026 +spring,5,0,0,0,mist,13.228348999999998,13.581464,76.2083,7.125718,3351 +summer,7,0,0,0,mist,25.683349,28.125950000000003,68.25,15.333486,4649 +summer,7,0,2,1,clear,29.325848999999998,32.79215,55.9167,13.417017999999999,4258 +winter,1,0,5,1,mist,1.563466,-1.2610780000000013,79.30430000000001,8.2611,1167 +spring,4,0,1,1,clear,18.946651,19.833314,42.625,25.833257,3115 +summer,8,0,4,1,clear,25.800849,27.209408000000003,51.916700000000006,9.500332,7605 +winter,1,0,5,1,clear,0.34249999999999936,-5.583022,45.7083,23.667214,1543 +spring,6,0,3,1,clear,21.453349,22.791764,47.1667,11.250104,5180 +winter,2,0,0,0,clear,5.405199,2.3037799999999997,40.7826,14.956745,1812 +fall,11,0,2,1,mist,13.541650999999998,13.790750000000003,83.08330000000001,17.292164,2914 +fall,10,0,1,1,clear,14.912499999999998,15.874171999999998,56.833299999999994,5.4593811,7058 +summer,9,0,6,0,mist,15.108349,15.581792,71.8333,12.708225,4511 +spring,4,0,5,1,mist,7.784151,5.4156140000000015,83.625,15.208464,1471 +winter,1,0,0,0,clear,-0.16665099999999988,-5.332750000000001,41.9167,16.834286,2311 +winter,1,0,4,1,clear,0.9299999999999997,-3.4574919999999985,49.75,14.750586,3292 +winter,1,0,0,0,mist,9.083466000000001,7.3467740000000035,69.6087,16.652113,801 +summer,8,0,1,1,clear,21.923348999999998,24.125228,55.45830000000001,10.708275,4634 +spring,5,0,3,1,mist,11.465848999999999,10.706900000000001,73.70830000000001,22.042732,2633 +fall,10,0,6,0,clear,18.045848999999997,19.542386,66.41669999999999,17.957675000000002,7965 +winter,3,0,0,0,clear,7.627500000000001,5.499499999999998,47.375,13.917307,2471 +fall,12,0,4,1,clear,6.6875,4.166564000000001,52.4583,14.750586,3727 +fall,9,0,1,1,clear,16.165849,17.165858,49.2917,9.541068000000001,7436 +summer,9,0,0,0,mist,24.743349000000002,26.834000000000003,81.5,4.2927436,5810 +winter,1,0,6,0,clear,6.844150999999998,5.541013999999997,54.333299999999994,14.125542999999999,4023 +winter,1,0,1,1,clear,-3.4226089,-8.21662,49.173899999999996,10.60811,1416 +fall,9,0,1,1,mist,22.510849000000004,22.876771999999995,84.8333,7.4169,4630 +spring,5,0,5,1,clear,17.066650999999997,18.374978,36.0417,15.874779,7030 +summer,7,0,6,0,clear,32.498349,37.124258,49.2083,10.958118,4840 +spring,5,0,4,1,clear,15.774151,16.457678,55.2083,21.042220999999998,6572 +winter,1,0,4,1,clear,1.165000000000001,-1.4998000000000005,68.75,7.627078999999999,431 +summer,9,0,3,1,clear,17.9675,19.666663999999997,53.6667,14.416725,7591 +fall,12,0,6,0,clear,6.060849000000001,4.499863999999999,61.291700000000006,6.4174811,3614 +winter,1,1,1,0,clear,0.9299999999999997,-3.4162420000000004,52.25,15.500986000000001,2298 +summer,6,0,5,1,clear,31.205849,35.916458,48.875,11.082939000000001,5463 +summer,8,0,3,1,clear,25.213349,27.166441999999996,61.9583,11.374657000000001,7347 +winter,1,0,3,1,mist,4.885849,2.6661859999999997,84.75,8.791807,2177 +spring,5,0,4,1,clear,19.886651,21.792458000000003,52.0,15.374825000000001,7384 +summer,7,0,4,1,clear,25.644151,27.209077999999998,52.9583,9.833924999999999,7446 +summer,7,0,4,1,clear,28.620849,32.8334,58.3333,11.958093,4390 +spring,5,0,1,1,clear,19.142500000000002,20.333792000000003,78.79169999999999,8.500357000000001,3958 +winter,1,0,1,1,clear,4.650848999999999,1.3332499999999996,40.0833,14.458064,3624 +spring,6,0,3,1,clear,28.464151,32.000414,62.20830000000001,9.166739,4401 +summer,8,0,0,0,mist,23.803349,25.209608000000003,81.75,14.916411,3820 +summer,7,0,0,0,clear,27.054150999999997,30.542936000000005,71.7917,11.166689,6031 +summer,7,0,4,1,clear,28.503349,32.791358,59.66669999999999,19.082470999999998,6861 +spring,5,0,2,1,mist,23.020000000000003,23.917658000000003,74.0833,13.875164,4492 +fall,10,0,1,1,mist,10.055848999999998,9.875036000000001,76.0833,5.5841686,3570 +winter,12,0,1,1,mist,2.144150999999999,-1.2498579999999997,57.75,10.374682,2729 +summer,7,0,0,0,mist,23.3725,25.126250000000006,76.25,6.2926936,7410 +summer,8,0,3,1,clear,23.646651,25.625672,60.5,16.958236,5130 +spring,5,0,3,1,clear,18.515849,19.501136000000002,79.7083,9.249885999999998,6169 +fall,12,0,0,0,clear,7.549151,7.040600000000001,77.5833,5.6252061,3485 +summer,8,0,4,1,clear,27.3675,30.667808,65.95830000000001,8.666718,7261 +spring,5,0,3,1,clear,23.059151,24.625772000000005,69.625,10.333611000000001,4978 +summer,6,0,2,1,clear,21.649151,23.250464,37.3333,23.292014,7442 +summer,7,0,5,1,mist,25.330849,28.251878000000005,75.79169999999999,15.083643,4040 +summer,8,0,2,1,clear,26.231651,27.9593,62.0,12.791975,7040 +spring,6,0,5,1,clear,22.510849000000004,23.458892,46.791700000000006,11.750661000000001,7736 +fall,11,0,3,1,clear,10.8,10.999214000000002,75.83330000000001,4.1671186,4109 +winter,1,0,6,0,clear,-5.2208711999999995,-10.7814064,40.0,11.52199,981 +summer,9,1,1,0,mist,23.646651,25.292636,79.0417,14.250632,3351 +spring,4,0,2,1,mist,11.465848999999999,10.2911,64.2083,26.000488999999998,1795 +summer,8,0,6,0,clear,29.286651,33.583622,61.3333,17.249686,6824 +winter,2,0,0,0,clear,-2.0075000000000003,-9.290572000000001,46.4583,27.417204,1529 +spring,3,0,1,1,mist,12.230445,11.04251,73.73910000000001,19.348461,2077 +fall,10,0,2,1,mist,18.633349000000003,20.042336,80.875,9.583814,4563 +fall,10,0,5,1,clear,16.009151,17.290664,68.41669999999999,1.5002439,4985 +fall,11,0,0,0,clear,11.779150999999999,11.833058000000001,65.91669999999999,8.5425,6852 +fall,11,0,4,1,mist,9.194151000000002,8.416172,58.166700000000006,10.542182,5986 +summer,7,0,4,1,clear,28.189999999999998,31.166372000000003,60.0417,11.083743,6591 +winter,2,0,1,1,clear,2.4574999999999996,-0.9577419999999996,41.125,11.207961,3422 +spring,4,0,0,0,mist,12.014151000000002,11.540942000000001,67.625,11.541889,6041 +summer,9,0,6,0,mist,22.980849,24.333986000000003,79.91669999999999,18.833968000000002,5976 +summer,7,0,5,1,clear,30.931651000000002,33.667178,45.0833,9.791514,6207 +spring,6,0,2,1,clear,24.351650999999997,27.209672000000005,69.0,9.917139,6825 +spring,6,0,5,1,clear,22.040849,23.583764000000002,58.958299999999994,11.833339,7665 +summer,9,0,3,1,clear,26.623348999999997,30.792878,74.125,12.583136,7112 +spring,4,0,2,1,clear,20.591651,23.500141999999997,39.0417,18.333143,6691 +summer,7,0,3,1,clear,26.035849,27.167564,45.0,11.0416,8173 +winter,3,0,0,0,clear,7.314150999999999,3.9993859999999977,40.3333,22.416257,3423 +winter,2,0,2,1,clear,8.645849000000002,7.832863999999997,49.625,9.874393000000001,4375 +winter,2,0,5,1,clear,8.136651,7.207514000000003,63.4583,13.791682,4154 +spring,5,0,3,1,clear,17.4975,18.8744,63.29169999999999,8.083014,4182 +spring,6,0,0,0,clear,20.317500000000003,21.958778000000002,49.3333,12.333829,7641 +summer,9,0,4,1,clear,22.785,24.584786,63.916700000000006,9.500332,5115 +summer,8,0,3,1,mist,27.25,30.709322,67.29169999999999,7.4169,7534 +spring,6,0,0,0,mist,24.860849,26.625836,66.6667,6.834,4744 +winter,1,0,3,1,clear,2.6669789999999995,-0.8681800000000006,43.695699999999995,12.522300000000001,1600 +summer,8,0,2,1,clear,22.119151,24.000422,45.5417,9.833121,5895 +summer,8,0,5,1,clear,25.409151,27.333422,63.083299999999994,12.374632,4866 +summer,7,0,6,0,clear,27.524150999999996,30.041863999999997,61.3333,10.542449999999999,6685 +winter,2,0,6,0,light precip,2.5358490000000007,-2.0827779999999994,73.125,19.416332,2169 +fall,11,0,2,1,light precip,11.583349000000002,11.831936000000002,96.25,7.959064,1607 +summer,7,0,0,0,clear,29.874150999999998,32.166536,48.0833,11.042470999999999,4302 +fall,11,0,2,1,mist,9.585849,9.124022,68.5,5.542575,5634 +summer,8,0,5,1,mist,25.644151,28.084172000000002,71.58330000000001,16.000471,5786 +fall,11,0,5,1,mist,10.956651,10.623871999999999,62.25000000000001,18.209193,4046 +fall,10,1,1,0,clear,18.829151000000003,19.83305,73.375,2.8343814,5117 +spring,4,0,6,0,light precip,12.249151000000001,12.082472,88.83330000000001,22.834136,795 +fall,10,0,0,0,clear,13.815849,14.124313999999998,51.0,11.125618000000001,6824 +fall,11,0,1,1,clear,7.000848999999999,4.33295,49.416700000000006,15.833775000000001,5259 +fall,10,1,1,0,mist,10.016651,9.582128,70.9583,12.708492999999999,5478 +spring,4,0,3,1,mist,13.776651000000001,14.164508000000001,56.9167,11.250104,4367 +fall,10,0,1,1,clear,13.776651000000001,14.166422,77.2083,7.959064,4187 +winter,3,0,4,1,clear,18.2025,19.166449999999998,57.958299999999994,10.042161,6192 +spring,6,0,4,1,mist,21.531651,23.292836,68.8333,13.833556999999999,3767 +spring,5,0,6,0,mist,21.218349,22.584128,75.66669999999999,10.250464,6883 +spring,4,0,0,0,mist,12.053349,12.164641999999997,85.75,9.833389,2895 +winter,3,0,4,1,clear,14.834151000000002,15.374485999999997,61.5417,15.208129,4990 +spring,6,0,4,1,clear,22.471651,25.209278000000005,56.958299999999994,17.000111,7363 +spring,5,0,1,1,mist,20.121651,21.334022000000004,81.125,15.624668,4359 +fall,10,0,4,1,mist,14.09,14.165036,81.2917,13.250121,2659 +spring,6,0,2,1,mist,22.706651,23.45975,83.3333,14.374581999999998,4972 +spring,5,0,2,1,mist,20.983348999999997,22.417214,69.7083,22.958689,4451 +winter,1,0,3,1,light precip,2.2225,-2.5624000000000002,86.25,19.68795,506 +summer,9,0,5,1,mist,22.55,22.210435999999994,89.79169999999999,8.333393000000001,3544 +spring,4,0,4,1,clear,12.445,12.456758,37.416700000000006,14.708443,6457 +winter,1,0,0,0,clear,-3.4634801,-9.4766194,43.6522,16.5222,986 +summer,8,0,4,1,clear,25.213349,27.083413999999998,59.0417,5.1668189,7713 +summer,7,0,0,0,clear,31.009999999999998,36.458714,55.0833,11.334456999999999,3606 +summer,9,0,3,1,mist,19.965,20.335178000000006,90.0,6.4590814,4352 +summer,7,0,0,0,clear,30.344151000000004,33.541514,51.87500000000001,11.291443000000001,5531 +fall,10,0,4,1,mist,16.557499999999997,17.83325,72.8333,15.874779,7509 +winter,2,0,4,1,clear,-1.2156440000000002,-6.129832,43.7391,14.869645,1538 +fall,10,0,2,1,light precip,19.769151,19.793977999999996,87.16669999999999,6.9998249999999995,4639 +summer,6,0,6,0,clear,24.665,26.458658,48.3333,14.041257,5202 +fall,9,0,5,1,mist,20.630849,18.460249999999995,97.25,5.2505689,2395 +winter,3,0,1,1,clear,7.285198999999999,5.912000000000003,49.695699999999995,9.174042,2046 +winter,2,0,4,1,clear,12.484151,12.291428,50.5,15.416968,2475 +spring,4,0,3,1,mist,11.3875,11.540678,81.91669999999999,16.791338999999997,2162 +summer,8,0,1,1,clear,25.056651000000002,27.209408000000003,73.04169999999999,8.625110999999999,6917 +summer,7,0,3,1,clear,28.425000000000004,29.584022000000004,40.291700000000006,12.292020999999998,4656 +fall,9,0,3,1,mist,21.845,21.960427999999993,84.875,9.958143000000002,3907 +fall,11,0,4,1,mist,7.1183489999999985,5.416471999999999,62.0417,10.250129000000001,5445 +summer,8,0,0,0,clear,28.150849,32.251214000000004,65.25,19.458206999999998,5464 +spring,5,0,2,1,clear,28.425000000000004,31.875278,63.6667,7.459042999999999,3982 +fall,11,0,2,1,mist,16.91,17.500214,68.875,13.375411,4195 +summer,9,0,0,0,clear,19.259999999999998,21.166249999999998,56.99999999999999,6.042281099999999,7333 +fall,11,0,6,0,clear,5.081650999999999,0.4169719999999977,40.4583,25.250357,2277 +fall,12,0,6,0,clear,7.235849,6.333278,65.0417,7.12545,5047 +winter,1,0,6,0,clear,1.2365340000000007,-1.9996840000000002,65.17389999999999,9.739455,1098 +spring,4,0,5,1,mist,6.1,2.7079640000000005,68.625,17.333436,2227 +summer,7,0,4,1,clear,27.25,29.333486,65.125,10.666400000000001,4592 +summer,9,0,4,1,clear,20.7875,22.250828,63.708299999999994,5.5422936,7804 +summer,7,0,2,1,clear,28.268349,30.000614,54.0833,13.417285999999999,4590 +spring,3,0,4,1,mist,5.395,1.8749779999999987,80.58330000000001,16.333729,1865 +winter,1,0,3,1,clear,0.11816899999999997,-5.408782,59.954499999999996,20.410009,1162 +fall,12,0,1,1,clear,3.201651,1.8329360000000001,67.08330000000001,4.251150000000001,3310 +fall,10,0,2,1,mist,17.0275,18.499586,70.1667,7.375829,4748 +summer,7,0,0,0,clear,27.1325,29.541649999999997,57.8333,12.292557,4881 +winter,3,0,6,0,mist,11.465848999999999,11.290472000000001,62.125,10.792293,4066 +fall,10,0,3,1,mist,14.755849000000001,15.207836,72.04169999999999,9.959014,3894 +summer,9,0,4,1,mist,24.743349000000002,27.251714000000007,81.0417,9.542207,6203 +summer,9,0,5,1,mist,22.236651000000002,23.917327999999998,72.70830000000001,9.375243,4727 +fall,11,0,4,1,mist,8.058349,5.332585999999999,57.5833,20.459254,3053 +winter,12,0,1,1,mist,2.871288,1.0874000000000024,79.1304,5.1744368000000005,920 +summer,9,0,5,1,clear,21.766651000000003,23.209478000000004,67.25,6.9588209999999995,8009 +spring,4,0,0,0,light precip,10.643349,9.707264000000002,83.54169999999999,23.084582,1027 +spring,5,0,5,1,mist,16.0875,16.623800000000003,86.33330000000001,12.041575,4105 +winter,1,0,6,0,mist,0.14665099999999853,-4.45825,83.125,14.917014,1301 +summer,8,0,3,1,clear,23.3725,25.042364,67.7083,4.708981100000001,7375 +winter,3,0,6,0,clear,7.470848999999999,5.499499999999998,59.458299999999994,14.791924999999999,2132 +winter,3,0,3,1,mist,9.165199000000001,8.217379999999999,77.65220000000001,13.608839,2192 +winter,1,0,4,1,clear,-0.24499999999999922,-6.041722,47.0417,20.166999999999998,1406 +winter,2,0,0,0,clear,5.120849,1.708327999999998,41.0,13.750342999999999,3389 +winter,12,1,2,0,mist,5.691288,3.43469,73.47829999999999,11.304642,1013 +summer,7,0,5,1,clear,28.738349,32.458322,59.458299999999994,10.250464,6904 +winter,12,0,2,1,mist,7.275,5.6237780000000015,76.25,12.62615,1162 +summer,7,0,4,1,mist,30.305,38.540486,69.125,14.875407,3784 +fall,10,0,0,0,clear,16.048348999999998,17.208163999999996,48.6667,18.875039,5041 +spring,4,0,3,1,clear,19.965,21.249872000000003,61.4167,16.208975,3944 +summer,9,0,2,1,mist,18.398349,19.126322000000002,88.125,9.041917999999999,3641 +fall,9,0,5,1,mist,21.100849,22.666958,69.0,10.999993,7415 +winter,2,0,2,1,clear,8.880849000000001,7.3326499999999974,39.5833,12.958939,4363 +fall,11,0,5,1,clear,9.311651000000001,8.999413999999998,56.875,9.917407,3910 +winter,3,0,1,1,clear,3.4366509999999995,-0.08271400000000106,50.625,15.333486,3333 +spring,4,0,6,0,mist,13.620000000000001,13.707985999999998,88.79169999999999,15.458575000000002,4036 +spring,6,0,3,1,clear,22.824151,24.333722,58.2083,22.999693,7421 +fall,11,1,4,0,clear,9.546651,8.583086000000002,54.9167,11.209368000000001,1495 +summer,9,0,1,1,mist,19.299151000000002,20.5013,73.4583,10.166713999999999,6869 +spring,4,0,4,1,clear,10.682500000000001,9.581864,46.625,19.458743,5409 +fall,10,0,0,0,mist,11.544151,11.707657999999999,70.8333,9.457854000000001,3510 +spring,4,0,5,1,clear,10.956651,9.790621999999999,37.7083,20.125996,6460 +spring,5,0,0,0,clear,16.831651,18.249578,63.1667,5.0007125,4333 +spring,5,0,4,1,clear,22.785,24.0422,71.6667,11.584031999999999,6770 +winter,2,0,0,0,clear,5.434151,3.2502860000000027,56.833299999999994,9.5006,1623 +summer,8,0,4,1,clear,27.524150999999996,30.167528000000004,62.0417,10.458699999999999,7286 +spring,4,0,6,0,mist,8.0975,6.540914000000001,87.75,8.916561,2455 +spring,3,0,1,1,clear,4.424356000000001,0.9998840000000015,30.2174,14.217668,2028 +winter,3,0,6,0,clear,14.2075,14.790650000000003,37.9167,24.667189,3117 +spring,6,0,6,0,clear,25.409151,26.792222000000002,43.7083,9.667229,7498 +summer,7,0,4,1,clear,30.8925,34.250222,45.7917,12.999943,6241 +spring,5,0,2,1,clear,25.957500000000003,28.417472000000004,68.45830000000001,19.7918,5743 +fall,9,0,2,1,clear,17.85,19.915813999999997,56.99999999999999,15.833507,7538 +fall,11,0,5,1,clear,6.021651,5.375222000000001,64.95830000000001,3.9175436,5668 +winter,1,1,1,0,clear,4.833020999999999,0.6520639999999993,38.130399999999995,22.087555,1951 +fall,11,0,4,1,clear,5.200890000000001,3.695852000000002,55.565200000000004,7.739974,5323 +spring,3,0,4,1,clear,18.045848999999997,19.083422,83.125,7.876654,6871 +winter,3,0,4,1,clear,1.321651000000001,-2.7912219999999994,31.833299999999998,15.125518000000001,1685 +fall,11,0,0,0,mist,13.776651000000001,14.165828000000001,68.45830000000001,12.45865,3520 +summer,7,0,6,0,clear,26.701651,28.042327999999998,44.4583,7.709154,5119 +spring,4,0,1,1,clear,12.393911,12.215857999999997,50.4348,20.913313,5936 +winter,2,0,5,1,mist,1.9312880000000003,-0.913257999999999,58.521699999999996,8.565213,1708 +spring,5,0,5,1,clear,18.515849,20.373985999999995,52.33330000000001,9.166739,7639 +winter,2,0,5,1,clear,16.518349,17.790878,51.6667,17.749975000000003,2927 +summer,7,0,2,1,clear,27.093348999999996,29.958308000000002,59.0417,8.459286,4665 +summer,7,0,3,1,clear,27.093348999999996,29.500664,63.1667,9.790911000000001,4342 +winter,3,0,6,0,mist,10.055848999999998,8.999413999999998,78.91669999999999,16.875357,2077 +summer,8,0,3,1,clear,22.863349,24.333986000000003,59.791700000000006,5.5833311,5058 +summer,8,0,4,1,clear,24.704151000000003,26.042528000000004,63.5833,5.6679186,7765 +spring,3,0,4,1,light precip,4.611651,0.9999499999999983,91.8333,14.582282000000001,1685 +winter,3,0,5,1,mist,12.484151,12.791114,84.20830000000001,7.583864,4378 +fall,12,0,6,0,clear,4.925000000000001,1.5831920000000004,50.74999999999999,15.625807,3190 +spring,6,0,1,1,clear,21.845,23.709164,49.4583,20.45845,5020 +fall,12,0,5,1,clear,5.2383489999999995,3.416672000000002,64.29169999999999,8.792343,5611 +summer,7,0,5,1,clear,31.871651,39.499136,58.0417,8.9177,3387 +fall,10,0,6,0,mist,11.27,11.248957999999998,75.375,19.583832,2429 +summer,8,0,5,1,mist,25.0175,26.708600000000004,61.5,4.8337686,7582 +spring,4,0,2,1,clear,11.426651,10.748678000000002,45.4167,16.708661,5633 +spring,6,0,6,0,clear,26.074999999999996,28.750508000000004,65.45830000000001,10.374949999999998,4966 +summer,9,0,4,1,clear,17.693349,19.124671999999997,61.8333,7.917189,7720 +spring,6,0,1,1,clear,23.881650999999998,25.042628,60.0,8.167032,4548 +spring,5,0,3,1,clear,21.923348999999998,23.33435,69.7917,8.208304,7424 +winter,2,0,6,0,mist,4.415849,1.9998499999999986,77.95830000000001,8.167032,2832 +spring,5,0,0,0,clear,20.7875,22.625708000000003,57.62500000000001,15.082839,6118 +fall,10,0,3,1,clear,17.301650999999996,18.791108,64.79169999999999,11.87575,4826 +spring,5,0,1,1,mist,17.2625,18.791372000000003,66.41669999999999,15.458307000000001,6273 +fall,12,0,2,1,clear,5.277499999999998,3.875108000000001,59.0,9.41685,3523 +summer,7,0,6,0,light precip,20.004150999999997,20.294191999999995,86.5417,14.2911,4459 +spring,5,0,4,1,clear,13.580849,13.166522,44.4167,19.791263999999998,4433 +winter,12,0,5,1,clear,9.546651,8.915858,68.625,18.374482,2209 +summer,7,0,3,1,clear,28.111651000000002,33.2921,70.70830000000001,7.625739,4332 +fall,11,1,4,0,clear,7.98,7.124485999999997,58.0417,3.5423436,2425 +spring,5,0,2,1,mist,20.748348999999997,22.042664000000002,79.45830000000001,9.875264,5115 +fall,10,0,1,1,light precip,12.68,13.000399999999999,88.0,23.9994,22 +winter,3,0,6,0,mist,16.165849,17.333036,75.5833,7.417168,7836 +winter,2,0,0,0,clear,6.876534000000001,5.391458,45.7391,17.479160999999998,1589 +fall,11,0,0,0,clear,8.0975,6.291235999999998,69.2917,15.250004,4669 +spring,3,0,6,0,mist,15.617499999999996,16.124378,88.5417,12.916461,3372 +winter,2,0,2,1,mist,1.032178,-0.52102,82.9565,3.565271,1360 +summer,9,0,1,1,clear,19.416650999999998,21.333163999999996,50.375,17.333771000000002,7525 +summer,7,0,6,0,clear,31.910849,37.082941999999996,50.0,8.791807,3285 +fall,10,0,0,0,clear,7.000848999999999,5.207713999999999,62.375,11.833339,3331 +summer,9,0,4,1,light precip,21.793910999999998,20.653826000000002,93.9565,12.914116,1842 +fall,11,0,3,1,clear,7.275,4.540586000000001,61.3333,18.167586,3613 +summer,9,0,2,1,clear,22.589151,23.834564,71.25,9.500868,4763 +summer,8,0,2,1,clear,28.425000000000004,31.791986,57.04169999999999,10.125107,4602 +fall,9,0,0,0,mist,21.805849000000002,21.794041999999997,84.5,3.3754064,5010 +winter,2,0,0,0,mist,5.160000000000002,1.5420080000000027,51.5833,16.958504,2689 +summer,8,0,6,0,mist,23.96,25.946696000000003,85.0,25.166338999999997,1115 +spring,5,0,0,0,clear,21.179150999999997,22.541822000000003,53.041700000000006,17.042589,7129 +fall,11,0,4,1,mist,9.86,8.665586000000001,81.3333,12.667489000000002,2933 +fall,12,0,1,1,mist,12.484151,12.74795,92.5,12.750636,5170 +fall,10,0,5,1,mist,18.476651,19.501136000000002,81.5,9.041917999999999,5424 +winter,12,0,0,0,clear,3.554150999999999,1.1250859999999996,51.5417,8.916561,1787 +winter,12,0,3,1,light precip,3.4366509999999995,-1.4580219999999997,82.3333,21.208582,441 +fall,12,0,2,1,clear,11.309151,11.040728000000001,66.625,14.834067999999998,5557 +winter,1,0,2,1,clear,8.0975,7.041127999999997,83.58330000000001,8.292389,4339 +summer,9,0,6,0,clear,22.55,24.292208000000002,64.6667,19.000061,8395 +winter,1,0,5,1,mist,11.974999999999998,11.415278,74.125,22.958689,3456 +spring,4,0,5,1,clear,13.502500000000001,13.874042,40.0833,23.291411,6233 +summer,8,0,2,1,mist,26.584151,30.042986000000006,70.375,7.8328359999999995,7273 +winter,2,0,1,1,clear,5.282622999999999,3.564116000000002,62.2174,10.304599999999999,3784 +winter,2,0,4,1,clear,0.7869790000000005,-4.260052,43.7826,18.609384,1550 +winter,2,1,1,0,mist,6.2566510000000015,2.7489500000000007,60.5,20.625682,1107 +winter,3,0,0,0,mist,14.2075,14.624000000000002,81.0,8.501161,5892 +fall,11,0,3,1,clear,8.606651,8.082871999999998,61.375,6.917482000000001,5146 +fall,11,0,2,1,mist,8.136651,5.3328500000000005,66.29169999999999,22.917082,4094 +winter,2,0,2,1,clear,4.506089000000001,0.7820840000000011,31.434800000000003,19.522058,1815 +summer,8,0,6,0,clear,24.7825,26.833736000000002,67.41669999999999,6.999289,5191 +summer,7,0,3,1,clear,29.286651,33.208478,57.7083,9.208614,5713 +fall,10,0,2,1,mist,12.993348999999998,12.915392,76.16669999999999,12.7501,6392 +spring,4,0,5,1,mist,7.823349,5.248964000000001,72.9583,14.707906999999999,1683 +spring,5,0,1,1,mist,17.810848999999997,19.166978,73.0,12.291418,4401 +winter,1,0,3,1,mist,5.732177999999999,3.695852000000002,74.1739,13.957239,1650 +summer,6,0,5,1,clear,26.035849,27.334478000000004,57.3333,14.875675,4991 +summer,6,0,3,1,clear,24.7825,26.292272000000004,36.0,18.208925,7335 +winter,12,0,3,1,mist,12.131651000000002,12.249122,85.83330000000001,14.8338,2660 +summer,6,0,5,1,clear,28.542499999999997,31.791986,57.375,12.250414000000001,5823 +fall,12,0,2,1,mist,8.606651,6.331958,59.66669999999999,19.834478999999998,5501 +fall,11,0,2,1,mist,5.708349,2.5828279999999992,78.6667,15.916654,3959 +fall,12,0,2,1,mist,10.134151,10.165963999999999,59.5417,4.1252436,3750 +spring,5,0,0,0,mist,18.4375,19.376000000000005,86.70830000000001,10.249593,4553 +fall,9,0,6,0,mist,20.513348999999998,21.251192000000003,86.25,5.2516811,5423 +winter,3,0,5,1,mist,11.309151,10.207478000000002,40.708299999999994,27.7916,4569 +fall,12,0,3,1,mist,5.9825,3.624307999999999,53.833299999999994,10.916779,5319 +winter,2,0,3,1,mist,4.063348999999999,1.583786,72.2917,8.959307,2802 +winter,12,0,4,1,mist,3.945848999999999,-1.0416279999999993,65.29169999999999,23.458911,2114 +summer,6,0,4,1,clear,27.210849000000003,28.583792000000003,42.25,11.50055,6879 +winter,3,0,4,1,clear,11.504999999999999,11.081978,60.291700000000006,14.041792999999998,2744 +summer,9,0,3,1,light precip,20.160849,19.919114,91.7083,6.500393600000001,1996 +winter,12,0,3,1,clear,6.05911,2.4782839999999986,50.39130000000001,19.695386999999997,2302 +spring,4,0,2,1,mist,15.774151,16.291028000000004,66.5833,10.584057,3204 +fall,12,0,6,0,mist,6.021651,4.915664,80.6667,4.0001814,5191 +fall,12,0,3,1,clear,12.601651,12.248792000000002,48.5,21.709407,5729 +spring,5,0,0,0,clear,23.3725,24.6263,81.875,14.333846,4788 +fall,12,0,4,1,clear,5.904151000000001,3.4164080000000006,48.5833,11.666643,5532 +spring,6,0,4,1,clear,25.604999999999997,26.500172,30.5,19.583229000000003,4968 +summer,7,1,1,0,mist,26.153349,27.917522000000005,63.7917,5.4591064000000005,6043 +summer,8,0,6,0,clear,23.881650999999998,24.792686000000003,60.3333,11.917089,7865 +fall,12,0,6,0,mist,4.1416509999999995,0.4584860000000006,56.0833,16.292189,2739 +fall,10,0,6,0,clear,15.813348999999999,16.91585,48.3333,17.291561,5217 +spring,6,0,5,1,mist,22.745849000000003,24.125492,75.5417,15.916720999999999,4127 +summer,7,0,2,1,clear,28.699151,30.374899999999997,49.2083,8.457878999999998,6660 +fall,11,0,2,1,clear,11.191651,11.208236,72.16669999999999,4.6255125,4205 +summer,7,0,5,1,clear,25.957500000000003,27.042692000000002,39.625,6.874736,5362 +summer,7,0,2,1,clear,30.461650999999996,33.875078,50.583299999999994,7.666743,6786 +spring,5,0,6,0,clear,18.515849,19.957921999999996,48.0417,8.249911,7429 +summer,8,0,3,1,mist,26.388348999999998,28.875842,65.75,9.084061,3574 +summer,7,0,5,1,clear,31.401651,35.873822,54.25,11.667245999999999,3846 +summer,8,0,0,0,clear,24.939151000000003,26.542214,53.041700000000006,8.208304,6544 +fall,10,0,2,1,clear,14.755849000000001,15.208628000000001,71.0,13.792218,4456 +fall,11,0,5,1,clear,4.885849,1.9995860000000008,41.0,11.291711,3392 +spring,5,1,1,0,clear,26.466651,28.292072000000005,68.5,8.792075,4098 +winter,2,0,3,1,mist,8.184356000000001,6.999020000000002,80.4783,12.000839,1834 +winter,1,0,3,1,clear,5.825849,3.4581860000000013,64.375,10.791756999999999,4270 +spring,4,0,2,1,mist,15.617499999999996,16.541564,73.9167,18.416892999999998,2034 +winter,2,0,2,1,clear,0.5644340000000003,-3.7218219999999995,57.7778,13.110761,1450 +fall,10,0,4,1,clear,15.225849,15.70805,62.083299999999994,9.041917999999999,4765 +spring,3,0,2,1,clear,12.758348999999999,13.082372,62.4583,15.125250000000001,2703 +winter,3,0,5,1,mist,6.876534000000001,4.129999999999999,64.95649999999999,15.60899,1977 +spring,4,0,0,0,clear,13.541650999999998,13.707985999999998,58.7083,7.8328359999999995,6304 +spring,4,1,1,0,clear,23.215849,24.585050000000003,56.166700000000006,19.083543,6370 +spring,6,0,6,0,clear,21.688349,23.250728000000002,50.416700000000006,11.166689,7702 +winter,12,0,0,0,clear,4.914801000000001,2.4774260000000012,68.13040000000001,10.391097,754 +summer,7,0,2,1,clear,28.503349,33.333614,65.0417,8.7502,4541 +spring,5,0,4,1,clear,23.96,25.667714000000004,49.2917,13.083693,7338 +fall,9,0,2,1,mist,21.923348999999998,21.91865,88.5417,7.917457,4120 +summer,9,0,5,1,mist,14.050849,14.457350000000002,59.0417,11.000261,4760 +winter,12,0,5,1,mist,3.9066509999999983,0.8330359999999999,59.0,10.416557,3095 +spring,3,0,0,0,mist,3.893021000000001,0.5223739999999992,49.3913,12.348099999999999,1693 +summer,6,0,3,1,clear,26.231651,27.209408000000003,49.7917,17.542007,5225 +winter,2,0,6,0,clear,8.293349,7.45805,53.4583,12.792243,4318 +fall,10,0,5,1,mist,17.889151000000002,18.958550000000002,71.625,15.000161,3644 +winter,12,0,6,0,clear,4.494150999999999,-0.4165420000000015,44.125,27.292182,1749 +fall,10,0,2,1,clear,14.011651,14.415835999999999,55.833299999999994,12.208807,7534 +winter,2,0,3,1,clear,2.4217329999999997,0.21732199999999935,42.3043,6.3055710000000005,1917 +summer,7,0,5,1,mist,23.294151,24.459650000000003,84.4167,14.000789000000001,5870 +summer,7,0,0,0,clear,30.6575,36.166136,57.375,8.417143,4672 +fall,10,0,3,1,mist,22.9025,23.542778,79.375,4.4585686,7572 +summer,8,0,6,0,mist,24.234151000000004,26.626628000000004,72.9583,14.167418,4150 +spring,4,0,0,0,mist,19.338349,20.416358000000002,81.08330000000001,12.875725000000001,4191 +summer,6,0,6,0,clear,26.388348999999998,27.084272000000006,53.4583,12.041307,7458 +fall,11,0,0,0,clear,13.580849,14.082799999999999,69.8333,13.999918,3071 +summer,9,0,4,1,mist,21.531651,20.627558,90.2083,8.584375,4795 +fall,12,0,4,1,clear,4.024151,1.0414639999999977,50.875,11.708518,5375 +fall,12,0,3,1,mist,6.922499999999999,6.331892,66.375,4.0842061,3740 +winter,1,0,4,1,mist,8.058349,7.499299999999998,76.95830000000001,4.9175186,4075 +spring,4,0,6,0,clear,14.2075,14.625386000000002,50.3333,15.750025,5312 +fall,11,0,6,0,clear,9.664151,9.415742000000002,68.1667,4.5841936,3068 +winter,3,0,0,0,clear,9.001732999999998,7.738219999999998,47.6957,14.913329000000001,4911 +summer,9,0,0,0,clear,22.706651,24.209114,71.375,7.708618,5046 +winter,3,0,5,1,mist,4.298349,0.8748140000000006,61.0417,13.624182,1944 +fall,10,0,3,1,light precip,17.461733,17.913967999999997,89.52170000000001,16.303713,2424 +fall,10,0,6,0,light precip,3.945848999999999,-0.9577419999999996,88.25,23.541857,627 +summer,9,0,5,1,clear,25.056651000000002,27.375464,73.625,11.500282,7504 +winter,1,0,1,1,mist,0.49915099999999946,-3.7074999999999996,60.375,12.541864,1501 +spring,4,0,1,1,mist,7.1183489999999985,3.87425,76.66669999999999,20.334232,3214 +summer,9,0,2,1,mist,21.296651,21.294421999999997,87.25,23.958329,4073 +fall,10,0,4,1,clear,12.445,12.457022000000002,46.3333,12.166932000000001,7570 +summer,7,0,6,0,clear,26.466651,27.834428000000003,60.9167,11.250104,5336 +spring,5,0,6,0,clear,24.5475,26.417936000000005,73.25,13.332464,6536 +spring,6,0,0,0,clear,24.5475,26.459450000000004,74.7917,10.958988999999999,4460 +winter,3,0,5,1,clear,17.380000000000003,18.782593999999996,52.5217,15.478139,3239 +spring,3,0,3,1,clear,14.755849000000001,15.082699999999999,48.125,19.541957,5698 +winter,3,0,2,1,clear,5.747499999999999,3.958400000000001,42.0833,8.083549999999999,2133 +spring,3,0,3,1,mist,16.988349,17.875028,82.125,6.000406099999999,6230 +fall,9,0,5,1,clear,18.515849,19.958714,64.75,13.833825,5202 +fall,11,0,2,1,clear,10.839151000000001,10.207808,68.375,9.083257,4068 +fall,12,0,0,0,clear,3.201651,0.20821399999999812,58.62500000000001,11.375193000000001,2431 +fall,10,0,6,0,mist,16.91,17.998777999999994,72.0,15.791364000000002,7852 +summer,8,0,2,1,clear,22.510849000000004,23.667650000000002,67.375,4.8756436,7006 +winter,1,0,6,0,mist,2.9666510000000006,0.3753920000000015,49.875,10.583521,1248 +spring,4,0,1,1,clear,14.990849,15.458108,31.75,23.999132,5585 +fall,11,0,5,1,clear,9.625,9.124285999999998,64.375,6.6260186,2792 +winter,1,0,6,0,mist,8.175849,7.99925,80.58330000000001,10.749882,985 +winter,2,0,4,1,clear,13.345849000000001,13.333435999999999,55.45830000000001,12.791171,5062 +summer,8,0,3,1,clear,28.033349,29.208878,42.4167,13.417285999999999,4780 +winter,2,0,2,1,clear,5.5278220000000005,3.477458000000002,59.434799999999996,13.783039,3777 +spring,6,0,5,1,clear,27.485,30.417272000000004,60.5,9.417118,4586 +winter,2,0,4,1,mist,5.895644000000001,3.0866059999999997,69.7391,16.783231999999998,1807 +winter,1,0,0,0,clear,2.176534,0.5212520000000005,72.2174,4.956834199999999,1096 +winter,1,0,2,1,clear,1.4000000000000004,-1.9999479999999998,59.0435,10.739832,1562 +summer,8,0,2,1,clear,28.816651,30.666686,49.125,13.79195,4845 +summer,7,0,5,1,clear,23.176650999999996,25.208486,59.12500000000001,12.249811,5538 +fall,10,0,4,1,mist,22.9025,24.126350000000002,72.2917,7.875582,7328 +fall,10,0,3,1,clear,19.6125,20.875586,63.625,4.8762064,7693 +spring,6,0,1,1,mist,18.711651,19.959572,77.79169999999999,11.707982000000001,5099 +winter,2,0,3,1,clear,14.050849,14.791508,50.79169999999999,12.667489000000002,4579 +summer,7,0,5,1,mist,26.388348999999998,28.083578000000003,48.5833,5.41695,7499 +spring,6,0,1,1,mist,21.845,23.292836,74.625,10.416825,4010 +summer,8,0,6,0,mist,23.098349,24.833936,71.2917,16.375336,6053 +winter,1,0,5,1,mist,1.2365340000000007,-2.2166259999999998,49.8696,11.304642,1510 +fall,11,1,1,0,clear,14.794999999999998,15.375278000000002,74.16669999999999,11.625639,6269 +spring,3,0,1,1,clear,12.954151,12.9575,47.7917,25.917007,5558 +fall,11,0,1,1,clear,16.91,18.624392,58.7083,20.541932,4486 +spring,4,0,4,1,mist,21.0225,22.209314,70.08330000000001,21.500836000000003,4058 +fall,11,0,6,0,clear,7.353349,5.374364,51.916700000000006,12.667154,3926 +spring,4,0,1,1,mist,19.995644,21.304322,71.6956,21.739758,3348 +summer,8,0,0,0,mist,21.884151,23.834564,71.1667,5.79215,4549 +fall,11,0,6,0,clear,7.470848999999999,5.415878000000003,50.208299999999994,15.041232,3663 +fall,11,0,0,0,clear,8.371651,7.915628000000002,73.4583,6.1676314,3649 +summer,8,0,5,1,clear,25.996651,-0.0015999999999998238,57.0833,15.500718,7148 +winter,12,0,6,0,mist,3.9066509999999983,-0.0015999999999998238,75.29169999999999,8.333661,1341 +spring,4,0,6,0,mist,6.805,4.832042000000001,65.375,13.208782,2252 +winter,2,0,3,1,mist,4.220000000000001,0.7915220000000005,77.5417,17.708636,1526 +spring,6,0,4,1,mist,29.991650999999997,34.000214,56.833299999999994,10.042161,3915 +fall,11,0,6,0,mist,8.136651,5.373836000000001,49.125,18.125443,5138 +spring,5,0,6,0,mist,16.479150999999998,17.041514,92.25,9.041649999999999,3409 +summer,6,0,6,0,clear,27.955,29.375528000000003,60.12499999999999,10.791756999999999,5687 +spring,6,0,3,1,mist,27.915849,31.583822000000005,67.7083,13.875164,3974 +summer,6,0,1,1,mist,24.0775,26.042264000000003,65.8333,7.2083960000000005,4708 +winter,2,0,0,0,mist,4.494150999999999,1.4583860000000008,68.7917,11.791732000000001,2947 +winter,3,0,0,0,mist,9.696534,8.172632,94.8261,23.000229,605 +fall,11,0,3,1,light precip,13.463349000000001,13.831208,93.0,9.167543,1817 +winter,2,0,3,1,clear,10.604150999999998,9.916021999999998,56.7917,15.709557,4773 +summer,8,0,1,1,mist,27.3675,30.876236000000006,65.4167,8.666718,7013 +spring,4,0,4,1,clear,15.421651,16.541036,61.25000000000001,4.417256399999999,6565 +fall,11,0,1,1,clear,15.663466,16.348052000000003,74.3043,9.522174,3867 +spring,5,0,2,1,mist,19.338349,20.793086000000002,68.58330000000001,19.833942999999998,5728 +fall,12,0,1,1,mist,10.134151,9.99905,82.70830000000001,4.1679561000000005,3811 +fall,11,0,6,0,clear,8.763349000000002,7.624171999999998,55.2917,14.208154,4067 +spring,3,0,5,1,mist,20.278349,21.624422000000003,69.41669999999999,7.7921000000000005,8362 +spring,6,0,0,0,clear,26.153349,27.792122,53.833299999999994,8.959307,6598 +fall,11,0,5,1,clear,8.684999999999999,7.498772000000002,52.208299999999994,17.833725,5847 +summer,7,0,1,1,clear,27.093348999999996,30.459050000000005,65.125,14.458867999999999,4458 +summer,7,0,1,1,clear,26.349151,29.209142,66.875,10.292339,7105 +fall,10,0,6,0,clear,16.518349,17.873971999999995,70.125,3.0420814,5409 +summer,7,0,4,1,clear,23.999151,25.916863999999997,47.625,16.124689,5084 +fall,9,0,3,1,clear,21.845,23.376458,63.083299999999994,16.3748,7733 +winter,1,0,4,1,clear,4.494150999999999,2.3751919999999984,52.416700000000006,8.709128999999999,3272 +spring,4,0,6,0,clear,15.265,16.207735999999997,50.29169999999999,12.791439,7460 +fall,10,0,5,1,mist,17.654151,18.5015,80.7083,8.875289,7444 +winter,1,0,2,1,mist,-0.0527230000000003,-3.3633760000000006,68.6364,8.182844000000001,1263 +fall,10,0,5,1,clear,12.092500000000001,11.957335999999998,57.4167,14.833532,4304 +summer,7,0,1,1,clear,27.876651000000003,31.792250000000003,64.5,11.000529,6830 +summer,8,0,4,1,mist,24.155848999999996,26.626364000000002,77.1667,14.125810999999999,3542 +fall,10,0,2,1,clear,14.168349,14.582749999999997,62.291700000000006,11.166086,4687 +spring,5,0,6,0,clear,16.44,17.832986,54.125,10.750150000000001,4714 +fall,9,0,6,0,clear,17.4975,18.999536,54.2917,15.249468,8555 +spring,3,0,0,0,mist,12.5625,12.874208,88.0833,14.791924999999999,4996 +winter,1,0,0,0,clear,-1.4983489999999993,-8.33245,43.416700000000006,24.25065,822 +summer,8,0,1,1,clear,23.294151,24.667022000000003,71.20830000000001,13.999918,4338 +fall,10,0,3,1,light precip,17.536651,18.169322,90.625,16.62605,2416 +fall,11,0,4,1,clear,11.191651,10.790785999999997,70.20830000000001,9.166739,3974 +winter,12,0,0,0,clear,4.024151,-0.7078000000000007,48.3333,23.500518,1796 +winter,12,0,4,1,mist,11.896651000000002,12.123986000000002,75.75,3.1674249999999997,3068 +spring,4,0,4,1,clear,12.5625,12.623936,60.291700000000006,10.874904,3141 +summer,7,0,1,1,clear,27.8375,32.167064,63.5833,18.916579,4086 +summer,7,0,0,0,clear,25.800849,28.208978000000002,60.4167,16.417211,5302 +fall,11,0,3,1,mist,5.904151000000001,2.124986,54.75,20.375236,5035 +summer,8,0,5,1,clear,24.9,27.542378,76.125,5.625487499999999,4661 +winter,3,0,3,1,clear,18.9075,20.208722,50.708299999999994,7.709154,6312 +fall,10,0,1,1,clear,17.105849,17.70785,57.958299999999994,11.750393,4570 +fall,12,0,5,1,mist,7.0791509999999995,5.249228000000002,76.4167,8.7502,5008 +summer,9,0,3,1,clear,23.646651,25.3754,69.7083,11.2091,4785 +fall,9,0,4,1,clear,20.983348999999997,21.917792,69.91669999999999,11.583161,4839 +fall,12,0,1,1,mist,10.486650999999998,10.499000000000002,90.7083,6.583306100000001,4585 +winter,1,0,6,0,mist,-0.24499999999999922,-5.291236,53.5833,17.875868,959 +fall,10,0,2,1,clear,17.575848999999998,19.000064000000002,64.16669999999999,6.3345686,7466 +winter,2,0,3,1,clear,8.371651,7.207514000000003,53.125,12.167200000000001,4169 +fall,12,0,0,0,clear,2.3791510000000002,0.708164,49.0,4.458293899999999,2743 +spring,4,0,3,1,clear,8.388712000000002,6.260084000000003,46.9565,19.783358,4862 +winter,2,0,5,1,mist,11.1525,11.124085999999998,73.75,15.916989,3487 +summer,9,0,0,0,clear,15.8525,16.375442,69.5,11.958361,4274 +winter,1,0,0,0,clear,7.862500000000001,6.457028000000001,46.5,12.833314,3425 +winter,1,0,4,1,mist,4.298349,0.8333000000000013,53.833299999999994,13.125568,1927 +fall,10,0,6,0,clear,10.486650999999998,9.832136000000002,49.4583,9.791514,7109 +winter,3,0,2,1,mist,6.917376999999998,4.999748,65.5652,12.348703,2056 +spring,5,0,6,0,clear,20.2,21.415928,45.625,5.626325,8294 +summer,8,0,1,1,clear,24.508349000000003,26.124764,47.0,18.54225,4758 +summer,8,0,0,0,clear,26.8975,31.209272000000006,75.29169999999999,13.499629,3785 +winter,2,0,1,1,clear,11.504999999999999,10.2911,37.583299999999994,27.999836,1913 +summer,8,0,6,0,mist,24.5475,26.125622,73.2917,13.834093,6299 +spring,4,0,1,1,mist,13.815849,14.207936,56.99999999999999,11.499746,5572 +summer,8,0,6,0,mist,25.683349,28.626164000000003,75.5,15.29275,4294 +fall,10,0,0,0,mist,14.442499999999999,14.872886000000001,69.4583,26.666535999999997,4459 +fall,12,0,4,1,mist,7.510000000000002,6.124321999999999,66.79169999999999,8.875021,4128 +fall,10,0,3,1,clear,13.424150999999998,13.707128,69.2917,6.791857,7461 +fall,11,0,1,1,mist,9.899151,8.790986,62.3333,15.749489,5499 +winter,1,0,1,1,clear,-0.9108490000000007,-6.041392,48.2917,14.958889,1321 +fall,10,0,2,1,mist,6.954554000000002,4.453993999999998,82.5455,14.271603,1096 +winter,2,0,4,1,mist,10.760849,10.332086,67.29169999999999,12.541528999999999,3761 +spring,3,0,2,1,clear,6.217499999999999,3.3319280000000013,31.4167,15.208732,2425 +summer,7,0,6,0,mist,25.056651000000002,27.958772000000003,69.91669999999999,9.626493,6969 +summer,9,0,2,1,light precip,17.380000000000003,18.0032,88.6957,23.044181,2710 +summer,9,0,6,0,clear,23.020000000000003,24.125492,75.375,10.291736,5345 +winter,2,0,3,1,mist,-1.6651990000000003,-6.477322000000001,49.4783,12.652213,1605 +spring,6,0,5,1,clear,22.510849000000004,23.625278,73.5833,9.582942999999998,4844 +winter,2,0,6,0,clear,5.669151000000001,0.8745499999999993,39.5833,28.250014,2732 +fall,10,0,6,0,clear,11.857499999999998,12.082472,62.916700000000006,6.2086689,4308 +fall,10,0,4,1,clear,14.364151,14.790650000000003,63.625,28.292425,4195 +winter,2,0,4,1,mist,6.883348999999999,5.790692,75.29169999999999,6.125475000000001,3005 +winter,2,0,5,1,clear,0.887277000000001,-1.9084059999999994,50.6364,7.272849999999999,1746 +summer,6,0,0,0,clear,26.936650999999998,28.500764000000004,47.9167,9.750174999999999,6891 +spring,3,0,4,1,clear,15.225849,15.832063999999999,43.9167,21.41655,6133 +fall,12,0,4,1,mist,11.857499999999998,11.207642,63.416700000000006,17.958814,3709 +summer,9,0,6,0,mist,27.406651000000004,30.375163999999998,63.8333,7.5835289999999995,6140 +fall,11,0,3,1,mist,12.719151,12.415441999999999,75.79169999999999,22.500275,2566 +winter,1,0,5,1,clear,-0.43910999999999945,-3.5647420000000007,53.7826,8.478716,1421 +winter,3,0,3,1,clear,10.995849,9.416599999999999,51.3333,23.167193,4916 +spring,6,0,6,0,clear,21.845,23.249935999999998,45.625,8.250514,5342 +summer,9,0,0,0,clear,25.330849,27.918314000000002,74.20830000000001,13.833289,4940 +winter,3,0,0,0,clear,10.064356,9.086006000000001,52.73910000000001,18.130468,2417 +fall,10,0,1,1,mist,16.479150999999998,17.792,64.9167,6.0838814,6778 +spring,4,0,5,1,clear,12.7975,12.499328000000002,40.8333,10.416557,6398 +winter,3,0,6,0,clear,5.512499999999999,2.3326220000000006,35.0417,15.125250000000001,4118 +winter,3,0,4,1,clear,16.792499999999997,18.623863999999998,56.75,29.584721,5382 +fall,10,0,1,1,mist,18.398349,19.5839,70.75,19.834478999999998,5875 +fall,11,0,2,1,clear,5.1991510000000005,2.5834220000000023,56.7083,11.625371,5686 diff --git a/server/src/test/resources/bikes/data/bikes-2021-ny.csv b/server/src/test/resources/bikes/data/bikes-2021-ny.csv new file mode 100644 index 0000000..e9c2b59 --- /dev/null +++ b/server/src/test/resources/bikes/data/bikes-2021-ny.csv @@ -0,0 +1,221 @@ +season,mnth,holiday,weekday,workingday,weathersit,temp,atemp,hum,windspeed,cnt +summer,7,0,1,1,clear,26.936650999999998,31.583228,75.7083,6.0841560999999995,3840 +spring,6,0,3,1,clear,18.045848999999997,19.791272,61.12499999999999,5.167375,7055 +spring,4,0,3,1,clear,10.369150999999999,9.582128,47.0833,17.625221,2808 +spring,5,0,3,1,mist,21.218349,22.584392,77.45830000000001,6.834,5260 +winter,1,0,4,1,mist,9.9775,9.207908,80.2917,12.124789,4097 +summer,9,0,2,1,clear,26.114151,29.334608000000003,75.5,15.833507,6864 +winter,1,0,5,1,clear,7.705848999999999,6.457622000000001,54.2083,11.249836,4098 +spring,5,0,1,1,clear,17.0275,18.666235999999998,58.875,11.792,4362 +fall,10,0,4,1,mist,19.690849,20.419064,89.6667,9.499729,2913 +summer,9,0,3,1,clear,20.160849,21.62495,57.7083,8.833682,7870 +spring,5,0,6,0,clear,22.824151,24.417014,72.9583,15.416164,4758 +winter,12,0,5,1,clear,6.648349000000001,5.041591999999998,63.6667,9.000579,2999 +summer,8,0,0,0,clear,25.231772999999997,26.765293999999997,56.1765,20.412153,4334 +winter,1,1,1,0,mist,0.264151,-4.333113999999998,53.75,12.999139,1000 +winter,2,0,5,1,mist,5.1991510000000005,3.374828000000001,54.0,7.834243000000001,3831 +spring,4,0,3,1,clear,14.403348999999999,15.040921999999998,42.7917,7.959064,6196 +winter,2,0,4,1,clear,4.455,1.291208000000001,56.2083,13.000478999999999,3830 +summer,8,0,4,1,clear,25.722500000000004,27.000386,42.375,11.041332,4792 +fall,11,0,1,1,clear,10.565000000000001,10.457486,75.875,3.834075,4035 +winter,3,0,1,1,clear,13.933349,14.333072000000001,48.9167,13.916771,5298 +winter,1,0,2,1,clear,10.330000000000002,9.166922,41.6667,17.541739,4509 +winter,2,0,6,0,clear,5.277499999999998,2.6246720000000003,53.7917,12.500257,1969 +summer,7,0,0,0,clear,25.918349,28.083578000000003,62.375,11.416532,6597 +fall,11,0,0,0,clear,3.554150999999999,1.000478000000001,46.8333,10.083499999999999,2424 +summer,7,0,1,1,clear,28.738349,30.334508000000007,44.7083,13.082889,6227 +summer,7,0,1,1,mist,25.409151,27.167564,68.3333,12.125325,6569 +summer,8,0,5,1,clear,27.915849,29.5004,58.75,11.291711,7350 +spring,6,0,0,0,mist,22.471651,24.709063999999998,65.25,9.292364000000001,4906 +winter,2,0,6,0,mist,2.9666510000000006,0.041827999999998866,92.91669999999999,10.792293,1005 +spring,6,0,1,1,clear,20.082500000000003,22.166677999999997,48.7083,19.083811,6998 +fall,12,0,3,1,light precip,11.27,10.416235999999998,97.04169999999999,17.833725,705 +winter,3,0,4,1,light precip,10.287277,9.454088000000002,0.0,17.545759,623 +spring,4,0,0,0,clear,20.513348999999998,21.875750000000004,50.79169999999999,15.083643,7132 +summer,7,0,6,0,clear,29.795848999999997,32.083442,46.5833,11.291979,4475 +summer,8,0,4,1,mist,25.369999999999997,27.876008000000006,75.75,13.20905,4576 +summer,9,0,6,0,clear,23.450848999999998,25.792058000000004,71.6667,12.416775,4484 +winter,2,0,1,1,clear,4.768348999999999,4.041428,73.83330000000001,3.0423561,1712 +fall,10,0,5,1,clear,12.5625,12.582685999999999,53.91669999999999,15.751164,7282 +spring,4,0,1,1,clear,16.0875,17.207636,54.25,10.958988999999999,3429 +summer,8,0,1,1,clear,25.879151,27.708764000000002,54.541700000000006,9.126204,6883 +winter,2,0,2,1,mist,7.000848999999999,6.040436,50.875,9.458993,3922 +spring,5,0,5,1,clear,21.492499999999996,23.000522000000004,73.54169999999999,10.916846,6296 +summer,9,1,1,0,clear,25.252499999999998,27.667514000000004,79.0833,10.125107,6034 +winter,2,0,5,1,mist,9.124355999999999,7.130426,71.2174,23.218113,1461 +summer,9,0,1,1,clear,22.284356000000002,23.74058,69.2174,5.957171000000001,4713 +spring,5,0,1,1,mist,18.946651,20.293400000000005,78.95830000000001,14.250364,2843 +spring,4,0,5,1,clear,16.753349,18.041150000000002,69.4583,10.041357,7290 +winter,2,0,0,0,clear,8.143466,7.173193999999999,68.0,8.391615999999999,2402 +spring,3,0,5,1,clear,4.415849,0.9165919999999979,49.5,15.458575000000002,2210 +spring,4,0,6,0,clear,12.5625,12.124513999999998,25.4167,18.416356999999998,6857 +fall,10,0,0,0,clear,11.818349000000001,11.873978000000001,74.125,6.6673375,4381 +summer,7,0,2,1,clear,25.526651,27.751136000000002,70.41669999999999,11.083475,7216 +winter,1,0,3,1,mist,-2.9475,-8.123758,41.4583,12.3749,2368 +winter,2,0,2,1,clear,2.3791510000000002,-2.9157639999999994,53.7917,24.25065,1530 +fall,11,0,4,1,clear,8.552177999999998,6.564806000000001,33.3478,23.304945,5315 +winter,1,0,2,1,mist,9.533021000000002,8.042348,71.6087,23.39171,2935 +fall,10,0,1,1,clear,7.98,7.5001580000000025,70.3333,7.12545,3669 +summer,7,0,6,0,clear,24.273349000000003,26.125358,58.5,13.958914,5923 +spring,6,0,2,1,clear,25.252499999999998,27.292700000000004,59.791700000000006,12.583136,4833 +spring,5,0,6,0,clear,20.317500000000003,21.750349999999997,62.6667,8.083549999999999,5805 +summer,8,0,3,1,clear,25.996651,28.000286000000003,57.5417,9.625689,4694 +summer,6,0,0,0,clear,23.96,26.083514,51.3333,6.3337311,5305 +summer,8,0,5,1,clear,25.291651,27.166772,41.5,8.416607,4905 +summer,8,0,1,1,clear,27.955,30.416677999999997,59.2083,12.875725000000001,4326 +summer,8,0,1,1,mist,21.884151,23.333822000000005,73.4167,8.708593,6530 +winter,1,0,3,1,clear,6.2566510000000015,2.1667640000000006,44.333299999999994,27.833743,3376 +spring,4,0,4,1,clear,13.9725,14.540972,54.041700000000006,7.4169,3267 +fall,10,0,5,1,mist,7.549151,5.041591999999998,58.58330000000001,15.375093,3747 +fall,11,0,1,1,light precip,13.032499999999999,13.374092000000001,91.0,9.249618,2765 +spring,5,0,0,0,clear,20.395849000000002,21.917,74.9583,9.916536,4660 +summer,6,0,4,1,clear,29.874150999999998,33.667772,56.7917,7.958729,5905 +fall,12,0,1,1,clear,13.267500000000002,14.082535999999998,76.75,5.5422936,6234 +fall,12,0,2,1,clear,14.364151,14.957564000000001,73.375,11.666643,6606 +fall,12,0,5,1,mist,9.625,7.748450000000002,50.0417,17.458525,3577 +summer,8,0,2,1,clear,22.040849,23.250464,54.833299999999994,8.375536,5204 +spring,6,0,5,1,clear,21.14,22.750778000000004,35.4167,16.959107,5312 +spring,6,0,6,0,clear,24.743349000000002,26.500964000000003,67.0417,8.000336,5119 +spring,4,0,6,0,mist,9.703349,8.915264,48.9583,8.708325,4220 +fall,11,0,6,0,clear,7.275,5.541277999999998,54.541700000000006,12.041843,5629 +spring,6,0,1,1,mist,25.879151,27.541586000000002,58.791700000000006,13.916771,6664 +fall,11,0,5,1,clear,8.998349000000001,7.4572579999999995,54.0833,14.375386,5992 +winter,2,0,1,1,mist,11.141831,10.407787999999996,87.63640000000001,19.408962,1446 +fall,9,0,0,0,clear,16.870849000000003,18.249578,46.7083,14.958286000000001,7907 +winter,1,0,2,1,mist,2.5034660000000013,-0.5212839999999996,61.695699999999995,8.696332,1985 +winter,1,0,0,0,clear,9.39,8.790986,69.25,12.875189,2294 +winter,2,0,6,0,clear,10.760849,9.832663999999998,18.7917,34.000021,1635 +spring,5,0,5,1,clear,24.038349000000004,26.042528000000004,65.375,16.125493,4679 +summer,7,0,1,1,clear,26.858348999999997,29.541122,69.41669999999999,9.291761,6966 +winter,3,0,3,1,clear,7.745000000000001,5.1246860000000005,44.9583,20.624811,2134 +summer,7,1,3,0,clear,29.090849,32.334242,53.87499999999999,9.041649999999999,7403 +winter,3,0,5,1,mist,8.606651,7.749572000000001,65.7083,9.708568,3194 +spring,5,0,3,1,mist,19.025,20.499650000000003,74.4167,14.499604,4717 +spring,5,0,5,1,clear,17.223349,18.916772,71.9583,8.375871,4917 +summer,8,0,3,1,clear,25.722500000000004,28.042327999999998,67.75,9.458993,7580 +spring,5,1,1,0,clear,25.487500000000004,28.833800000000004,67.625,13.166907,6043 +summer,6,0,2,1,clear,26.975849000000004,29.708827999999997,63.416700000000006,9.666960999999999,4648 +spring,6,0,6,0,clear,19.416650999999998,21.375008,54.9167,12.499654,8120 +fall,9,0,4,1,mist,22.55,24.126350000000002,69.08330000000001,9.000914,7393 +spring,6,0,0,0,clear,19.8475,21.750349999999997,59.875,9.708568,6978 +spring,5,0,3,1,mist,22.863349,24.334514,67.0,9.000043,6855 +summer,9,0,4,1,mist,19.142500000000002,20.542286000000004,70.9167,18.166782,3659 +fall,10,0,5,1,clear,20.905,22.292341999999998,62.74999999999999,7.12545,8156 +summer,8,0,2,1,clear,26.153349,28.667414,68.6667,11.333586,6784 +summer,7,0,3,1,clear,25.683349,26.917886000000003,63.3333,10.166111,7264 +winter,2,0,5,1,clear,6.726650999999999,4.416836,52.6667,11.959232,4151 +spring,4,0,4,1,clear,13.580849,13.956872,40.708299999999994,21.792286,4189 +winter,1,0,0,0,mist,-0.3624999999999998,-5.291499999999999,79.625,13.375746000000001,1977 +summer,6,0,4,1,mist,26.231651,29.792978000000005,70.3333,15.999868,4790 +summer,6,0,2,1,mist,23.999151,26.084636000000003,77.0417,11.458675000000001,4835 +spring,4,0,3,1,mist,21.14,21.959372000000002,83.54169999999999,20.917399999999997,3872 +spring,5,0,4,1,mist,18.320000000000004,19.457971999999998,76.83330000000001,8.957632,6421 +fall,12,0,2,1,light precip,13.7375,14.122399999999999,94.9583,15.583061,2594 +winter,1,0,5,1,clear,4.885849,0.4578920000000011,50.74999999999999,25.333236,3214 +winter,1,0,2,1,clear,-0.9500000000000002,-7.665850000000001,44.125,24.499957000000002,2236 +fall,11,0,3,1,clear,5.590849,2.583158000000001,55.2083,13.374875,5495 +spring,5,0,1,1,mist,21.688349,22.959536,81.0,15.667413999999999,4274 +spring,5,0,0,0,mist,18.4375,20.084642000000002,74.0,10.041893,6359 +fall,10,0,3,1,clear,16.165849,17.207372,63.083299999999994,12.584007,7691 +fall,11,1,5,0,clear,7.235849,4.2499220000000015,44.625,21.083225,3368 +summer,6,0,4,1,clear,24.743349000000002,26.042528000000004,43.416700000000006,12.415904000000001,5515 +summer,8,0,5,1,mist,24.195,25.792586,72.2917,9.333635999999998,4153 +winter,3,0,2,1,clear,4.533348999999999,1.416013999999997,53.5,14.500475,1851 +spring,5,0,0,0,clear,24.43,26.334050000000005,69.7083,14.416457,6591 +spring,3,0,6,0,mist,11.935849000000001,11.832728000000003,73.83330000000001,16.791338999999997,6235 +winter,1,0,1,1,mist,2.5358490000000007,0.33361400000000074,70.1667,6.6263000000000005,2376 +fall,11,0,1,1,clear,6.726650999999999,6.374264,53.541700000000006,3.1255499999999996,5087 +spring,5,0,4,1,clear,25.291651,27.209408000000003,67.75,13.376014,4677 +winter,2,0,1,1,clear,9.233349,7.6249639999999985,49.0833,17.958211000000002,4322 +summer,9,0,6,0,clear,20.591651,22.667222000000002,50.1667,16.583907,8714 +spring,4,0,5,1,clear,15.969999999999999,16.832558,45.7083,16.084221,4595 +winter,12,0,6,0,clear,11.27,11.331986,61.583299999999994,14.750318,2485 +fall,10,0,0,0,clear,16.518349,17.541463999999998,64.0417,18.667004000000002,6639 +winter,1,0,4,1,clear,1.6043559999999992,-0.6082059999999991,51.8261,6.0008684,1606 +spring,5,0,4,1,clear,17.145000000000003,18.541958,74.75,12.707689,4864 +fall,12,0,4,1,clear,4.494150999999999,0.9579079999999998,57.99999999999999,16.083886,3322 +summer,8,0,5,1,mist,27.994151000000002,31.709222000000004,64.25,14.458064,7175 +fall,11,0,3,1,clear,5.943349000000001,3.1242920000000005,50.625,14.125007,5260 +spring,5,0,2,1,clear,17.0275,18.499586,48.9167,7.749956999999999,4803 +spring,4,0,0,0,clear,13.463349000000001,13.415935999999999,47.9583,20.334232,3744 +summer,9,0,2,1,clear,19.142500000000002,20.583272,52.0,6.1676314,7767 +fall,12,0,0,0,mist,9.037499999999998,8.415907999999998,83.875,6.749714,3786 +summer,7,0,2,1,mist,25.879151,27.876536,66.75,10.166379000000001,6290 +winter,3,0,1,1,clear,4.3017330000000005,-0.26157399999999953,55.1304,22.870584,1872 +spring,6,0,2,1,mist,17.419151,18.708872,61.3333,14.041525,7001 +fall,11,0,6,0,clear,10.290848999999998,9.999841999999997,64.5417,3.8756686,6536 +winter,12,0,5,1,mist,7.353349,3.9166220000000003,55.666700000000006,25.083661000000003,3623 +winter,2,0,3,1,clear,6.958267000000001,4.869199999999999,42.3478,16.869996999999998,2115 +winter,1,0,5,1,mist,2.2225,-1.4167720000000017,45.0,13.58425,3163 +spring,5,0,2,1,mist,18.398349,19.542914000000003,83.7917,18.582718,4123 +summer,8,0,2,1,clear,24.939151000000003,26.625242,57.8333,15.834043000000001,4725 +winter,12,0,6,0,clear,6.217499999999999,3.7499719999999996,54.25,12.750368,1011 +spring,5,0,4,1,mist,16.949151,17.708972000000003,82.9583,7.250271000000001,4575 +summer,8,0,3,1,clear,24.195,25.958377999999996,55.2083,7.541653999999999,7697 +summer,8,0,0,0,clear,25.409151,28.584650000000003,77.0,16.666518,3873 +fall,10,0,0,0,clear,17.419151,18.582878,72.75,4.251150000000001,5511 +winter,2,1,1,0,clear,5.160000000000002,2.043806,50.7826,15.348561,3129 +summer,9,0,5,1,clear,20.160849,21.750086000000003,66.875,10.333343000000001,8167 +spring,4,0,1,1,clear,20.513348999999998,21.917,77.6667,12.417311,4073 +spring,5,0,2,1,mist,20.905,22.334450000000004,76.5833,7.917189,6073 +spring,3,0,3,1,mist,8.306979000000002,6.303974,83.9565,15.695487,2121 +spring,6,0,4,1,clear,20.317500000000003,21.583171999999998,56.7083,10.542449999999999,7494 +winter,1,0,6,0,clear,0.4599999999999991,-3.916257999999999,45.75,12.541260999999999,2493 +spring,4,0,3,1,clear,17.458349000000002,19.207699999999996,46.958299999999994,12.125325,6436 +spring,4,0,0,0,clear,15.5,16.50005,27.5833,15.583931999999999,5169 +summer,7,0,2,1,clear,27.289150999999997,30.625700000000002,65.5,14.167418,7592 +fall,11,0,5,1,clear,8.215,6.915464,52.4583,11.458675000000001,5698 +spring,3,0,6,0,clear,4.494150999999999,0.9996860000000005,39.4167,14.041257,2496 +fall,9,0,0,0,clear,16.753349,18.165757999999997,58.3333,9.042186,6889 +spring,4,0,2,1,clear,13.933349,14.457877999999997,39.6667,6.708911,6772 +spring,4,0,0,0,clear,9.781651,8.998622000000001,48.0,12.208271000000002,3249 +winter,1,0,1,1,clear,1.229108,-3.499270000000001,43.7273,16.636703,1349 +winter,1,0,0,0,clear,5.277499999999998,1.9995860000000008,31.125000000000004,16.08335,3243 +fall,10,0,3,1,mist,8.802499999999998,7.832599999999999,66.6667,11.166689,5566 +winter,3,0,2,1,clear,4.1416509999999995,0.8327719999999985,45.6667,13.458625,3956 +fall,12,0,0,0,mist,10.055848999999998,9.749635999999999,90.5417,10.584325,3228 +summer,8,0,4,1,clear,25.448349,27.709028000000004,65.45830000000001,15.624936,3805 +winter,1,0,0,0,clear,2.888349,-0.5416779999999992,48.375,12.625010999999999,1204 +winter,1,0,2,1,clear,6.508712000000001,5.042515999999999,64.65220000000001,12.565984,3598 +winter,1,0,2,1,mist,2.1833489999999998,-0.6660219999999999,86.16669999999999,9.833924999999999,683 +fall,12,0,5,1,clear,6.765848999999999,5.874578,62.583299999999994,6.750518,3940 +spring,4,0,2,1,clear,21.688349,23.209478000000004,72.9167,21.875500000000002,4400 +spring,4,1,5,0,clear,12.993348999999998,13.166258,67.125,15.167124999999999,3126 +fall,11,0,3,1,clear,9.7425,9.748778000000001,71.875,5.5001439,4186 +fall,11,0,0,0,clear,7.314150999999999,5.749507999999999,53.2917,12.000236,5107 +summer,9,0,1,1,mist,17.810848999999997,18.958550000000002,69.0,10.166713999999999,4539 +winter,3,0,2,1,clear,18.359151,19.543178000000005,80.7917,8.125157,6093 +fall,10,0,6,0,clear,14.755849000000001,15.207571999999999,57.2917,7.874979,8090 +winter,1,0,1,1,mist,2.2616510000000005,0.041827999999998866,91.125,7.417436,2432 +fall,12,0,3,1,clear,7.627500000000001,6.5826920000000015,62.5417,12.334164000000001,5267 +winter,12,0,4,1,clear,3.6716510000000007,1.4168720000000015,57.4167,8.000604000000001,2423 +spring,3,0,5,1,mist,9.39,8.790986,58.0833,9.250489,5459 +fall,12,0,6,0,mist,9.938348999999999,9.707528,91.125,6.792393,5582 +summer,6,0,3,1,clear,26.466651,29.792714000000004,70.75,11.541554,4507 +winter,2,0,6,0,clear,2.4574999999999996,-0.6250359999999997,54.4167,13.625589,1472 +spring,3,0,3,1,mist,6.1,3.6250999999999998,64.6667,11.583495999999998,1536 +summer,7,0,3,1,clear,25.839999999999996,29.251778,74.33330000000001,10.042161,4629 +fall,12,0,0,0,mist,8.3325,7.707728000000003,82.3333,8.333393000000001,4649 +spring,5,0,3,1,mist,17.85,18.792428,87.0,13.499964,3855 +spring,5,0,5,1,clear,23.96,25.416914,74.70830000000001,9.41685,6734 +winter,3,0,1,1,clear,17.615000000000002,19.166185999999996,72.875,10.875238999999999,6153 +spring,4,0,6,0,clear,18.79,19.832786,68.2917,19.000328999999997,6624 +fall,10,0,0,0,mist,8.763349000000002,6.790921999999998,79.1667,14.874870999999999,2918 +fall,10,0,4,1,mist,17.85,18.959408000000003,80.0417,8.333125,7359 +fall,12,0,5,1,clear,5.669151000000001,4.9577719999999985,69.58330000000001,5.5420189,3620 +fall,11,0,0,0,clear,12.719151,12.4163,45.8333,18.875307,3717 +summer,8,0,1,1,clear,28.268349,30.417272000000004,55.0833,10.500039,4266 +fall,12,0,1,1,clear,5.003349,2.5415779999999977,63.74999999999999,11.584031999999999,3403 +spring,6,0,2,1,clear,20.395849000000002,23.042036000000003,50.708299999999994,18.041961,4891 +summer,9,0,0,0,clear,20.669999999999998,22.209050000000005,54.75,15.041232,8227 +spring,5,0,2,1,mist,20.826651000000002,22.083385999999997,65.95830000000001,10.458432,5740 +winter,3,0,3,1,mist,5.904151000000001,2.916127999999997,77.5417,14.750050000000002,1891 +summer,6,0,1,1,clear,25.644151,27.166772,50.416700000000006,20.125661,6779 +spring,3,0,2,1,clear,7.196650999999999,4.833164,28.999999999999996,12.541864,5102 +spring,4,0,2,1,clear,12.993348999999998,12.791377999999998,43.5,16.708125000000003,5918 +spring,5,0,5,1,clear,14.520849000000002,15.291722,59.0,15.292482,4608 +summer,8,0,0,0,mist,22.706651,23.335735999999997,84.5833,15.333486,5255 +winter,1,0,6,0,clear,10.486650999999998,9.791414,53.1667,11.708786,4521 +spring,6,0,3,1,clear,28.777499999999996,31.58435,59.2083,7.625404,6211 +winter,12,1,1,0,clear,7.121732999999999,4.825310000000002,50.6957,16.044155,1317 +winter,3,0,2,1,clear,18.554999999999996,19.833314,61.75000000000001,15.87565,5847 diff --git a/server/src/test/resources/bikes/request/createNewWorkflow.json b/server/src/test/resources/bikes/request/createNewWorkflow.json new file mode 100644 index 0000000..63bce3a --- /dev/null +++ b/server/src/test/resources/bikes/request/createNewWorkflow.json @@ -0,0 +1,34 @@ + { + "inputs": { + "files": [ + "data/bikes-2019-2020-ny.csv" + ], + "parameters": { + "databikes": "data/bikes-2019-2020-ny.csv", + "variables": "Temperature Humidity" + } + }, + "outputs": { + "files": [ + "results/r_squared.txt", + "results/summary.txt" + ] + }, + "version": "0.6.0", + "workflow": { + "specification": { + "steps": [ + { + "commands": [ + "mkdir -p results", + "python /usr/src/app/main.py --inputs ${databikes} --variables ${variables} --r_squared results/r_squared.txt --summary results/summary.txt" + ], + "environment": "ikcap/bikes_rent:bcd48921c6e2ed502b84c768b7408e9466a9d576", + "kubernetes_memory_limit": "256Mi", + "name": "predict" + } + ] + }, + "type": "serial" + } + } \ No newline at end of file diff --git a/server/src/test/resources/bikes/specification.json b/server/src/test/resources/bikes/specification.json new file mode 100644 index 0000000..813fbe3 --- /dev/null +++ b/server/src/test/resources/bikes/specification.json @@ -0,0 +1,34 @@ +{ + "inputs": { + "files": [ + "data/bikes-2019-2020-ny.csv" + ], + "parameters": { + "databikes": "data/bikes-2019-2020-ny.csv", + "variables": "Temperature Humidity" + } + }, + "outputs": { + "files": [ + "results/r_squared.txt", + "results/summary.txt" + ] + }, + "version": "0.6.0", + "workflow": { + "specification": { + "steps": [ + { + "commands": [ + "mkdir -p results", + "python /usr/src/app/main.py --inputs ${databikes} --variables ${variables} --r_squared results/r_squared.txt --summary results/summary.txt" + ], + "environment": "ikcap/bikes_rent:bcd48921c6e2ed502b84c768b7408e9466a9d576", + "kubernetes_memory_limit": "256Mi", + "name": "predict" + } + ] + }, + "type": "serial" + } +} diff --git a/server/src/test/resources/dev/getSpecificationResponse.json b/server/src/test/resources/dev/getSpecificationResponse.json new file mode 100644 index 0000000..bc5591a --- /dev/null +++ b/server/src/test/resources/dev/getSpecificationResponse.json @@ -0,0 +1,8 @@ +{ + "specification": { + "inputs": { "files": ["data/bikes-2019-2020-ny.csv"], "parameters": { "databikes": "data/bikes-2019-2020-ny.csv", "variables": "Temperature Humidity" } }, + "outputs": { "files": ["results/r_squared.txt", "results/summary.txt"] }, + "version": "0.6.0", + "workflow": { "specification": { "steps": [{ "commands": ["mkdir -p results", "python /usr/src/app/main.py --inputs ${databikes} --variables ${variables} --r_squared results/r_squared.txt --summary results/summary.txt"], "environment": "ikcap/bikes_rent:bcd48921c6e2ed502b84c768b7408e9466a9d576", "kubernetes_memory_limit": "256Mi", "name": "predict" }] }, "type": "serial" } + } +} diff --git a/server/src/test/resources/root/specification.json b/server/src/test/resources/root/specification.json new file mode 100644 index 0000000..a47feb8 --- /dev/null +++ b/server/src/test/resources/root/specification.json @@ -0,0 +1,47 @@ +{ + "parameters": { + "events": "10" + }, + "specification": { + "inputs": { + "files": [ + "code/gendata.C", + "code/fitdata.C" + ], + "parameters": { + "data": "results/data.root", + "events": 20000, + "plot": "results/plot.png" + } + }, + "outputs": { + "files": [ + "results/plot.png" + ] + }, + "version": "0.6.0", + "workflow": { + "specification": { + "steps": [ + { + "commands": [ + "mkdir -p results && root -b -q 'code/gendata.C(${events},\"${data}\")'" + ], + "environment": "reanahub/reana-env-root6:6.18.04", + "kubernetes_memory_limit": "256Mi", + "name": "gendata" + }, + { + "commands": [ + "root -b -q 'code/fitdata.C(\"${data}\",\"${plot}\")'" + ], + "environment": "reanahub/reana-env-root6:6.18.04", + "kubernetes_memory_limit": "256Mi", + "name": "fitdata" + } + ] + }, + "type": "serial" + } + } +} \ No newline at end of file diff --git a/shared/pom.xml b/shared/pom.xml index f531ad8..8fa27ba 100644 --- a/shared/pom.xml +++ b/shared/pom.xml @@ -1,6 +1,5 @@ - + 4.0.0 @@ -12,15 +11,15 @@ org.disk-project disk-project - 3.0.0 - ../pom.xml + 3.0.0 + ../pom.xml 2.0.1 3.1.0 2.13.3 - + ${project.build.directory}/${project.build.finalName} UTF-8 @@ -32,7 +31,7 @@ javax.ws.rs-api ${javax.rs-api.version} - + javax.servlet @@ -40,25 +39,34 @@ ${servlet-api.version} provided - + com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider ${jackson.version} - + org.apache.commons commons-lang3 3.4 - + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.0.0 + + 1.8 + 1.8 + + org.apache.maven.plugins maven-source-plugin @@ -74,4 +82,4 @@ - + \ No newline at end of file diff --git a/shared/src/main/java/org/diskproject/shared/classes/adapters/MethodAdapter.java b/shared/src/main/java/org/diskproject/shared/classes/adapters/MethodAdapter.java index 7ff8057..ead705f 100644 --- a/shared/src/main/java/org/diskproject/shared/classes/adapters/MethodAdapter.java +++ b/shared/src/main/java/org/diskproject/shared/classes/adapters/MethodAdapter.java @@ -1,96 +1,190 @@ package org.diskproject.shared.classes.adapters; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import org.diskproject.shared.classes.loi.TriggeredLOI.Status; import org.diskproject.shared.classes.workflow.Variable; import org.diskproject.shared.classes.workflow.VariableBinding; import org.diskproject.shared.classes.workflow.Workflow; import org.diskproject.shared.classes.workflow.WorkflowRun; public abstract class MethodAdapter { + public HashMap status = new HashMap(); private String name; private String endpointUrl; private String username; private String password; private String description; - private Float version; + private Float version; - public MethodAdapter (String adapterName, String url) { + public MethodAdapter(String adapterName, String url) { this.name = adapterName; this.endpointUrl = url; } - public MethodAdapter (String adapterName, String url, String username, String password) { + public MethodAdapter(String adapterName, String url, String username, String password) { this.name = adapterName; this.endpointUrl = url; this.username = username; this.password = password; } - public Float getVersion () { + + /** + * @return Float + */ + public Float getVersion() { return this.version; } - public void setVersion (Float v) { + + /** + * @param v + */ + public void setVersion(Float v) { this.version = v; } - public String getDescription () { + + /** + * @return String + */ + public String getDescription() { return this.description; } - public void setDescription (String desc) { + + /** + * @param desc + */ + public void setDescription(String desc) { this.description = desc; } + - public String getName () { + /** + * @return String + */ + public String getName() { return this.name; } + - public String getEndpointUrl () { + /** + * @return String + */ + public String getEndpointUrl() { return this.endpointUrl; } + - protected String getUsername () { + /** + * @return String + */ + protected String getUsername() { return this.username; } + - protected String getPassword () { + /** + * @return String + */ + protected String getPassword() { return this.password; } - public String toString () { + + /** + * @return String + */ + public String toString() { return "[" + this.name + "] " + (this.username != null ? this.username + "@" : "") + this.endpointUrl; } - - public abstract List getWorkflowList(); - - public abstract List getWorkflowVariables(String id); - - public abstract String getWorkflowId(String id); - - public abstract String getWorkflowUri(String id); - - public abstract String getWorkflowLink(String id); - - public abstract String getDataUri (String id); - - public abstract List areFilesAvailable (Set fileList, String dType); - - public abstract String addData (String url, String name, String dType) throws Exception; - - public abstract Map getWorkflowInputs (String id); - - public abstract String runWorkflow (String wfId, List vBindings, Map inputVariables); - - public abstract WorkflowRun getRunStatus (String runId); - - public abstract byte[] fetchData (String dataId); - public abstract Map getRunVariableBindings (String runId); + /** + * Returns a list of available methods for the adapter. + * + * @return + */ + public abstract List getWorkflowList(); + + /** + * Return the list of variable of a given workflow. + * A variable can be a parameter or a input + * + * @param id + * @return + * @throws Exception + */ + public abstract List getWorkflowVariables(String id) throws Exception; + + public abstract String getWorkflowId(String id); + + public abstract String getWorkflowUri(String id); + + public abstract String getWorkflowLink(String id); + + public abstract String getDataUri(String dataId, String workflowId); + + /** + * If the method adapter needs to store the data, check if it is already stored. + * + * @param fileList A list of files to check. + * @param dataType The RDF file type, this is used by Wings to determine the + * file type. + * @return + */ + public abstract List areFilesAvailable(Set fileList, String dataType); + + public abstract String addData(String url, String name, String dataType) throws Exception; + + /** + * Upload a remote file to the method adapter. + * Used by the REANA adapter. + * + * @param url The URL of the file to upload + * @param workSpaceId The workspace id where the file will be uploaded + * @param name The name of the file to upload + * @param dataType The data type of the file to upload + * @return + * @throws Exception + */ + public abstract String addData(String url, String workSpaceId, String name, String dataType) throws Exception; + + /** + * Return a Map of the variable name and variable for a given workflow run. + * + * @param id + * @return + * @throws Exception + */ + public abstract Map getWorkflowInputs(String id) throws Exception; + + /** + * Run the workflow. + * + * @param workflowName Workflow ID + * @param vBindings An array of VariableBindings from the Disk System. A + * VariableBinding is a pair of a variable name and a + * value. + * @param inputVariables A map of input variables from the Workflow System. The + * key is the variable name and the value is the variable. + * @return + * @throws Exception + */ + public abstract String runWorkflow(String workflowName, String workflowLink, List vBindings, + Map inputVariables) throws Exception; + + public abstract WorkflowRun getRunStatus(String runId) throws Exception; + + public abstract byte[] fetchData (String dataId) throws Exception; + public abstract Map getRunVariableBindings(String runId); + + public abstract String duplicateWorkflow(String workflowId, String newName) throws Exception; // Test connection with source - public abstract boolean ping (); + public abstract boolean ping(); } \ No newline at end of file diff --git a/shared/src/main/java/org/diskproject/shared/classes/util/ExternalDataRequest.java b/shared/src/main/java/org/diskproject/shared/classes/util/ExternalDataRequest.java index a05b04e..447aba1 100644 --- a/shared/src/main/java/org/diskproject/shared/classes/util/ExternalDataRequest.java +++ b/shared/src/main/java/org/diskproject/shared/classes/util/ExternalDataRequest.java @@ -1,7 +1,21 @@ package org.diskproject.shared.classes.util; public class ExternalDataRequest { - String source, dataId; + String source, dataId, runId; + + public ExternalDataRequest(String source, String dataId, String runId) { + this.source = source; + this.dataId = dataId; + this.runId = runId; + } + + public String getRunId() { + return runId; + } + + public void setRunId(String runId) { + this.runId = runId; + } public ExternalDataRequest () { } diff --git a/shared/src/main/java/org/diskproject/shared/classes/workflow/Workflow.java b/shared/src/main/java/org/diskproject/shared/classes/workflow/Workflow.java index 8ba7c2e..b483fe1 100644 --- a/shared/src/main/java/org/diskproject/shared/classes/workflow/Workflow.java +++ b/shared/src/main/java/org/diskproject/shared/classes/workflow/Workflow.java @@ -1,9 +1,19 @@ package org.diskproject.shared.classes.workflow; +import java.net.MalformedURLException; + public class Workflow { String id, name, link, source; - public Workflow (String id, String name, String link, String source) { + /** + * Workflows define the set of computational tasks and dependencies needed to carry out in silico experiments + * @param id a unique identified for the workflow + * @param name a name for the workflow + * @param link a public link to the workflow + * @param source the source of the workflow + * @throws MalformedURLException + */ + public Workflow (String id, String name, String link, String source) { this.id = id; this.name = name; this.link = link; diff --git a/shared/src/main/java/org/diskproject/shared/classes/workflow/WorkflowRun.java b/shared/src/main/java/org/diskproject/shared/classes/workflow/WorkflowRun.java index 170a458..56d58c8 100644 --- a/shared/src/main/java/org/diskproject/shared/classes/workflow/WorkflowRun.java +++ b/shared/src/main/java/org/diskproject/shared/classes/workflow/WorkflowRun.java @@ -9,6 +9,7 @@ public class WorkflowRun { String status; Map outputs; + //TODO: files is good name for this file? Map files; String startDate, endDate;