Platform initial implementation

jupyter-platform/platform-api/pom.xml

@@ -149,6 +149,11 @@
             <artifactId>guava</artifactId>
             <version>29.0-jre</version>
         </dependency>
+        <dependency>
+            <groupId>com.hierynomus</groupId>
+            <artifactId>sshj</artifactId>
+            <version>0.32.0</version>
+        </dependency>
 
     </dependencies>
 

jupyter-platform/platform-api/src/main/java/org/apache/airavata/jupyter/api/Config.java

@@ -76,9 +76,18 @@ public void addCorsMappings(CorsRegistry registry) {
     public void addInterceptors(InterceptorRegistry registry) {
 
         if (superUserMode) {
-            registry.addInterceptor(authenticator).excludePathPatterns("/api/archive/**").excludePathPatterns("/api/admin/**");
+            registry.addInterceptor(authenticator)
+                    .excludePathPatterns("/api/archive/**")
+                    .excludePathPatterns("/api/remote/run/**")
+                    .excludePathPatterns("/api/admin/**")
+                    .excludePathPatterns("/api/job/**")
+                    .excludePathPatterns("/error");
         } else {
-            registry.addInterceptor(authenticator).excludePathPatterns("/api/archive/**");
+            registry.addInterceptor(authenticator)
+                    .excludePathPatterns("/api/archive/**")
+                    .excludePathPatterns("/api/remote/run/**")
+                    .excludePathPatterns("/api/job/**")
+                    .excludePathPatterns("/error");
         }
     }
 }

jupyter-platform/platform-api/src/main/java/org/apache/airavata/jupyter/api/controller/ArchiveController.java

@@ -19,17 +19,30 @@
 
 import org.apache.airavata.jupyter.api.entity.ArchiveEntity;
 import org.apache.airavata.jupyter.api.entity.NotebookEntity;
+import org.apache.airavata.jupyter.api.entity.job.JobEntity;
 import org.apache.airavata.jupyter.api.repo.ArchiveRepository;
+import org.apache.airavata.jupyter.api.repo.JobRepository;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
 import org.springframework.web.multipart.MultipartFile;
+import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
 
+import javax.servlet.http.HttpServletResponse;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.*;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
 
 @RestController
 @RequestMapping(path = "/api/archive")
@@ -41,6 +54,9 @@ public class ArchiveController {
     @Autowired
     private ArchiveRepository archiveRepository;
 
+    @Autowired
+    private JobRepository jobRepository;
+
     @PostMapping(path = "/", consumes = "application/json", produces = "application/json")
     public ArchiveEntity createArchive(@RequestBody ArchiveEntity archiveEntity) {
         ArchiveEntity saved = archiveRepository.save(archiveEntity);
@@ -70,4 +86,43 @@ public Map<String, String> singleFileUpload(@RequestParam("file") MultipartFile
 
         return Collections.singletonMap("path", path.toAbsolutePath().toString());
     }
+
+    @GetMapping (value = "/download/{jobId}", produces = MediaType.APPLICATION_JSON_VALUE)
+    public ResponseEntity<StreamingResponseBody> download(@PathVariable String jobId, final HttpServletResponse response)
+            throws Exception {
+        response.setContentType("application/zip");
+        response.setHeader(
+                "Content-Disposition",
+                "attachment;filename=REMOTE_STATE.zip");
+
+        StreamingResponseBody stream = out -> {
+
+
+            Optional<JobEntity> jobOp = jobRepository.findById(jobId);
+            JobEntity job = jobOp.get();
+            final File directory = new File(job.getLocalWorkingPath());
+            final ZipOutputStream zipOut = new ZipOutputStream(response.getOutputStream());
+
+            if(directory.exists() && directory.isDirectory()) {
+                try {
+                    for (final File file : directory.listFiles()) {
+                        final InputStream inputStream=new FileInputStream(file);
+                        final ZipEntry zipEntry = new ZipEntry(file.getName());
+                        zipOut.putNextEntry(zipEntry);
+                        byte[] bytes=new byte[1024];
+                        int length;
+                        while ((length=inputStream.read(bytes)) >= 0) {
+                            zipOut.write(bytes, 0, length);
+                        }
+                        inputStream.close();
+                    }
+                    zipOut.close();
+                } catch (final IOException e) {
+                    logger.error("Exception while reading and streaming data {} ", e);
+                }
+            }
+        };
+        logger.info("steaming response {} ", stream);
+        return new ResponseEntity(stream, HttpStatus.OK);
+    }
 }

jupyter-platform/platform-api/src/main/java/org/apache/airavata/jupyter/api/controller/JobController.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.jupyter.api.controller;
+
+import org.apache.airavata.jupyter.api.entity.job.JobStatusEntity;
+import org.apache.airavata.jupyter.api.repo.JobRepository;
+import org.apache.airavata.jupyter.api.repo.JobStatusRepository;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+import java.util.Optional;
+
+@RestController
+@RequestMapping(path = "/api/job")
+public class JobController {
+
+    @Autowired
+    private JobStatusRepository jobStatusRepository;
+
+    @Autowired
+    private JobRepository jobRepository;
+
+    @GetMapping(path = "/status/{jobId}")
+    public JobStatusEntity getJobStatus(@PathVariable String jobId) throws Exception {
+        Optional<JobStatusEntity> jobSt = jobStatusRepository.findFirstByJobIdOrderByUpdatedTimeAsc(jobId);
+        return jobSt.orElseThrow(() -> new Exception("Could not find job status for job id " + jobId));
+    }
+}

jupyter-platform/platform-api/src/main/java/org/apache/airavata/jupyter/api/controller/RemoteExecController.java

@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.jupyter.api.controller;
+
+import org.apache.airavata.jupyter.api.entity.ArchiveEntity;
+import org.apache.airavata.jupyter.api.entity.interfacing.LocalInterfaceEntity;
+import org.apache.airavata.jupyter.api.entity.interfacing.SSHInterfaceEntity;
+import org.apache.airavata.jupyter.api.entity.remote.ComputeEntity;
+import org.apache.airavata.jupyter.api.repo.*;
+import org.apache.airavata.jupyter.api.util.remote.interfacing.InterfacingProtocol;
+import org.apache.airavata.jupyter.api.util.remote.interfacing.LocalInterfacingProtocol;
+import org.apache.airavata.jupyter.api.util.remote.interfacing.SSHInterfacingProtocol;
+import org.apache.airavata.jupyter.api.util.remote.submitters.ForkJobSubmitter;
+import org.apache.airavata.jupyter.api.util.remote.submitters.JobSubmitter;
+import org.apache.airavata.jupyter.api.util.remote.submitters.SlurmJobSubmitter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.security.core.Authentication;
+import org.springframework.web.bind.annotation.*;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Optional;
+import java.util.UUID;
+
+@RestController
+@RequestMapping(path = "/api/remote")
+public class RemoteExecController {
+    private static final Logger logger = LoggerFactory.getLogger(RemoteExecController.class);
+
+    private String localWorkingDir = "/tmp";
+
+    @Autowired
+    private ComputeRepository computeRepository;
+
+    @Autowired
+    private LocalInterfaceRepository localInterfaceRepository;
+
+    @Autowired
+    private SSHInterfaceRepository sshInterfaceRepository;
+
+    @Autowired
+    private ArchiveRepository archiveRepository;
+
+    @Autowired
+    private JobRepository jobRepository;
+
+    @Autowired
+    private JobStatusRepository jobStatusRepository;
+
+    public class RunCellResponse {
+        private String jobId;
+
+        public String getJobId() {
+            return jobId;
+        }
+
+        public void setJobId(String jobId) {
+            this.jobId = jobId;
+        }
+    }
+
+    @GetMapping(path = "/run/{computeId}/{archiveId}/{sessionId}")
+    public RunCellResponse runCell(@PathVariable String computeId, @PathVariable String archiveId, @PathVariable String sessionId) throws Exception {
+
+        logger.info("Running cell for compute {} with state archive uploaded in to archive {}", computeId, archiveId);
+
+        Optional<ArchiveEntity> archiveOp = archiveRepository.findById(archiveId);
+        Optional<ComputeEntity> computeOp = computeRepository.findById(computeId);
+        if (computeOp.isPresent() && archiveOp.isPresent()) {
+            ComputeEntity computeEntity = computeOp.get();
+            ArchiveEntity archiveEntity = archiveOp.get();
+            InterfacingProtocol interfacingProtocol = resolveInterface(computeEntity.getInterfaceType(), computeEntity.getInterfaceId());
+
+            // Creating local working directory
+            String workDirForCurrent = localWorkingDir + "/"  + UUID.randomUUID().toString();
+            Files.createDirectory(Path.of(workDirForCurrent));
+
+            JobSubmitter jobSubmitter = resolveJobSubmitter(interfacingProtocol, computeEntity.getSubmitterType(), workDirForCurrent);
+            String jobId = jobSubmitter.submitJob(archiveEntity.getPath(), sessionId);
+            RunCellResponse response = new RunCellResponse();
+            response.setJobId(jobId);
+            return response;
+        } else {
+            throw new Exception("Could not find a compute resource with id " + computeId + " or archive with id " + archiveId);
+        }
+    }
+
+    @PostMapping(path = "/interface/local", consumes = "application/json", produces = "application/json")
+    public LocalInterfaceEntity createLocalInterface(Authentication authentication, @RequestBody LocalInterfaceEntity localInterfaceEntity) {
+        LocalInterfaceEntity saved = localInterfaceRepository.save(localInterfaceEntity);
+        return saved;
+    }
+
+    @PostMapping(path = "/interface/ssh", consumes = "application/json", produces = "application/json")
+    public SSHInterfaceEntity createSSHInterface(Authentication authentication, @RequestBody SSHInterfaceEntity sshInterfaceEntity) {
+        SSHInterfaceEntity saved = sshInterfaceRepository.save(sshInterfaceEntity);
+        return saved;
+    }
+
+    @PostMapping(path = "/compute", consumes = "application/json", produces = "application/json")
+    public ComputeEntity createCompute(Authentication authentication, @RequestBody ComputeEntity computeEntity) {
+        ComputeEntity saved = computeRepository.save(computeEntity);
+        return saved;
+    }
+
+    private JobSubmitter resolveJobSubmitter(InterfacingProtocol interfacingProtocol,
+                                             ComputeEntity.SubmitterType submitterType,
+                                             String workDir) throws Exception {
+        switch (submitterType) {
+            case FORK:
+                return new ForkJobSubmitter(workDir, interfacingProtocol, jobRepository, jobStatusRepository);
+            case SLURM:
+                return new SlurmJobSubmitter();
+        }
+
+        throw new Exception("Could not find a job submitter with type " + submitterType.name());
+
+    }
+
+    private InterfacingProtocol resolveInterface(ComputeEntity.InterfaceType interfaceType, String interfaceId) throws Exception {
+
+        switch (interfaceType) {
+            case LOCAL:
+                Optional<LocalInterfaceEntity> localInterfaceOp = localInterfaceRepository.findById(interfaceId);
+                if (localInterfaceOp.isPresent()) {
+                    return new LocalInterfacingProtocol(localInterfaceOp.get().getWorkingDirectory());
+                } else {
+                    throw new Exception("Could not find a local interface with id " + interfaceId);
+                }
+            case SSH:
+                Optional<SSHInterfaceEntity> sshInterfaceOp = sshInterfaceRepository.findById(interfaceId);
+                if (sshInterfaceOp.isPresent()) {
+                     return new SSHInterfacingProtocol(sshInterfaceOp.get(), sshInterfaceOp.get().getWorkingDirectory());
+                } else {
+                    throw new Exception("Could not find a SSH interface with id " + interfaceId);
+                }
+        }
+
+        throw new Exception("Could not find a valid interface for type " + interfaceType.name() + " and id " + interfaceId);
+    }
+}

jupyter-platform/platform-api/src/main/java/org/apache/airavata/jupyter/api/entity/interfacing/LocalInterfaceEntity.java

@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.jupyter.api.entity.interfacing;
+
+import org.hibernate.annotations.GenericGenerator;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.Id;
+
+@Entity(name = "LOCAL_INTERFACE")
+public class LocalInterfaceEntity {
+
+    @Id
+    @Column(name = "ARCHIVE_ID")
+    @GeneratedValue(generator = "uuid")
+    @GenericGenerator(name = "uuid", strategy = "uuid2")
+    private String id;
+
+    @Column(name = "WORKING_DIR")
+    private String workingDirectory;
+
+    public String getId() {
+        return id;
+    }
+
+    public void setId(String id) {
+        this.id = id;
+    }
+
+    public String getWorkingDirectory() {
+        return workingDirectory;
+    }
+
+    public void setWorkingDirectory(String workingDirectory) {
+        this.workingDirectory = workingDirectory;
+    }
+}