From cf3e7634f0293db80bfdc9dd3445db505fc7d667 Mon Sep 17 00:00:00 2001 From: fx13349 Date: Mon, 21 Jan 2019 10:42:16 +0100 Subject: [PATCH 1/2] feat(settings): App configuration in individual files (issue 91) The initial ShinyProxySpecMainProvider was able to load Add settings from application.yml. The new version will delegate the loading task to ShinyProxySpecMainProvider and ShinyProxySpecExternalProvider ShinyProxyspecMainProvider does the same then the initial ShinyProxySpecMainProvider, ie. it loads settings from application.yml. ShinyProxySpecExternalProvider will try to load files from a specific path. This path can be set in applicaton.yml using the new proxy.specs-external-path attribute. The attribute used in the individual settings files are quite the same (using camel case) : - id, displayName, description, containerCmd, logoUrl, accessGroups, ... --- .../ShinyProxySpecExternalProvider.java | 115 ++++++++ .../ShinyProxySpecMainProvider.java | 278 ++++++++++++++++++ .../shinyproxy/ShinyProxySpecProvider.java | 242 +++------------ 3 files changed, 430 insertions(+), 205 deletions(-) create mode 100644 src/main/java/eu/openanalytics/shinyproxy/ShinyProxySpecExternalProvider.java create mode 100644 src/main/java/eu/openanalytics/shinyproxy/ShinyProxySpecMainProvider.java diff --git a/src/main/java/eu/openanalytics/shinyproxy/ShinyProxySpecExternalProvider.java b/src/main/java/eu/openanalytics/shinyproxy/ShinyProxySpecExternalProvider.java new file mode 100644 index 00000000..68d22fb0 --- /dev/null +++ b/src/main/java/eu/openanalytics/shinyproxy/ShinyProxySpecExternalProvider.java @@ -0,0 +1,115 @@ +/** + * ShinyProxy + * + * Copyright (C) 2016-2018 Open Analytics + * + * =========================================================================== + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Apache License as published by + * The Apache Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details. + * + * You should have received a copy of the Apache License + * along with this program. If not, see + */ +package eu.openanalytics.shinyproxy; + +import java.io.File; +import java.io.FileFilter; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import javax.annotation.PostConstruct; +import javax.inject.Inject; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Component; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.Constructor; + +import eu.openanalytics.containerproxy.model.spec.ProxySpec; +import eu.openanalytics.shinyproxy.ShinyProxySpecMainProvider.ShinyProxySpec; + +/** + * This component converts proxy specs from the external specs path. + */ +@Component +public class ShinyProxySpecExternalProvider { + + protected final Logger log = LogManager.getLogger(getClass()); + + @Inject + private Environment environment; + + private String specsExternalPath = null; + + @PostConstruct + public void init() { + specsExternalPath = environment.getProperty("proxy.specs-external-path"); + } + + public List getSpecs() { + // No setting, we leave with empty collection + if (specsExternalPath == null) { + return Collections.emptyList(); + } + + final File appSettingsFolder = new File(specsExternalPath); + if (appSettingsFolder.exists() == false) { + return Collections.emptyList(); + } + + // We load the settings file + final File[] allSettingsFiles = appSettingsFolder.listFiles(new FileFilter() { + + @Override + public boolean accept(File pathname) { + return pathname.getName().endsWith("yml") || pathname.getName().endsWith("yaml"); + } + }); + + // For each file, we create a new ProxySpec + final List allExternalSpecs = new ArrayList<>(); + for (int i = 0; i < allSettingsFiles.length; i++) { + try { + allExternalSpecs.add(loadYamlFile(allSettingsFiles[i])); + } catch (Exception e) { + log.error("An error occured while trying to open " + allSettingsFiles[i].getName() + " " + + e.getMessage()); + } + } + + return allExternalSpecs; + } + + /** + * Map the yaml parameter with the shiny app object + * + * @param inputFile + * @return + * @throws IOException + */ + private ProxySpec loadYamlFile(final File inputFile) throws IOException { + try (final FileReader fileReader = new FileReader(inputFile)) { + final Yaml yaml = new Yaml(new Constructor(ShinyProxySpec.class)); + final ShinyProxySpec shinyProxySpec = yaml.load(fileReader); + log.debug(" Id " + shinyProxySpec.getId()); + log.debug(" DisplayName " + shinyProxySpec.getDisplayName()); + log.debug(" Description " + shinyProxySpec.getDescription()); + log.debug(" LogoURL " + shinyProxySpec.getLogoURL()); + // We reuse the Main provider converter + return ShinyProxySpecMainProvider.convert(shinyProxySpec); + } + } +} diff --git a/src/main/java/eu/openanalytics/shinyproxy/ShinyProxySpecMainProvider.java b/src/main/java/eu/openanalytics/shinyproxy/ShinyProxySpecMainProvider.java new file mode 100644 index 00000000..56a465c7 --- /dev/null +++ b/src/main/java/eu/openanalytics/shinyproxy/ShinyProxySpecMainProvider.java @@ -0,0 +1,278 @@ +/** + * ShinyProxy + * + * Copyright (C) 2016-2018 Open Analytics + * + * =========================================================================== + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Apache License as published by + * The Apache Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details. + * + * You should have received a copy of the Apache License + * along with this program. If not, see + */ +package eu.openanalytics.shinyproxy; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import eu.openanalytics.containerproxy.model.spec.ContainerSpec; +import eu.openanalytics.containerproxy.model.spec.ProxyAccessControl; +import eu.openanalytics.containerproxy.model.spec.ProxySpec; + +/** + * This component converts proxy specs from the 'ShinyProxy notation' into the + * 'ContainerProxy' notation. ShinyProxy notation is slightly more compact, and + * omits several things that Shiny apps do not need, such as definition of + * multiple containers. + * + * Also, if no port is specified, a port mapping is automatically created for + * Shiny port 3838. + */ +@Component +@ConfigurationProperties(prefix = "proxy") +public class ShinyProxySpecMainProvider { + + private List specs = new ArrayList<>(); + + public List getSpecs() { + return new ArrayList<>(specs); + } + + public ProxySpec getSpec(String id) { + if (id == null || id.isEmpty()) + return null; + return specs.stream().filter(s -> id.equals(s.getId())).findAny().orElse(null); + } + + public void setSpecs(List specs) { + this.specs = specs.stream().map(ShinyProxySpecMainProvider::convert).collect(Collectors.toList()); + } + + public static final ProxySpec convert(ShinyProxySpec from) { + ProxySpec to = new ProxySpec(); + to.setId(from.getId()); + to.setDisplayName(from.getDisplayName()); + to.setDescription(from.getDescription()); + to.setLogoURL(from.getLogoURL()); + + if (from.getAccessGroups() != null && from.getAccessGroups().length > 0) { + ProxyAccessControl acl = new ProxyAccessControl(); + acl.setGroups(from.getAccessGroups()); + to.setAccessControl(acl); + } + + ContainerSpec cSpec = new ContainerSpec(); + cSpec.setImage(from.getContainerImage()); + cSpec.setCmd(from.getContainerCmd()); + cSpec.setEnv(from.getContainerEnv()); + cSpec.setEnvFile(from.getContainerEnvFile()); + cSpec.setNetwork(from.getContainerNetwork()); + cSpec.setNetworkConnections(from.getContainerNetworkConnections()); + cSpec.setDns(from.getContainerDns()); + cSpec.setVolumes(from.getContainerVolumes()); + cSpec.setMemory(from.getContainerMemory()); + cSpec.setPrivileged(from.isContainerPrivileged()); + + Map portMapping = new HashMap<>(); + if (from.getPort() > 0) { + portMapping.put("default", from.getPort()); + } else { + portMapping.put("default", 3838); + } + cSpec.setPortMapping(portMapping); + + to.setContainerSpecs(Collections.singletonList(cSpec)); + + // Add some custom fields used in our template + to.setContact(from.getContact()); + to.setVersion(from.getVersion()); + // End customization + return to; + } + + public static class ShinyProxySpec { + + private String id; + private String displayName; + private String description; + private String logoURL; + + private String containerImage; + private String[] containerCmd; + private Map containerEnv; + private String containerEnvFile; + private String containerNetwork; + private String[] containerNetworkConnections; + private String[] containerDns; + private String[] containerVolumes; + private String containerMemory; + private boolean containerPrivileged; + + private int port; + private String[] accessGroups; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getDisplayName() { + return displayName; + } + + public void setDisplayName(String displayName) { + this.displayName = displayName; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getLogoURL() { + return logoURL; + } + + public void setLogoURL(String logoURL) { + this.logoURL = logoURL; + } + + public String getContainerImage() { + return containerImage; + } + + public void setContainerImage(String containerImage) { + this.containerImage = containerImage; + } + + public String[] getContainerCmd() { + return containerCmd; + } + + public void setContainerCmd(String[] containerCmd) { + this.containerCmd = containerCmd; + } + + public Map getContainerEnv() { + return containerEnv; + } + + public void setContainerEnv(Map containerEnv) { + this.containerEnv = containerEnv; + } + + public String getContainerEnvFile() { + return containerEnvFile; + } + + public void setContainerEnvFile(String containerEnvFile) { + this.containerEnvFile = containerEnvFile; + } + + public String getContainerNetwork() { + return containerNetwork; + } + + public void setContainerNetwork(String containerNetwork) { + this.containerNetwork = containerNetwork; + } + + public String[] getContainerNetworkConnections() { + return containerNetworkConnections; + } + + public void setContainerNetworkConnections(String[] containerNetworkConnections) { + this.containerNetworkConnections = containerNetworkConnections; + } + + public String[] getContainerDns() { + return containerDns; + } + + public void setContainerDns(String[] containerDns) { + this.containerDns = containerDns; + } + + public String[] getContainerVolumes() { + return containerVolumes; + } + + public void setContainerVolumes(String[] containerVolumes) { + this.containerVolumes = containerVolumes; + } + + public String getContainerMemory() { + return containerMemory; + } + + public void setContainerMemory(String containerMemory) { + this.containerMemory = containerMemory; + } + + public boolean isContainerPrivileged() { + return containerPrivileged; + } + + public void setContainerPrivileged(boolean containerPrivileged) { + this.containerPrivileged = containerPrivileged; + } + + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + + public String[] getAccessGroups() { + return accessGroups; + } + + public void setAccessGroups(String[] accessGroups) { + this.accessGroups = accessGroups; + } + + // Add some custom fields used in our template + private String contact; + private String version; + + public String getContact() { + return contact; + } + + public void setContact(String contact) { + this.contact = contact; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + // End customization + } +} diff --git a/src/main/java/eu/openanalytics/shinyproxy/ShinyProxySpecProvider.java b/src/main/java/eu/openanalytics/shinyproxy/ShinyProxySpecProvider.java index 905e5085..d6512d4b 100644 --- a/src/main/java/eu/openanalytics/shinyproxy/ShinyProxySpecProvider.java +++ b/src/main/java/eu/openanalytics/shinyproxy/ShinyProxySpecProvider.java @@ -22,232 +22,64 @@ import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; +import java.util.Comparator; import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import org.springframework.boot.context.properties.ConfigurationProperties; +import javax.inject.Inject; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Component; -import eu.openanalytics.containerproxy.model.spec.ContainerSpec; -import eu.openanalytics.containerproxy.model.spec.ProxyAccessControl; import eu.openanalytics.containerproxy.model.spec.ProxySpec; import eu.openanalytics.containerproxy.spec.IProxySpecProvider; /** - * This component converts proxy specs from the 'ShinyProxy notation' into the 'ContainerProxy' notation. - * ShinyProxy notation is slightly more compact, and omits several things that Shiny apps do not need, - * such as definition of multiple containers. - * - * Also, if no port is specified, a port mapping is automatically created for Shiny port 3838. + * This component aggregate the proxy specs from the Main Provider (mainning the + * application.yml file) and the others settings files as definied in the + * proxy.specs-external-path properties. */ @Component @Primary -@ConfigurationProperties(prefix = "proxy") public class ShinyProxySpecProvider implements IProxySpecProvider { - private List specs = new ArrayList<>(); + protected final Logger log = LogManager.getLogger(getClass()); + @Inject + private ShinyProxySpecMainProvider mainProvider; + + @Inject + private ShinyProxySpecExternalProvider externalProvider; + public List getSpecs() { - return new ArrayList<>(specs); + final List allProxySpec = new ArrayList<>(mainProvider.getSpecs()); + allProxySpec.addAll(externalProvider.getSpecs()); + Collections.sort(allProxySpec, PROXY_SPEC_COMPARATOR); + log.debug("We have found " + allProxySpec.size() + " spec(s)"); + return allProxySpec; } - + public ProxySpec getSpec(String id) { - if (id == null || id.isEmpty()) return null; - return specs.stream().filter(s -> id.equals(s.getId())).findAny().orElse(null); - } - - public void setSpecs(List specs) { - this.specs = specs.stream().map(ShinyProxySpecProvider::convert).collect(Collectors.toList()); - } - - private static ProxySpec convert(ShinyProxySpec from) { - ProxySpec to = new ProxySpec(); - to.setId(from.getId()); - to.setDisplayName(from.getDisplayName()); - to.setDescription(from.getDescription()); - to.setLogoURL(from.getLogoURL()); - - if (from.getAccessGroups() != null && from.getAccessGroups().length > 0) { - ProxyAccessControl acl = new ProxyAccessControl(); - acl.setGroups(from.getAccessGroups()); - to.setAccessControl(acl); - } - - ContainerSpec cSpec = new ContainerSpec(); - cSpec.setImage(from.getContainerImage()); - cSpec.setCmd(from.getContainerCmd()); - cSpec.setEnv(from.getContainerEnv()); - cSpec.setEnvFile(from.getContainerEnvFile()); - cSpec.setNetwork(from.getContainerNetwork()); - cSpec.setNetworkConnections(from.getContainerNetworkConnections()); - cSpec.setDns(from.getContainerDns()); - cSpec.setVolumes(from.getContainerVolumes()); - cSpec.setMemory(from.getContainerMemory()); - cSpec.setPrivileged(from.isContainerPrivileged()); - - Map portMapping = new HashMap<>(); - if (from.getPort() > 0) { - portMapping.put("default", from.getPort()); - } else { - portMapping.put("default", 3838); - } - cSpec.setPortMapping(portMapping); - - to.setContainerSpecs(Collections.singletonList(cSpec)); - - return to; + if (id == null || id.isEmpty()) + return null; + return getSpecs().stream().filter(s -> id.equals(s.getId())).findAny().orElse(null); } - - public static class ShinyProxySpec { - - private String id; - private String displayName; - private String description; - private String logoURL; - - private String containerImage; - private String[] containerCmd; - private Map containerEnv; - private String containerEnvFile; - private String containerNetwork; - private String[] containerNetworkConnections; - private String[] containerDns; - private String[] containerVolumes; - private String containerMemory; - private boolean containerPrivileged; - - private int port; - private String[] accessGroups; - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getDisplayName() { - return displayName; - } - public void setDisplayName(String displayName) { - this.displayName = displayName; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getLogoURL() { - return logoURL; - } - - public void setLogoURL(String logoURL) { - this.logoURL = logoURL; - } + // This comparator uses the display name to sort + private static final Comparator PROXY_SPEC_COMPARATOR = new Comparator() { - public String getContainerImage() { - return containerImage; + @Override + public int compare(ProxySpec o1, ProxySpec o2) { + if (o1 == o2) + return 0; + if (o1 == null) + return -1; + if (o2 == null) + return 1; + return StringUtils.compare(o1.getDisplayName(), o2.getDisplayName()); } + }; - public void setContainerImage(String containerImage) { - this.containerImage = containerImage; - } - - public String[] getContainerCmd() { - return containerCmd; - } - - public void setContainerCmd(String[] containerCmd) { - this.containerCmd = containerCmd; - } - - public Map getContainerEnv() { - return containerEnv; - } - - public void setContainerEnv(Map containerEnv) { - this.containerEnv = containerEnv; - } - - public String getContainerEnvFile() { - return containerEnvFile; - } - - public void setContainerEnvFile(String containerEnvFile) { - this.containerEnvFile = containerEnvFile; - } - - public String getContainerNetwork() { - return containerNetwork; - } - - public void setContainerNetwork(String containerNetwork) { - this.containerNetwork = containerNetwork; - } - - public String[] getContainerNetworkConnections() { - return containerNetworkConnections; - } - - public void setContainerNetworkConnections(String[] containerNetworkConnections) { - this.containerNetworkConnections = containerNetworkConnections; - } - - public String[] getContainerDns() { - return containerDns; - } - - public void setContainerDns(String[] containerDns) { - this.containerDns = containerDns; - } - - public String[] getContainerVolumes() { - return containerVolumes; - } - - public void setContainerVolumes(String[] containerVolumes) { - this.containerVolumes = containerVolumes; - } - - public String getContainerMemory() { - return containerMemory; - } - - public void setContainerMemory(String containerMemory) { - this.containerMemory = containerMemory; - } - - public boolean isContainerPrivileged() { - return containerPrivileged; - } - - public void setContainerPrivileged(boolean containerPrivileged) { - this.containerPrivileged = containerPrivileged; - } - - public int getPort() { - return port; - } - - public void setPort(int port) { - this.port = port; - } - - public String[] getAccessGroups() { - return accessGroups; - } - - public void setAccessGroups(String[] accessGroups) { - this.accessGroups = accessGroups; - } - } } From e3377187ce67a3af6aff34bec811558c4c782303 Mon Sep 17 00:00:00 2001 From: fx13349 Date: Mon, 21 Jan 2019 10:49:56 +0100 Subject: [PATCH 2/2] fix : fix an issue with bad attribute name --- .../ShinyProxySpecMainProvider.java | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/src/main/java/eu/openanalytics/shinyproxy/ShinyProxySpecMainProvider.java b/src/main/java/eu/openanalytics/shinyproxy/ShinyProxySpecMainProvider.java index 56a465c7..469278fc 100644 --- a/src/main/java/eu/openanalytics/shinyproxy/ShinyProxySpecMainProvider.java +++ b/src/main/java/eu/openanalytics/shinyproxy/ShinyProxySpecMainProvider.java @@ -98,10 +98,6 @@ public static final ProxySpec convert(ShinyProxySpec from) { to.setContainerSpecs(Collections.singletonList(cSpec)); - // Add some custom fields used in our template - to.setContact(from.getContact()); - to.setVersion(from.getVersion()); - // End customization return to; } @@ -254,25 +250,5 @@ public void setAccessGroups(String[] accessGroups) { this.accessGroups = accessGroups; } - // Add some custom fields used in our template - private String contact; - private String version; - - public String getContact() { - return contact; - } - - public void setContact(String contact) { - this.contact = contact; - } - - public String getVersion() { - return version; - } - - public void setVersion(String version) { - this.version = version; - } - // End customization } }