diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..aea7be0 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +gradlew text eol=lf +*.sh text eol=lf +*.conf text eol=lf diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..b32eff0 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,16 @@ +language: java +jdk: +- oraclejdk7 +deploy: + provider: releases + api_key: + secure: HYh0fPSByuuJB/r7YKc+iHRBzo13lScGz1Kign/Uj3GnAcDL+NpnYCvEl1JCymd+v8vPyFDxWmFJ6mJSyAlQzhxIRoeHpOQhECVJXuuB9IUk+o2hb+MhUuPpOwGtlUDW2Rry9qnz73fX5boEBzrYHF3vwbbqdfqc+sugkPcPJubtwlsH49CWHA0kMila/esZyuB7KvXuEmVCy3BCHBPK9KZ9Wo8LlB3pQhUwJWkSJID5BJl/zYDqOrKo4kjXpu6p5xcwAi+HtpCmV3gAMgcHPTO8sXy2b9JKcko0C52/+nyHpQ39gmwS0y+q0lnGZJy38IF52JDzXLAq1DxIXm2hPvI/zMNlhYJ6ovCuT7xTXypKnktwetQJ5dT6hvtNmtBMxkXww43svAb/k9nr2bo9z65iOqKpqdFQYzFB/wltToMbX50xC9Yi+fYv219H3Fc6vioJw5bkRCoapC7ImmkhVjsv5YDEGbqv3dyEbGOWyI0phnJBd6+UjMFCSxX9pinu+RUhLU0uga9/765kiMYsQAH7svzyo5iYcN0HB4X1amAKjaLmp2so2AqKk4FKznVHhtY0Mm74Xgq6tmQTk7iuCwbvf2M1pCjhV4zDBio2XODHGGxJeIZAQTx0CIXz0F20o6K9XC2vDM6BWHt2YCN6qjR0buYP/NdsOPcjeSnCVMk= + file_glob: true + file: + - "./rcloud-gist-service/build/distributions/*.deb" + - "./rcloud-gist-service/build/distributions/*.rpm" + - "./rcloud-gist-service/build/libs/*.jar" + skip_cleanup: true + on: + repo: MangoTheCat/rcloud-gist-services + tags: true diff --git a/README.md b/README.md index 4f7eb2f..41a2694 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ +[![Build Status](https://travis-ci.org/MangoTheCat/rcloud-gist-services.svg?branch=master)](https://travis-ci.org/MangoTheCat/rcloud-gist-services) + # RCloud Gist Service + + ## Overview The RCloud Gist Service is a Java based service for enabling gist access to various different backend storage systems. @@ -31,6 +35,10 @@ There are two mechanisms to load the project into the Eclipse IDE: 1. Use the Eclipse gradle plugin [BuildShip](https://github.com/eclipse/buildship), this is an Eclipse plugin that understands gradle projects. BuildShip does not include syntax highlighting editor, you will have to install Groovy Eclipse plugin. 2. Use the gradle eclipse plugin (this is a plugin in the gradle build file that will generate the appropriate eclipse project). To generate the eclipse files run the following `gradlew eclipse`, you can then import the project in as an existing project. When you add a new dependency in you will need to run this again to regenerate the eclipse project files and then refresh the project in eclipse and it will pick up the new settings. If you want to just generate the eclipse files for a specific sub module then run the command `gradlew :store:eclipse` for the store sub project. + +## Vagrant +The Vagrantfile sets up 80% of the environment needed to run rcloud. It takes a long time to finish the provisioning. Once done rcloud will be in `/opt/rcloud/rcloud-1.7/` you will need to setup the `rcloud.conf` file and then call `sudo ./scrtips/fresh_start.sh`. The `bootstrapR.sh` has already been called as part of the provisioning. + ## Components ## LICENSE & Copyright diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000..d581ce3 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,122 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# All Vagrant configuration is done below. The "2" in Vagrant.configure +# configures the configuration version (we support older styles for +# backwards compatibility). Please don't change it unless you know what +# you're doing. +Vagrant.configure(2) do |config| + # The most common configuration options are documented and commented below. + # For a complete reference, please see the online documentation at + # https://docs.vagrantup.com. + + # Every Vagrant development environment requires a box. You can search for + # boxes at https://atlas.hashicorp.com/search. + #config.vm.box = "ubuntu/trusty64" + config.vm.box = "janihur/ubuntu-1404-desktop" + + + config.vm.hostname = "rcloud.local" + + # Disable automatic box update checking. If you disable this, then + # boxes will only be checked for updates when the user runs + # `vagrant box outdated`. This is not recommended. + # config.vm.box_check_update = false + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine. In the example below, + # accessing "localhost:8080" will access port 80 on the guest machine. + config.vm.network "forwarded_port", guest: 80, host: 80 + config.vm.network "forwarded_port", guest: 13010, host: 13110 + config.vm.network "forwarded_port", guest: 13011, host: 13111 + + #config.ssh.forward_agent = true + + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + # config.vm.network "private_network", ip: "192.168.33.10" + config.vm.network "private_network", type: "dhcp" + + # Need to manage the hosts file to add in the real ip address, using + # vagrant-hostmanager for this: + # (https://github.com/devopsgroup-io/vagrant-hostmanager) + # To install run vagrant plugin install vagrant-hostmanager + config.hostmanager.enabled = true + config.hostmanager.manage_host = true + config.hostmanager.manage_guest = true + config.hostmanager.ignore_private_ip = false + config.hostmanager.include_offline = true + config.hostmanager.ip_resolver = proc do |vm, resolving_vm| + if hostname = (vm.ssh_info && vm.ssh_info[:host]) + `vagrant ssh -c "hostname -I"`.split()[1] + end + end + + # Create a public network, which generally matched to bridged network. + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network "public_network" + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. + # config.vm.synced_folder "../data", "/vagrant_data" + + # Provider-specific configuration so you can fine-tune various + # backing providers for Vagrant. These expose provider-specific options. + # Example for VirtualBox: + # + config.vm.provider "virtualbox" do |vb| + # Display the VirtualBox GUI when booting the machine + vb.gui = true + # Customize the amount of memory on the VM: + vb.memory = "4096" + vb.cpus = "2" + vb.customize ["modifyvm", :id, "--ioapic", "on"] + vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] + end + # + # View the documentation for the provider you are using for more + # information on available options. + + # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies + # such as FTP and Heroku are also available. See the documentation at + # https://docs.vagrantup.com/v2/push/atlas.html for more information. + # config.push.define "atlas" do |push| + # push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME" + # end + + # Adds the hostname of this machine into the hosts file + config.vm.provision :shell, inline: "sed -i'' 's/^127.0.0.1\\t#{config.vm.hostname}.*$//' /etc/hosts" + # Enable provisioning with a shell script. Additional provisioners such as + # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the + # documentation for more information about their specific syntax and use. + + config.vm.provision "shell", inline: "sudo apt-get update" + config.vm.provision "shell", inline: "sudo apt-get install -y curl" + #config.vm.provision "shell", inline: "sudo /usr/share/debconf/fix_db.pl" + #config.vm.provision "shell", inline: "sudo apt-get upgrade" + #config.vm.provision "shell", inline: "sudo apt-get install -y xfce4 virtualbox-guest-dkms virtualbox-guest-utils virtualbox-guest-x11" + #config.vm.provision "shell", inline: "sudo sed -i 's/allowed_users=.*$/allowed_users=anybody/' /etc/X11/Xwrapper.config" + + + config.vm.provision "shell", inline: <<-SHELL + curl -L https://bootstrap.saltstack.com | sudo sh -s -- stable + SHELL + + config.vm.synced_folder "salt/roots/salt", "/srv/salt/" + config.vm.synced_folder "salt/roots/pillar/", "/srv/pillar/" + + config.vm.provision :salt do |salt| + salt.masterless = true + salt.run_highstate = true +# salt.bootstrap_options = "-F -c /tmp -P" +## salt.minion_config = "salt/minion.yml" +# salt.run_highstate = true +# salt.colorize = true +# salt.log_level = 'debug' + end + +end diff --git a/build.gradle b/build.gradle index 7cb961e..7e7abe5 100644 --- a/build.gradle +++ b/build.gradle @@ -38,7 +38,7 @@ allprojects { } group = 'com.mangosolutions.rcloud' project.version = scmVersion.version - + ext.projectUrl = "https://github.com/MangoTheCat/rcloud-gist-services" } subprojects { @@ -110,15 +110,6 @@ configure(subprojects.findAll {it.name =~ /.*service/}) { buildInfo() } - ext.installPath='/opt/' + archivesBaseName - ext.jarPath=installPath + '/' + archivesBaseName + '-' + scmVersion.version + '.jar' - ext.initPath='/etc/init.d/' + archivesBaseName - project.description = "TBD" - ext.projectVendor = "TBD" - ext.projectUrl = "TBD" - ext.projectPackageGroup = "TBD" - ext.buildId = new Date().format('yyyyMMddHHmmss') - dependencies { compile("org.springframework.boot:spring-boot-starter-undertow:1.4.3.RELEASE") } @@ -131,31 +122,13 @@ configure(subprojects.findAll {it.name =~ /.*service/}) { archives buildRpm } -/* - ospackage { - packageName = archivesBaseName - version = project.version.replaceAll("-", ".") - release = 1 - arch = NOARCH - os = LINUX - user = "prov" - group = "prov" - - into installPath + task distRpm(type: Rpm, dependsOn: bootRepackage) {} - from(jar.outputs.files) { - fileMode 0500 - } + task distDeb(type: Deb, dependsOn: bootRepackage) {} - link(initPath, jarPath) - - } -*/ - - task distRpm(type: Rpm, dependsOn: bootRepackage) { - - } assemble.dependsOn distRpm + assemble.dependsOn distDeb + } diff --git a/gradlew b/gradlew old mode 100644 new mode 100755 diff --git a/rcloud-gist-service/README.md b/rcloud-gist-service/README.md index e7f76dc..47c0d4b 100644 --- a/rcloud-gist-service/README.md +++ b/rcloud-gist-service/README.md @@ -1 +1,71 @@ -# Implementation of the RCLoud Gist Service. +# Implementation of the RCloud Gist Service. +A Java based service to provide GitHub gist functionality to RCloud. + +## Building +The code uses the Gradle build system and includes the gradle wrapper in the root +of the project. The only requirement for building is a Java JDK version 7 or greater +and an internet connection, all dependencies will be downloaded for the build. + +To build the software run the following command which runs the tests and generates +the artifacts. + +`./gradlew build` + +### Outputs +The build generates 3 artifacts +1. An executable jar file created here: `./rcloud=gist-service/build/libs` +2. An rpm install file created here: `./rcloud=gist-service/build/distributions` +3. An deb install file created here: `./rcloud=gist-service/build/distributions` + +## Installation + +The redhat and debian install archives will install the application to +`/opt/rcloud-gist-service` and create an entry in the `/etc/init.d/` folder +which can be used to start and stop the service. + +### User and Groups +The installation creates a user and group for the service called `rcloudgistservice` which the service runs as. + +### Debian based systems +To install the service on debian based systems the following command can be used, you will need to use the correct name for the deb file for the version you are installing. + +`sudo dpkg -i ./rcloud-gist-service_0.1.0-20170126123855_all.deb` + +## Default ports +The service uses two ports, one for the gist api and the other for the service management functionality, these can be controlled in configuration. The management port is secured using basic auth. + +## Service Configuration + +Configuration is held within the `/opt/rcloud-gist-service/application.yml` file. +The following parameters are configurable: + +| Property | Description | Default | +|----------|-------------|---------| +| `github.api.url` | The URL to the root of the GitHub installation that this should use | `https://api.github.com` | +| `service.port` | The port that the gist api is accessible over | `13010` | +| `management.port` | The port that the service management api is accessible over | `13011` | +| `security.user.name` | The username that is required for basic auth access to the management port | `admin` | +| `security.user.password` | The username that is required for basic auth access to the management port | If not specified the password is generated at service startup and can be identified in the `/var/log/rcloud-gist-service/rcloud-gist-service-file.log` file. The following command can be used to find the password. `cat /var/log/rcloud-gist-service/rcloud-gist-service-file.log | grep "Using default security"`. More information can be found on the [spring boot documentation.](http://docs.spring.io/spring-boot/docs/current/reference/html/production-ready-monitoring.html) | + +### Java Configuration +The startup parameters for the JVM are stored in the conf file in the installation directory, this must have the same name as the jar file. + + +## RCloud Configuration +To configure RCloud edit the `rcloud.conf` and change the `api.github.url` value to the URL of this service if running with the defaults then this would be `http://localhost:13010` e.g. : + +`github.api.url: http://localhost:13010` + +## Starting and stopping the service + +An System V startup script is installed as part of the installation, thissupports the following commands: + +| Command | Example | +|---------|--------------------------------------| +| start | `service rcloud-gist-service start` | +| stop | `service rcloud-gist-service stop` | +| status | `service rcloud-gist-service status` | + +## Logging +The service uses [Logback](https://logback.qos.ch/), this is controlled by the +configuration file in the installation directory `/opt/rcloud-gist-service/logback.xml`, this configuration file can be updated and the changes will be reloaded to alter the log output. The service writes log files to `/var/log/rcloud-gist-service/` access to this folder is restricted to `root` group and the `rcloudgistservice` user/group. diff --git a/rcloud-gist-service/build.gradle b/rcloud-gist-service/build.gradle index 0c38a52..eb84b1e 100644 --- a/rcloud-gist-service/build.gradle +++ b/rcloud-gist-service/build.gradle @@ -1,24 +1,15 @@ -ext.serviceInstallPath = '/opt/rcloud-gist-service' -ext.installPath = serviceInstallPath + '/' + archivesBaseName -ext.jarPath = installPath + '/' + archivesBaseName + '-' + project.version + '.jar' -ext.initPath = '/etc/init.d/' + archivesBaseName project.description = "RCloud Gist service providing gist access api." -ext.projectVendor = "TBD" -ext.projectUrl = "TBD" -ext.projectPackageGroup = "TBD" -ext.buildId = new Date().format('yyyyMMddHHmmss') dependencies { compile 'org.springframework.boot:spring-boot-starter' compile 'org.springframework.boot:spring-boot-starter-actuator' - //compile 'org.springframework.boot:spring-boot-starter-undertow' + compile 'org.springframework.boot:spring-boot-starter-security' compile 'org.springframework.boot:spring-boot-starter-web' compile 'org.springframework.cloud:spring-cloud-starter' compile 'org.springframework.cloud:spring-cloud-starter-zuul' compile 'com.eclipsesource.minimal-json:minimal-json:0.9.4' - testCompile 'junit:junit:4.11' testCompile 'org.mockito:mockito-all:1.10.8' testCompile 'org.springframework.boot:spring-boot-starter-test' @@ -26,40 +17,63 @@ dependencies { } +ext.pkg = [:] + pkg.name = archivesBaseName + pkg.installPath = '/opt/' + pkg.name + pkg.jarName = archivesBaseName + '-' + project.version + '.jar' + pkg.jarPath = pkg.installPath + '/' + pkg.jarName + pkg.initPath = '/etc/init.d/' + archivesBaseName + pkg.description = project.description + pkg.summary = "RCloud Gist Service" + pkg.vendor = "Mango Solutions Ltd." + pkg.url = projectUrl + pkg.buildId = new Date().format('yyyyMMddHHmmss') + pkg.version = project.version.replaceAll("-", ".") + pkg.user = 'rcloudgistservice' + pkg.group = pkg.user + ospackage { - packageName = archivesBaseName - version = project.version.replaceAll("-", ".") - release = buildId - arch = NOARCH + packageName = pkg.name + version = pkg.version + release = pkg.buildId os = LINUX - user = "rcloudgist" - group = "rcloudgist" - summary = "RCloud Gist Service" - vendor = projectVendor - packageDescription = description - packageGroup = projectPackageGroup - url = projectUrl + user = pkg.user + group = pkg.group + summary = pkg.summary + vendor = pkg.vendor + packageDescription = pkg.description + url = pkg.url type = BINARY - //preInstall file('src/main/rpm/scripts/preInstall.sh') + //create user and group + preInstall "/usr/sbin/useradd -c \"$pkg.summary\" -r -d $pkg.installPath $pkg.user 2>/dev/null || :" + + //create log directory + preInstall "mkdir -p /var/log/$pkg.name && chown $pkg.user:root /var/log/$pkg.name" - into(installPath) + into(pkg.installPath) from(jar.outputs.files) { fileMode 0500 } - from('src/main/rpm/home') { + from('src/main/pkg/home') { fileType CONFIG | NOREPLACE + filter (org.apache.tools.ant.filters.ReplaceTokens, tokens: pkg ) rename { String fileName -> if(fileName.equals("service.conf")) { fileName.replace('service', project.name + "-" + project.version) + } else if(fileName.equals("systemd.service")) { + fileName.replace('systemd', pkg.name) } else { fileName } } } - link(initPath, jarPath) + //systemv symlink + link(pkg.initPath, pkg.jarPath) + //The following link is for systemd based systems, at the moment the above systemv symlink will suffice + //link("/etc/systemd/system/" + pkg.name + ".service", pkg.installPath + "/" + pkg.name + ".service") } assemble.dependsOn distRpm diff --git a/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/GistsServiceConfiguration.java b/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/GistsServiceConfiguration.java index 318642e..b6f85e5 100644 --- a/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/GistsServiceConfiguration.java +++ b/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/GistsServiceConfiguration.java @@ -16,12 +16,12 @@ public class GistsServiceConfiguration { @Bean public ZuulFilter getUrlRewritingFilter() { - return new HeaderUrlRewritingFilter(); + return new HeaderUrlRewritingFilter(10); } @Bean public ZuulFilter getJsonContentUrlRewritingFilter() { - return new JsonContentUrlRewritingFilter(); + return new JsonContentUrlRewritingFilter(20); } } diff --git a/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/HeaderUrlRewritingFilter.java b/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/HeaderUrlRewritingFilter.java index 8713512..406da6c 100644 --- a/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/HeaderUrlRewritingFilter.java +++ b/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/HeaderUrlRewritingFilter.java @@ -3,10 +3,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.collect.Collections2.filter; -import java.net.MalformedURLException; -import java.net.URL; import java.util.Collection; -import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -24,11 +21,16 @@ public final class HeaderUrlRewritingFilter extends ZuulFilter { private static final Logger log = LoggerFactory.getLogger(HeaderUrlRewritingFilter.class); - private static final ImmutableSet DEFAULT_WHITELIST = ImmutableSet.of("Link", "Location"); + public static final ImmutableSet DEFAULT_WHITELIST = ImmutableSet.of("Link", "Location"); private final ImmutableSet whitelist = DEFAULT_WHITELIST; - + private int order = 100; + + + public HeaderUrlRewritingFilter(int order) { + this.order = order; + } @Override public String filterType() { @@ -37,7 +39,7 @@ public String filterType() { @Override public int filterOrder() { - return 100; + return order; } @Override @@ -59,6 +61,10 @@ public Object run() { } return null; } + + public ImmutableSet getWhitelist() { + return this.whitelist; + } private static void rewriteHeaders(final RequestContext context, final Collection whitelist) { assert context != null; diff --git a/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/JsonContentUrlRewritingFilter.java b/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/JsonContentUrlRewritingFilter.java index ca5488d..adf0d19 100644 --- a/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/JsonContentUrlRewritingFilter.java +++ b/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/JsonContentUrlRewritingFilter.java @@ -24,6 +24,14 @@ public class JsonContentUrlRewritingFilter extends ZuulFilter { private static final Logger logger = LoggerFactory.getLogger(JsonContentUrlRewritingFilter.class); + private static final MimeType[] JSON_MIME_TYPES = {MimeTypeUtils.APPLICATION_JSON, MimeTypeUtils.parseMimeType("application/*+json")}; + + private int order = 100; + + public JsonContentUrlRewritingFilter(int order) { + this.order = order; + } + @Override public String filterType() { return "post"; @@ -31,7 +39,7 @@ public String filterType() { @Override public int filterOrder() { - return 100; + return order; } @Override @@ -53,7 +61,12 @@ private boolean isJsonResponseContentType(List> zuulRespons String value = header.second(); try { MimeType mimeType = MimeTypeUtils.parseMimeType(value); - return mimeType.isCompatibleWith(MimeTypeUtils.APPLICATION_JSON); + for(MimeType jsonMimeType: JSON_MIME_TYPES) { + if(jsonMimeType.isCompatibleWith(mimeType)) { + return true; + } + + } } catch (InvalidMimeTypeException e) { logger.warn("Could not parse {} as a valid mimetype", value); } diff --git a/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/ZuulResponseContent.java b/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/ZuulResponseContent.java index 09c98e1..2dbe001 100644 --- a/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/ZuulResponseContent.java +++ b/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/ZuulResponseContent.java @@ -23,6 +23,10 @@ public ZuulResponseContent(RequestContext context) { this.context = context; } + /** + * Get the content from the response. + * @return the content obtained from the response. + */ public String getContent() { String content = context.getResponseBody(); if (content == null) { @@ -31,6 +35,9 @@ public String getContent() { return content; } + /** + * Clears the response content + */ public void clearContent() { if(context.getResponseDataStream() != null) { IOUtils.closeQuietly(context.getResponseDataStream()); @@ -39,6 +46,10 @@ public void clearContent() { context.setResponseBody(null); } + /** + * Sets the content onto the response. + * @param content the content to set + */ public void setContent(String content) { this.clearContent(); context.setResponseBody(content); diff --git a/rcloud-gist-service/src/main/pkg/home/application.yml b/rcloud-gist-service/src/main/pkg/home/application.yml new file mode 100644 index 0000000..7e41ce4 --- /dev/null +++ b/rcloud-gist-service/src/main/pkg/home/application.yml @@ -0,0 +1,20 @@ +server: + port: 13010 + +management: + port: 13011 + +logging: + config: logback.xml + file: /var/log/${spring.application.name}/${spring.application.name}-file.log + +github: + api: + url: https://api.github.com + + + +#security: +# user: +# name: admin +# password: secret diff --git a/rcloud-gist-service/src/main/pkg/home/logback.xml b/rcloud-gist-service/src/main/pkg/home/logback.xml new file mode 100644 index 0000000..0100a97 --- /dev/null +++ b/rcloud-gist-service/src/main/pkg/home/logback.xml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/rcloud-gist-service/src/main/pkg/home/service.conf b/rcloud-gist-service/src/main/pkg/home/service.conf new file mode 100644 index 0000000..5c8cff2 --- /dev/null +++ b/rcloud-gist-service/src/main/pkg/home/service.conf @@ -0,0 +1,49 @@ +MAX_MEM=512m +MIN_MEM=64m + +JAVA_OPTS="${JAVA_OPTS} -Xms${MIN_MEM} -Xmx${MAX_MEM}" + +# Increase maximum perm size for web base applications to 4x the default amount +# http://wiki.apache.org/tomcat/FAQ/Memoryhttp://wiki.apache.org/tomcat/FAQ/Memory +JAVA_OPTS="${JAVA_OPTS} -XX:MaxPermSize=256m" + +# Reset the default stack size for threads to a lower value (by 1/10th original) +# By default this can be anywhere between 512k -> 1024k depending on x32 or x64 +# bit Java version. +# http://www.springsource.com/files/uploads/tomcat/tomcatx-large-scale-deployments.pdf +# http://www.oracle.com/technetwork/java/hotspotfaq-138619.html +JAVA_OPTS="${JAVA_OPTS} -Xss228k" + +# Oracle Java as default, uses the serial garbage collector on the +# Full Tenured heap. The Young space is collected in parallel, but the +# Tenured is not. This means that at a time of load if a full collection +# event occurs, since the event is a 'stop-the-world' serial event then +# all application threads other than the garbage collector thread are +# taken off the CPU. This can have severe consequences if requests continue +# to accrue during these 'outage' periods. (specifically webservices, webapps) +# [Also enables adaptive sizing automatically] +JAVA_OPTS="${JAVA_OPTS} -XX:+UseParallelGC" + +# This is interpreted as a hint to the garbage collector that pause times +# of milliseconds or less are desired. The garbage collector will +# adjust the Java heap size and other garbage collection related parameters +# in an attempt to keep garbage collection pauses shorter than milliseconds. +# http://java.sun.com/docs/hotspot/gc5.0/ergo5.html +JAVA_OPTS="${JAVA_OPTS} -XX:MaxGCPauseMillis=1500" + +# A hint to the virtual machine that it.s desirable that not more than: +# 1 / (1 + GCTimeRation) of the application execution time be spent in +# the garbage collector. +# http://themindstorms.wordpress.com/2009/01/21/advanced-jvm-tuning-for-low-pause/ +JAVA_OPTS="${JAVA_OPTS} -XX:GCTimeRatio=9" + +# The hotspot server JVM has specific code-path optimizations +# which yield an approximate 10% gain over the client version. +JAVA_OPTS="${JAVA_OPTS} -server" + +# Disable remote (distributed) garbage collection by Java clients +# and remove ability for applications to call explicit GC collection +JAVA_OPTS="${JAVA_OPTS} -XX:+DisableExplicitGC" + +# LOG file is not in default location +LOG_FOLDER=/var/log/rcloud-gist-service diff --git a/rcloud-gist-service/src/main/pkg/home/systemd.service b/rcloud-gist-service/src/main/pkg/home/systemd.service new file mode 100644 index 0000000..da4668e --- /dev/null +++ b/rcloud-gist-service/src/main/pkg/home/systemd.service @@ -0,0 +1,12 @@ +[Unit] +Description="@summary@" +After=syslog.target + +[Service] +User=@user@ +Group=@group@ +ExecStart=@jarPath@ +SuccessExitStatus=143 + +[Install] +WantedBy=multi-user.target diff --git a/rcloud-gist-service/src/main/resources/application.yml b/rcloud-gist-service/src/main/resources/application.yml index c42d544..841e22a 100644 --- a/rcloud-gist-service/src/main/resources/application.yml +++ b/rcloud-gist-service/src/main/resources/application.yml @@ -2,7 +2,9 @@ server: port: 13010 management: - port: 13011 + port: 13011 + security: + enabled: true info: version: 1.0.0 @@ -10,14 +12,20 @@ info: spring: application: - name: gateway-service + name: rcloud-gist-service cloud: config: enabled: false +security: + user: + name: admin + basic: + enabled: false + zuul: routes: repos: path: /** - url: https://api.github.com + url: ${github.api.url:https://api.github.com} diff --git a/rcloud-gist-service/src/test/java/com/mangosolutions/rcloud/gists/filters/HeaderUrlRewritingFilterTest.java b/rcloud-gist-service/src/test/java/com/mangosolutions/rcloud/gists/filters/HeaderUrlRewritingFilterTest.java new file mode 100644 index 0000000..5b64694 --- /dev/null +++ b/rcloud-gist-service/src/test/java/com/mangosolutions/rcloud/gists/filters/HeaderUrlRewritingFilterTest.java @@ -0,0 +1,55 @@ +package com.mangosolutions.rcloud.gists.filters; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.List; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.netflix.util.Pair; +import com.netflix.zuul.context.RequestContext; + +public class HeaderUrlRewritingFilterTest { + + + @Before + public void setupHeaders() throws MalformedURLException { + RequestContext context = RequestContext.getCurrentContext(); + //add link header + context.addZuulResponseHeader("Link", "; rel=\"next\", ; rel=\"last\""); + //add location header + context.addZuulResponseHeader("Location", "https://api.github.com"); + //add another header + context.addZuulResponseHeader("Content-Type", "application/json"); + + context.addZuulResponseHeader("Blacklist-Header", "https://api.github.com"); + + context.addZuulRequestHeader("x-forwarded-host", "localhost:8080"); + context.addZuulRequestHeader("x-forwarded-proto", "http"); + context.addZuulRequestHeader("x-forwarded-port", "8080"); + context.setRouteHost(new URL("https://api.github.com")); + } + + @Test + public void filterHeaders() { + RequestContext context = RequestContext.getCurrentContext(); + Assert.assertNotNull(context); + HeaderUrlRewritingFilter filter = new HeaderUrlRewritingFilter(1); + filter.run(); + List> headers = context.getZuulResponseHeaders(); + for(Pair header: headers) { + if(header.first().startsWith("Blacklist")) { + Assert.assertTrue(header.second().contains("https://api.github.com")); + } else if(filter.getWhitelist().contains(header.first())) { + Assert.assertTrue(header.second().contains("http://localhost:8080")); + Assert.assertFalse(header.second().contains("https://api.github.com")); + } else { + Assert.assertFalse(header.second().contains("https://api.github.com")); + } + } + + } + +} diff --git a/rcloud-gist-service/src/test/java/com/mangosolutions/rcloud/gists/filters/JsonContentUrlRewritingFilterTest.java b/rcloud-gist-service/src/test/java/com/mangosolutions/rcloud/gists/filters/JsonContentUrlRewritingFilterTest.java new file mode 100644 index 0000000..46ada3f --- /dev/null +++ b/rcloud-gist-service/src/test/java/com/mangosolutions/rcloud/gists/filters/JsonContentUrlRewritingFilterTest.java @@ -0,0 +1,99 @@ +package com.mangosolutions.rcloud.gists.filters; + +import java.net.MalformedURLException; +import java.net.URL; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.springframework.http.MediaType; + +import com.netflix.zuul.context.RequestContext; + +public class JsonContentUrlRewritingFilterTest { + + private static final String REMOTE_URL = "https://api.github.com"; + private static final String LOCAL_URL = "http://localhost:8080"; + + @Before + public void setup() throws MalformedURLException { + setup(MediaType.APPLICATION_JSON_UTF8_VALUE); + } + + public void setup(String contentType) throws MalformedURLException { + RequestContext context = RequestContext.getCurrentContext(); + context.getZuulResponseHeaders().clear(); + context.addZuulRequestHeader("x-forwarded-host", "localhost:8080"); + context.addZuulRequestHeader("x-forwarded-proto", "http"); + context.addZuulRequestHeader("x-forwarded-port", "8080"); + context.addZuulResponseHeader("Content-Type", contentType); + context.setRouteHost(new URL("https://api.github.com")); + } + + public void setContent(String content) { + RequestContext.getCurrentContext().setResponseBody(content); + } + + @Test + public void testShouldFilterJsonContent() { + Assert.assertTrue(new JsonContentUrlRewritingFilter(1).shouldFilter()); + } + + @Test + public void testShouldNotFilterXmlContent() throws MalformedURLException { + setup(MediaType.APPLICATION_XML_VALUE); + Assert.assertFalse(new JsonContentUrlRewritingFilter(1).shouldFilter()); + } + + @Test + public void testShouldFilterJsonExtensionContent() throws MalformedURLException { + setup("application/hal+json"); + Assert.assertTrue(new JsonContentUrlRewritingFilter(1).shouldFilter()); + } + + @Test + public void replaceUrlInJsonArray() throws MalformedURLException { + String jsonArray = "[ \"https://api.github.com\" ]"; + setContent(jsonArray); + JsonContentUrlRewritingFilter filter = new JsonContentUrlRewritingFilter(1); + filter.run(); + String responseBody = RequestContext.getCurrentContext().getResponseBody(); + Assert.assertFalse(responseBody.contains(REMOTE_URL)); + Assert.assertTrue(responseBody.contains(LOCAL_URL)); + } + + @Test + public void replaceUrlInJsonObject() { + String jsonArray = "{ \"url\": \"https://api.github.com\" }"; + setContent(jsonArray); + JsonContentUrlRewritingFilter filter = new JsonContentUrlRewritingFilter(1); + filter.run(); + String responseBody = RequestContext.getCurrentContext().getResponseBody(); + Assert.assertFalse(responseBody.contains(REMOTE_URL)); + Assert.assertTrue(responseBody.contains(LOCAL_URL)); + } + + @Test + public void replaceUrlInNestedJsonObject() { + String jsonArray = "{ \"obj\": { \"url\": \"https://api.github.com\" } }"; + setContent(jsonArray); + JsonContentUrlRewritingFilter filter = new JsonContentUrlRewritingFilter(1); + filter.run(); + String responseBody = RequestContext.getCurrentContext().getResponseBody(); + Assert.assertFalse(responseBody.contains(REMOTE_URL)); + Assert.assertTrue(responseBody.contains(LOCAL_URL)); + } + + @Test + public void replaceUrlInSpecificJsonObject() { + String jsonArray = "{ \"obj\": { \"url\": \"https://api.github.com\", \"noUrl\": \"I am not a url\" } }"; + setContent(jsonArray); + JsonContentUrlRewritingFilter filter = new JsonContentUrlRewritingFilter(1); + filter.run(); + String responseBody = RequestContext.getCurrentContext().getResponseBody(); + Assert.assertFalse(responseBody.contains(REMOTE_URL)); + Assert.assertTrue(responseBody.contains(LOCAL_URL)); + Assert.assertTrue(responseBody.contains("I am not a url")); + } + +} diff --git a/rcloud-gist-service/src/test/java/com/mangosolutions/rcloud/gists/filters/ZuulResponseContentTest.java b/rcloud-gist-service/src/test/java/com/mangosolutions/rcloud/gists/filters/ZuulResponseContentTest.java new file mode 100644 index 0000000..7adc32f --- /dev/null +++ b/rcloud-gist-service/src/test/java/com/mangosolutions/rcloud/gists/filters/ZuulResponseContentTest.java @@ -0,0 +1,71 @@ +package com.mangosolutions.rcloud.gists.filters; + +import java.io.IOException; + +import org.apache.commons.io.IOUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.netflix.zuul.context.RequestContext; + + +public class ZuulResponseContentTest { + + private RequestContext context = null; + + + @Before + public void setup() { + context = new RequestContext(); + } + + @Test + public void testGetContentFromText() { + context.setResponseBody("I am some test text"); + ZuulResponseContent content = new ZuulResponseContent(context); + Assert.assertEquals("I am some test text", content.getContent()); + } + + @Test + public void testGetContentFromStream() { + context.setResponseDataStream(IOUtils.toInputStream("I am some test text")); + ZuulResponseContent content = new ZuulResponseContent(context); + Assert.assertEquals("I am some test text", content.getContent()); + } + + @Test + public void testSetContent() throws IOException { + ZuulResponseContent content = new ZuulResponseContent(context); + content.setContent("I am some test text"); + Assert.assertNull(context.getResponseDataStream()); + Assert.assertEquals("I am some test text", context.getResponseBody()); + } + + @Test + public void testClearContentWithNullTextAndStream() { + context.setResponseBody(null); + context.setResponseDataStream(null); + ZuulResponseContent content = new ZuulResponseContent(context); + content.clearContent(); + Assert.assertNull(content.getContent()); + } + + @Test + public void testClearContentWithStream() { + context.setResponseBody(null); + context.setResponseDataStream(IOUtils.toInputStream("I am some test text")); + ZuulResponseContent content = new ZuulResponseContent(context); + content.clearContent(); + Assert.assertNull(content.getContent()); + } + + @Test + public void testClearContentWithText() { + context.setResponseBody("I am some test text"); + context.setResponseDataStream(null); + ZuulResponseContent content = new ZuulResponseContent(context); + content.clearContent(); + Assert.assertNull(content.getContent()); + } +} diff --git a/salt/roots/pillar/settings.sls b/salt/roots/pillar/settings.sls new file mode 100644 index 0000000..5d51b31 --- /dev/null +++ b/salt/roots/pillar/settings.sls @@ -0,0 +1,3 @@ +rcloud: + version: "1.7" + diff --git a/salt/roots/pillar/top.sls b/salt/roots/pillar/top.sls new file mode 100644 index 0000000..cd5fa1f --- /dev/null +++ b/salt/roots/pillar/top.sls @@ -0,0 +1,3 @@ +base: + '*': + - settings diff --git a/salt/roots/salt/base/init.sls b/salt/roots/salt/base/init.sls new file mode 100644 index 0000000..30472ef --- /dev/null +++ b/salt/roots/salt/base/init.sls @@ -0,0 +1,8 @@ +system-uptodate: + pkg.uptodate: + - name: Ensure system is up to date. + - refresh: True + +iptables: + service.dead: + - enable: False diff --git a/salt/roots/salt/rcloud/init.sls b/salt/roots/salt/rcloud/init.sls new file mode 100644 index 0000000..435f767 --- /dev/null +++ b/salt/roots/salt/rcloud/init.sls @@ -0,0 +1,42 @@ +base: + pkgrepo.managed: + - humanname: MARutter PPA + - name: ppa:marutter/rrutter + +rcloud-dependencies: + pkg.installed: + - pkgs: + - openjdk-7-jdk + - gcc + - g++ + - gfortran + - libcairo-dev + - libreadline-dev + - libxt-dev + - libjpeg-dev + - libicu-dev + - libssl-dev + - libcurl4-openssl-dev + - subversion + - git + - automake + - make + - libtool + - libtiff-dev + - gettext + - redis-server + - rsync + - r-base-dev +# - nodejs +# - npm + +rcloud-deploy: + archive.extracted: + - name: /opt/rcloud + - source: https://github.com/att/rcloud/archive/1.7.tar.gz + - source_hash: md5=643eff16f448bf1306cbcd08930cfb99 + - source_hash_update: True + cmd.run: + - name: ./scripts/bootstrapR.sh + - cwd: /opt/rcloud/rcloud-1.7/ + - creates: /opt/rcloud/rcloud-1.7/conf/rcloud.conf diff --git a/salt/roots/salt/top.sls b/salt/roots/salt/top.sls new file mode 100644 index 0000000..014f577 --- /dev/null +++ b/salt/roots/salt/top.sls @@ -0,0 +1,5 @@ +#sudo salt-call --local -l info state.highstate +base: + '*': + - base + - rcloud