From 9291099e37c0c16a6d6632b76e16f748483d5c21 Mon Sep 17 00:00:00 2001 From: Andrew Dunn Date: Tue, 24 Jan 2017 10:08:24 +0000 Subject: [PATCH 01/12] Added travis file for CI builds --- .travis.yml | 3 +++ gradlew | 0 2 files changed, 3 insertions(+) create mode 100644 .travis.yml mode change 100644 => 100755 gradlew diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..e358d25 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,3 @@ +language: java +jdk: + - oraclejdk7 diff --git a/gradlew b/gradlew old mode 100644 new mode 100755 From 0792fb13bfe46996685201787448088211d1dee4 Mon Sep 17 00:00:00 2001 From: Andrew Dunn Date: Tue, 24 Jan 2017 10:18:50 +0000 Subject: [PATCH 02/12] Added travis build badge to the readme --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 4f7eb2f..6825e2f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ +[![Build Status](https://travis-ci.org/MangoTheCat/rcloud-gist-services.svg?branch=master)](https://travis-ci.org/MangoTheCat/rcloud-gist-services) + # RCloud Gist Service + + ## Overview The RCloud Gist Service is a Java based service for enabling gist access to various different backend storage systems. From 234aef0c646e76982ea285cf95241af4310b4abe Mon Sep 17 00:00:00 2001 From: Andrew Dunn Date: Tue, 24 Jan 2017 13:56:01 +0000 Subject: [PATCH 03/12] Vagrantfile now 90% preconfigured environment --- Vagrantfile | 119 ++++++++++++++++++++++++++++++++ salt/roots/pillar/settings.sls | 3 + salt/roots/pillar/top.sls | 3 + salt/roots/salt/base/init.sls | 8 +++ salt/roots/salt/rcloud/init.sls | 42 +++++++++++ salt/roots/salt/top.sls | 5 ++ 6 files changed, 180 insertions(+) create mode 100644 Vagrantfile create mode 100644 salt/roots/pillar/settings.sls create mode 100644 salt/roots/pillar/top.sls create mode 100644 salt/roots/salt/base/init.sls create mode 100644 salt/roots/salt/rcloud/init.sls create mode 100644 salt/roots/salt/top.sls diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000..4139d88 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,119 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# All Vagrant configuration is done below. The "2" in Vagrant.configure +# configures the configuration version (we support older styles for +# backwards compatibility). Please don't change it unless you know what +# you're doing. +Vagrant.configure(2) do |config| + # The most common configuration options are documented and commented below. + # For a complete reference, please see the online documentation at + # https://docs.vagrantup.com. + + # Every Vagrant development environment requires a box. You can search for + # boxes at https://atlas.hashicorp.com/search. + #config.vm.box = "ubuntu/trusty64" + config.vm.box = "janihur/ubuntu-1404-desktop" + + + config.vm.hostname = "rcloud.local" + + # Disable automatic box update checking. If you disable this, then + # boxes will only be checked for updates when the user runs + # `vagrant box outdated`. This is not recommended. + # config.vm.box_check_update = false + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine. In the example below, + # accessing "localhost:8080" will access port 80 on the guest machine. + config.vm.network "forwarded_port", guest: 80, host: 80 + config.vm.network "forwarded_port", guest: 8080, host: 8080 + + #config.ssh.forward_agent = true + + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + # config.vm.network "private_network", ip: "192.168.33.10" + config.vm.network "private_network", type: "dhcp" + + # Need to manage the hosts file to add in the real ip address, using + # vagrant-hostmanager for this: + # (https://github.com/devopsgroup-io/vagrant-hostmanager) + # To install run vagrant plugin install vagrant-hostmanager + config.hostmanager.enabled = true + config.hostmanager.manage_host = true + config.hostmanager.manage_guest = true + config.hostmanager.ignore_private_ip = false + config.hostmanager.include_offline = true + config.hostmanager.ip_resolver = proc do |vm, resolving_vm| + if hostname = (vm.ssh_info && vm.ssh_info[:host]) + `vagrant ssh -c "hostname -I"`.split()[1] + end + end + + # Create a public network, which generally matched to bridged network. + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network "public_network" + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. + # config.vm.synced_folder "../data", "/vagrant_data" + + # Provider-specific configuration so you can fine-tune various + # backing providers for Vagrant. These expose provider-specific options. + # Example for VirtualBox: + # + config.vm.provider "virtualbox" do |vb| + # Display the VirtualBox GUI when booting the machine + vb.gui = true + # Customize the amount of memory on the VM: + vb.memory = "4096" + vb.cpus = "2" + vb.customize ["modifyvm", :id, "--ioapic", "on"] + vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] + end + # + # View the documentation for the provider you are using for more + # information on available options. + + # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies + # such as FTP and Heroku are also available. See the documentation at + # https://docs.vagrantup.com/v2/push/atlas.html for more information. + # config.push.define "atlas" do |push| + # push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME" + # end + + # Adds the hostname of this machine into the hosts file + config.vm.provision :shell, inline: "sed -i'' 's/^127.0.0.1\\t#{config.vm.hostname}.*$//' /etc/hosts" + # Enable provisioning with a shell script. Additional provisioners such as + # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the + # documentation for more information about their specific syntax and use. + + config.vm.provision "shell", inline: "sudo apt-get update" + config.vm.provision "shell", inline: "sudo apt-get install -y curl" + #config.vm.provision "shell", inline: "sudo /usr/share/debconf/fix_db.pl" + #config.vm.provision "shell", inline: "sudo apt-get upgrade" + #config.vm.provision "shell", inline: "sudo apt-get install -y xfce4 virtualbox-guest-dkms virtualbox-guest-utils virtualbox-guest-x11" + #config.vm.provision "shell", inline: "sudo sed -i 's/allowed_users=.*$/allowed_users=anybody/' /etc/X11/Xwrapper.config" + + + config.vm.provision "shell", inline: <<-SHELL + curl -L https://bootstrap.saltstack.com | sudo sh -s -- stable + SHELL + + config.vm.synced_folder "salt/roots/salt", "/srv/salt/" + config.vm.synced_folder "salt/roots/pillar/", "/srv/pillar/" + +# config.vm.provision :salt do |salt| +# salt.bootstrap_options = "-F -c /tmp -P" +## salt.minion_config = "salt/minion.yml" +# salt.run_highstate = true +# salt.colorize = true +# salt.log_level = 'debug' +# end + +end diff --git a/salt/roots/pillar/settings.sls b/salt/roots/pillar/settings.sls new file mode 100644 index 0000000..5d51b31 --- /dev/null +++ b/salt/roots/pillar/settings.sls @@ -0,0 +1,3 @@ +rcloud: + version: "1.7" + diff --git a/salt/roots/pillar/top.sls b/salt/roots/pillar/top.sls new file mode 100644 index 0000000..cd5fa1f --- /dev/null +++ b/salt/roots/pillar/top.sls @@ -0,0 +1,3 @@ +base: + '*': + - settings diff --git a/salt/roots/salt/base/init.sls b/salt/roots/salt/base/init.sls new file mode 100644 index 0000000..30472ef --- /dev/null +++ b/salt/roots/salt/base/init.sls @@ -0,0 +1,8 @@ +system-uptodate: + pkg.uptodate: + - name: Ensure system is up to date. + - refresh: True + +iptables: + service.dead: + - enable: False diff --git a/salt/roots/salt/rcloud/init.sls b/salt/roots/salt/rcloud/init.sls new file mode 100644 index 0000000..435f767 --- /dev/null +++ b/salt/roots/salt/rcloud/init.sls @@ -0,0 +1,42 @@ +base: + pkgrepo.managed: + - humanname: MARutter PPA + - name: ppa:marutter/rrutter + +rcloud-dependencies: + pkg.installed: + - pkgs: + - openjdk-7-jdk + - gcc + - g++ + - gfortran + - libcairo-dev + - libreadline-dev + - libxt-dev + - libjpeg-dev + - libicu-dev + - libssl-dev + - libcurl4-openssl-dev + - subversion + - git + - automake + - make + - libtool + - libtiff-dev + - gettext + - redis-server + - rsync + - r-base-dev +# - nodejs +# - npm + +rcloud-deploy: + archive.extracted: + - name: /opt/rcloud + - source: https://github.com/att/rcloud/archive/1.7.tar.gz + - source_hash: md5=643eff16f448bf1306cbcd08930cfb99 + - source_hash_update: True + cmd.run: + - name: ./scripts/bootstrapR.sh + - cwd: /opt/rcloud/rcloud-1.7/ + - creates: /opt/rcloud/rcloud-1.7/conf/rcloud.conf diff --git a/salt/roots/salt/top.sls b/salt/roots/salt/top.sls new file mode 100644 index 0000000..014f577 --- /dev/null +++ b/salt/roots/salt/top.sls @@ -0,0 +1,5 @@ +#sudo salt-call --local -l info state.highstate +base: + '*': + - base + - rcloud From 20d54e144fc58f8b75b877c80ebc996461d0df7a Mon Sep 17 00:00:00 2001 From: Andrew Dunn Date: Tue, 24 Jan 2017 13:57:41 +0000 Subject: [PATCH 04/12] Added unit test for the ZuulResponseContent class. --- .../gists/GistsServiceConfiguration.java | 4 +- .../filters/HeaderUrlRewritingFilter.java | 12 ++-- .../JsonContentUrlRewritingFilter.java | 8 ++- .../gists/filters/ZuulResponseContent.java | 11 +++ .../filters/ZuulResponseContentTest.java | 71 +++++++++++++++++++ 5 files changed, 98 insertions(+), 8 deletions(-) create mode 100644 rcloud-gist-service/src/test/java/com/mangosolutions/rcloud/gists/filters/ZuulResponseContentTest.java diff --git a/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/GistsServiceConfiguration.java b/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/GistsServiceConfiguration.java index 318642e..b6f85e5 100644 --- a/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/GistsServiceConfiguration.java +++ b/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/GistsServiceConfiguration.java @@ -16,12 +16,12 @@ public class GistsServiceConfiguration { @Bean public ZuulFilter getUrlRewritingFilter() { - return new HeaderUrlRewritingFilter(); + return new HeaderUrlRewritingFilter(10); } @Bean public ZuulFilter getJsonContentUrlRewritingFilter() { - return new JsonContentUrlRewritingFilter(); + return new JsonContentUrlRewritingFilter(20); } } diff --git a/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/HeaderUrlRewritingFilter.java b/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/HeaderUrlRewritingFilter.java index 8713512..033f385 100644 --- a/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/HeaderUrlRewritingFilter.java +++ b/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/HeaderUrlRewritingFilter.java @@ -3,10 +3,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.collect.Collections2.filter; -import java.net.MalformedURLException; -import java.net.URL; import java.util.Collection; -import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -28,7 +25,12 @@ public final class HeaderUrlRewritingFilter extends ZuulFilter { private final ImmutableSet whitelist = DEFAULT_WHITELIST; - + private int order = 100; + + + public HeaderUrlRewritingFilter(int order) { + this.order = order; + } @Override public String filterType() { @@ -37,7 +39,7 @@ public String filterType() { @Override public int filterOrder() { - return 100; + return order; } @Override diff --git a/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/JsonContentUrlRewritingFilter.java b/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/JsonContentUrlRewritingFilter.java index ca5488d..943ee32 100644 --- a/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/JsonContentUrlRewritingFilter.java +++ b/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/JsonContentUrlRewritingFilter.java @@ -24,6 +24,12 @@ public class JsonContentUrlRewritingFilter extends ZuulFilter { private static final Logger logger = LoggerFactory.getLogger(JsonContentUrlRewritingFilter.class); + private int order = 100; + + public JsonContentUrlRewritingFilter(int order) { + this.order = order; + } + @Override public String filterType() { return "post"; @@ -31,7 +37,7 @@ public String filterType() { @Override public int filterOrder() { - return 100; + return order; } @Override diff --git a/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/ZuulResponseContent.java b/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/ZuulResponseContent.java index 09c98e1..2dbe001 100644 --- a/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/ZuulResponseContent.java +++ b/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/ZuulResponseContent.java @@ -23,6 +23,10 @@ public ZuulResponseContent(RequestContext context) { this.context = context; } + /** + * Get the content from the response. + * @return the content obtained from the response. + */ public String getContent() { String content = context.getResponseBody(); if (content == null) { @@ -31,6 +35,9 @@ public String getContent() { return content; } + /** + * Clears the response content + */ public void clearContent() { if(context.getResponseDataStream() != null) { IOUtils.closeQuietly(context.getResponseDataStream()); @@ -39,6 +46,10 @@ public void clearContent() { context.setResponseBody(null); } + /** + * Sets the content onto the response. + * @param content the content to set + */ public void setContent(String content) { this.clearContent(); context.setResponseBody(content); diff --git a/rcloud-gist-service/src/test/java/com/mangosolutions/rcloud/gists/filters/ZuulResponseContentTest.java b/rcloud-gist-service/src/test/java/com/mangosolutions/rcloud/gists/filters/ZuulResponseContentTest.java new file mode 100644 index 0000000..7adc32f --- /dev/null +++ b/rcloud-gist-service/src/test/java/com/mangosolutions/rcloud/gists/filters/ZuulResponseContentTest.java @@ -0,0 +1,71 @@ +package com.mangosolutions.rcloud.gists.filters; + +import java.io.IOException; + +import org.apache.commons.io.IOUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.netflix.zuul.context.RequestContext; + + +public class ZuulResponseContentTest { + + private RequestContext context = null; + + + @Before + public void setup() { + context = new RequestContext(); + } + + @Test + public void testGetContentFromText() { + context.setResponseBody("I am some test text"); + ZuulResponseContent content = new ZuulResponseContent(context); + Assert.assertEquals("I am some test text", content.getContent()); + } + + @Test + public void testGetContentFromStream() { + context.setResponseDataStream(IOUtils.toInputStream("I am some test text")); + ZuulResponseContent content = new ZuulResponseContent(context); + Assert.assertEquals("I am some test text", content.getContent()); + } + + @Test + public void testSetContent() throws IOException { + ZuulResponseContent content = new ZuulResponseContent(context); + content.setContent("I am some test text"); + Assert.assertNull(context.getResponseDataStream()); + Assert.assertEquals("I am some test text", context.getResponseBody()); + } + + @Test + public void testClearContentWithNullTextAndStream() { + context.setResponseBody(null); + context.setResponseDataStream(null); + ZuulResponseContent content = new ZuulResponseContent(context); + content.clearContent(); + Assert.assertNull(content.getContent()); + } + + @Test + public void testClearContentWithStream() { + context.setResponseBody(null); + context.setResponseDataStream(IOUtils.toInputStream("I am some test text")); + ZuulResponseContent content = new ZuulResponseContent(context); + content.clearContent(); + Assert.assertNull(content.getContent()); + } + + @Test + public void testClearContentWithText() { + context.setResponseBody("I am some test text"); + context.setResponseDataStream(null); + ZuulResponseContent content = new ZuulResponseContent(context); + content.clearContent(); + Assert.assertNull(content.getContent()); + } +} From e2c9eec743edad0c74b7046f671a89e0980582c7 Mon Sep 17 00:00:00 2001 From: Andrew Dunn Date: Tue, 24 Jan 2017 17:05:18 +0000 Subject: [PATCH 05/12] Now runs the provisioning using salt as part of the startup of the machine. Added a bit of documentation. --- README.md | 4 ++++ Vagrantfile | 6 ++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6825e2f..41a2694 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,10 @@ There are two mechanisms to load the project into the Eclipse IDE: 1. Use the Eclipse gradle plugin [BuildShip](https://github.com/eclipse/buildship), this is an Eclipse plugin that understands gradle projects. BuildShip does not include syntax highlighting editor, you will have to install Groovy Eclipse plugin. 2. Use the gradle eclipse plugin (this is a plugin in the gradle build file that will generate the appropriate eclipse project). To generate the eclipse files run the following `gradlew eclipse`, you can then import the project in as an existing project. When you add a new dependency in you will need to run this again to regenerate the eclipse project files and then refresh the project in eclipse and it will pick up the new settings. If you want to just generate the eclipse files for a specific sub module then run the command `gradlew :store:eclipse` for the store sub project. + +## Vagrant +The Vagrantfile sets up 80% of the environment needed to run rcloud. It takes a long time to finish the provisioning. Once done rcloud will be in `/opt/rcloud/rcloud-1.7/` you will need to setup the `rcloud.conf` file and then call `sudo ./scrtips/fresh_start.sh`. The `bootstrapR.sh` has already been called as part of the provisioning. + ## Components ## LICENSE & Copyright diff --git a/Vagrantfile b/Vagrantfile index 4139d88..c175ac0 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -108,12 +108,14 @@ Vagrant.configure(2) do |config| config.vm.synced_folder "salt/roots/salt", "/srv/salt/" config.vm.synced_folder "salt/roots/pillar/", "/srv/pillar/" -# config.vm.provision :salt do |salt| + config.vm.provision :salt do |salt| + salt.masterless = true + salt.run_highstate = true # salt.bootstrap_options = "-F -c /tmp -P" ## salt.minion_config = "salt/minion.yml" # salt.run_highstate = true # salt.colorize = true # salt.log_level = 'debug' -# end + end end From bc5a5b0462c9524bc3a80380dd8051c51e3065d1 Mon Sep 17 00:00:00 2001 From: Andrew Dunn Date: Tue, 24 Jan 2017 17:08:07 +0000 Subject: [PATCH 06/12] Now creating a debian archive as well as an rpm. --- build.gradle | 7 ++- rcloud-gist-service/build.gradle | 15 +++--- .../src/main/pkg/home/application.yml | 7 +++ .../src/main/pkg/home/logback.xml | 9 ++++ .../src/main/pkg/home/service.conf | 49 +++++++++++++++++++ .../src/main/pkg/scripts/preInstall.sh | 7 +++ .../src/main/resources/application.yml | 4 +- 7 files changed, 86 insertions(+), 12 deletions(-) create mode 100644 rcloud-gist-service/src/main/pkg/home/application.yml create mode 100644 rcloud-gist-service/src/main/pkg/home/logback.xml create mode 100644 rcloud-gist-service/src/main/pkg/home/service.conf create mode 100644 rcloud-gist-service/src/main/pkg/scripts/preInstall.sh diff --git a/build.gradle b/build.gradle index 7cb961e..26daa70 100644 --- a/build.gradle +++ b/build.gradle @@ -152,10 +152,13 @@ configure(subprojects.findAll {it.name =~ /.*service/}) { } */ - task distRpm(type: Rpm, dependsOn: bootRepackage) { + task distRpm(type: Rpm, dependsOn: bootRepackage) {} + + task distDeb(type: Deb, dependsOn: bootRepackage) {} - } assemble.dependsOn distRpm + assemble.dependsOn distDeb + } diff --git a/rcloud-gist-service/build.gradle b/rcloud-gist-service/build.gradle index 0c38a52..f849aca 100644 --- a/rcloud-gist-service/build.gradle +++ b/rcloud-gist-service/build.gradle @@ -1,18 +1,17 @@ -ext.serviceInstallPath = '/opt/rcloud-gist-service' +ext.serviceInstallPath = '/opt' ext.installPath = serviceInstallPath + '/' + archivesBaseName ext.jarPath = installPath + '/' + archivesBaseName + '-' + project.version + '.jar' ext.initPath = '/etc/init.d/' + archivesBaseName project.description = "RCloud Gist service providing gist access api." ext.projectVendor = "TBD" -ext.projectUrl = "TBD" +ext.projectUrl = "https://github.com/MangoTheCat/rcloud-gist-services" ext.projectPackageGroup = "TBD" ext.buildId = new Date().format('yyyyMMddHHmmss') dependencies { compile 'org.springframework.boot:spring-boot-starter' compile 'org.springframework.boot:spring-boot-starter-actuator' - //compile 'org.springframework.boot:spring-boot-starter-undertow' compile 'org.springframework.boot:spring-boot-starter-web' compile 'org.springframework.cloud:spring-cloud-starter' compile 'org.springframework.cloud:spring-cloud-starter-zuul' @@ -30,10 +29,10 @@ ospackage { packageName = archivesBaseName version = project.version.replaceAll("-", ".") release = buildId - arch = NOARCH + //arch = NOARCH os = LINUX - user = "rcloudgist" - group = "rcloudgist" + user = "rcloudgistservice" + group = "rcloudgistservice" summary = "RCloud Gist Service" vendor = projectVendor packageDescription = description @@ -41,7 +40,7 @@ ospackage { url = projectUrl type = BINARY - //preInstall file('src/main/rpm/scripts/preInstall.sh') + preInstall file('src/main/pkg/scripts/preInstall.sh') into(installPath) @@ -49,7 +48,7 @@ ospackage { fileMode 0500 } - from('src/main/rpm/home') { + from('src/main/pkg/home') { fileType CONFIG | NOREPLACE rename { String fileName -> if(fileName.equals("service.conf")) { diff --git a/rcloud-gist-service/src/main/pkg/home/application.yml b/rcloud-gist-service/src/main/pkg/home/application.yml new file mode 100644 index 0000000..54fc32a --- /dev/null +++ b/rcloud-gist-service/src/main/pkg/home/application.yml @@ -0,0 +1,7 @@ +logging: + config: logback.xml + file: /var/log/${spring.application.name}/${spring.application.name}-file.log + +github: + api: + url: https://api.github.com diff --git a/rcloud-gist-service/src/main/pkg/home/logback.xml b/rcloud-gist-service/src/main/pkg/home/logback.xml new file mode 100644 index 0000000..0100a97 --- /dev/null +++ b/rcloud-gist-service/src/main/pkg/home/logback.xml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/rcloud-gist-service/src/main/pkg/home/service.conf b/rcloud-gist-service/src/main/pkg/home/service.conf new file mode 100644 index 0000000..5c8cff2 --- /dev/null +++ b/rcloud-gist-service/src/main/pkg/home/service.conf @@ -0,0 +1,49 @@ +MAX_MEM=512m +MIN_MEM=64m + +JAVA_OPTS="${JAVA_OPTS} -Xms${MIN_MEM} -Xmx${MAX_MEM}" + +# Increase maximum perm size for web base applications to 4x the default amount +# http://wiki.apache.org/tomcat/FAQ/Memoryhttp://wiki.apache.org/tomcat/FAQ/Memory +JAVA_OPTS="${JAVA_OPTS} -XX:MaxPermSize=256m" + +# Reset the default stack size for threads to a lower value (by 1/10th original) +# By default this can be anywhere between 512k -> 1024k depending on x32 or x64 +# bit Java version. +# http://www.springsource.com/files/uploads/tomcat/tomcatx-large-scale-deployments.pdf +# http://www.oracle.com/technetwork/java/hotspotfaq-138619.html +JAVA_OPTS="${JAVA_OPTS} -Xss228k" + +# Oracle Java as default, uses the serial garbage collector on the +# Full Tenured heap. The Young space is collected in parallel, but the +# Tenured is not. This means that at a time of load if a full collection +# event occurs, since the event is a 'stop-the-world' serial event then +# all application threads other than the garbage collector thread are +# taken off the CPU. This can have severe consequences if requests continue +# to accrue during these 'outage' periods. (specifically webservices, webapps) +# [Also enables adaptive sizing automatically] +JAVA_OPTS="${JAVA_OPTS} -XX:+UseParallelGC" + +# This is interpreted as a hint to the garbage collector that pause times +# of milliseconds or less are desired. The garbage collector will +# adjust the Java heap size and other garbage collection related parameters +# in an attempt to keep garbage collection pauses shorter than milliseconds. +# http://java.sun.com/docs/hotspot/gc5.0/ergo5.html +JAVA_OPTS="${JAVA_OPTS} -XX:MaxGCPauseMillis=1500" + +# A hint to the virtual machine that it.s desirable that not more than: +# 1 / (1 + GCTimeRation) of the application execution time be spent in +# the garbage collector. +# http://themindstorms.wordpress.com/2009/01/21/advanced-jvm-tuning-for-low-pause/ +JAVA_OPTS="${JAVA_OPTS} -XX:GCTimeRatio=9" + +# The hotspot server JVM has specific code-path optimizations +# which yield an approximate 10% gain over the client version. +JAVA_OPTS="${JAVA_OPTS} -server" + +# Disable remote (distributed) garbage collection by Java clients +# and remove ability for applications to call explicit GC collection +JAVA_OPTS="${JAVA_OPTS} -XX:+DisableExplicitGC" + +# LOG file is not in default location +LOG_FOLDER=/var/log/rcloud-gist-service diff --git a/rcloud-gist-service/src/main/pkg/scripts/preInstall.sh b/rcloud-gist-service/src/main/pkg/scripts/preInstall.sh new file mode 100644 index 0000000..934ba8e --- /dev/null +++ b/rcloud-gist-service/src/main/pkg/scripts/preInstall.sh @@ -0,0 +1,7 @@ +# create the rcloudgistservice user and rcloudgistservice +/usr/sbin/useradd -c "RCloud Gist Service" \ + -r -d /opt/rcloud-gist-service rcloudgistservice 2>/dev/null || : + +LOG_DIR=/var/log/rcloud-gist-service +mkdir -p $LOG_DIR +chown rcloudgistservice:root $LOG_DIR diff --git a/rcloud-gist-service/src/main/resources/application.yml b/rcloud-gist-service/src/main/resources/application.yml index c42d544..4dce046 100644 --- a/rcloud-gist-service/src/main/resources/application.yml +++ b/rcloud-gist-service/src/main/resources/application.yml @@ -10,7 +10,7 @@ info: spring: application: - name: gateway-service + name: rcloud-gist-service cloud: config: enabled: false @@ -20,4 +20,4 @@ zuul: routes: repos: path: /** - url: https://api.github.com + url: ${github.api.url:https://api.github.com} From f938a8eae6f847d5328ef4fa2c16f39d9c89e9fe Mon Sep 17 00:00:00 2001 From: Andrew Dunn Date: Wed, 25 Jan 2017 14:34:02 +0000 Subject: [PATCH 07/12] Updated the build script to include a systemd service descriptor as well as refactoring the build script to make it a bit more generic --- build.gradle | 32 +--------- rcloud-gist-service/build.gradle | 60 ++++++++++++------- .../src/main/pkg/home/systemd.service | 12 ++++ .../src/main/pkg/scripts/preInstall.sh | 7 --- 4 files changed, 50 insertions(+), 61 deletions(-) create mode 100644 rcloud-gist-service/src/main/pkg/home/systemd.service delete mode 100644 rcloud-gist-service/src/main/pkg/scripts/preInstall.sh diff --git a/build.gradle b/build.gradle index 26daa70..7e7abe5 100644 --- a/build.gradle +++ b/build.gradle @@ -38,7 +38,7 @@ allprojects { } group = 'com.mangosolutions.rcloud' project.version = scmVersion.version - + ext.projectUrl = "https://github.com/MangoTheCat/rcloud-gist-services" } subprojects { @@ -110,15 +110,6 @@ configure(subprojects.findAll {it.name =~ /.*service/}) { buildInfo() } - ext.installPath='/opt/' + archivesBaseName - ext.jarPath=installPath + '/' + archivesBaseName + '-' + scmVersion.version + '.jar' - ext.initPath='/etc/init.d/' + archivesBaseName - project.description = "TBD" - ext.projectVendor = "TBD" - ext.projectUrl = "TBD" - ext.projectPackageGroup = "TBD" - ext.buildId = new Date().format('yyyyMMddHHmmss') - dependencies { compile("org.springframework.boot:spring-boot-starter-undertow:1.4.3.RELEASE") } @@ -131,27 +122,6 @@ configure(subprojects.findAll {it.name =~ /.*service/}) { archives buildRpm } -/* - ospackage { - packageName = archivesBaseName - version = project.version.replaceAll("-", ".") - release = 1 - arch = NOARCH - os = LINUX - user = "prov" - group = "prov" - - into installPath - - from(jar.outputs.files) { - fileMode 0500 - } - - link(initPath, jarPath) - - } -*/ - task distRpm(type: Rpm, dependsOn: bootRepackage) {} task distDeb(type: Deb, dependsOn: bootRepackage) {} diff --git a/rcloud-gist-service/build.gradle b/rcloud-gist-service/build.gradle index f849aca..718b518 100644 --- a/rcloud-gist-service/build.gradle +++ b/rcloud-gist-service/build.gradle @@ -1,13 +1,5 @@ -ext.serviceInstallPath = '/opt' -ext.installPath = serviceInstallPath + '/' + archivesBaseName -ext.jarPath = installPath + '/' + archivesBaseName + '-' + project.version + '.jar' -ext.initPath = '/etc/init.d/' + archivesBaseName project.description = "RCloud Gist service providing gist access api." -ext.projectVendor = "TBD" -ext.projectUrl = "https://github.com/MangoTheCat/rcloud-gist-services" -ext.projectPackageGroup = "TBD" -ext.buildId = new Date().format('yyyyMMddHHmmss') dependencies { compile 'org.springframework.boot:spring-boot-starter' @@ -17,7 +9,6 @@ dependencies { compile 'org.springframework.cloud:spring-cloud-starter-zuul' compile 'com.eclipsesource.minimal-json:minimal-json:0.9.4' - testCompile 'junit:junit:4.11' testCompile 'org.mockito:mockito-all:1.10.8' testCompile 'org.springframework.boot:spring-boot-starter-test' @@ -25,24 +16,41 @@ dependencies { } +ext.pkg = [:] + pkg.name = archivesBaseName + pkg.installPath = '/opt/' + pkg.name + pkg.jarName = archivesBaseName + '-' + project.version + '.jar' + pkg.jarPath = pkg.installPath + '/' + pkg.jarName + pkg.initPath = '/etc/init.d/' + archivesBaseName + pkg.description = project.description + pkg.summary = "RCloud Gist Service" + pkg.vendor = "Mango Solutions Ltd." + pkg.url = projectUrl + pkg.buildId = new Date().format('yyyyMMddHHmmss') + pkg.version = project.version.replaceAll("-", ".") + pkg.user = 'rcloudgistservice' + pkg.group = pkg.user + ospackage { - packageName = archivesBaseName - version = project.version.replaceAll("-", ".") - release = buildId - //arch = NOARCH + packageName = pkg.name + version = pkg.version + release = pkg.buildId os = LINUX - user = "rcloudgistservice" - group = "rcloudgistservice" - summary = "RCloud Gist Service" - vendor = projectVendor - packageDescription = description - packageGroup = projectPackageGroup - url = projectUrl + user = pkg.user + group = pkg.group + summary = pkg.summary + vendor = pkg.vendor + packageDescription = pkg.description + url = pkg.url type = BINARY - preInstall file('src/main/pkg/scripts/preInstall.sh') + //create user and group + preInstall "/usr/sbin/useradd -c \"$pkg.summary\" -r -d $pkg.installPath $pkg.user 2>/dev/null || :" + + //create log directory + preInstall "mkdir -p /var/log/$pkg.name && chown $pkg.user:root /var/log/$pkg.name" - into(installPath) + into(pkg.installPath) from(jar.outputs.files) { fileMode 0500 @@ -50,15 +58,21 @@ ospackage { from('src/main/pkg/home') { fileType CONFIG | NOREPLACE + filter (org.apache.tools.ant.filters.ReplaceTokens, tokens: pkg ) rename { String fileName -> if(fileName.equals("service.conf")) { fileName.replace('service', project.name + "-" + project.version) + } else if(fileName.equals("systemd.service")) { + fileName.replace('systemd', pkg.name) } else { fileName } } } - link(initPath, jarPath) + //systemv symlink + link(pkg.initPath, pkg.jarPath) + //The following link is for systemd based systems, at the moment the above systemv symlink will suffice + //link("/etc/systemd/system/" + pkg.name + ".service", pkg.installPath + "/" + pkg.name + ".service") } assemble.dependsOn distRpm diff --git a/rcloud-gist-service/src/main/pkg/home/systemd.service b/rcloud-gist-service/src/main/pkg/home/systemd.service new file mode 100644 index 0000000..da4668e --- /dev/null +++ b/rcloud-gist-service/src/main/pkg/home/systemd.service @@ -0,0 +1,12 @@ +[Unit] +Description="@summary@" +After=syslog.target + +[Service] +User=@user@ +Group=@group@ +ExecStart=@jarPath@ +SuccessExitStatus=143 + +[Install] +WantedBy=multi-user.target diff --git a/rcloud-gist-service/src/main/pkg/scripts/preInstall.sh b/rcloud-gist-service/src/main/pkg/scripts/preInstall.sh deleted file mode 100644 index 934ba8e..0000000 --- a/rcloud-gist-service/src/main/pkg/scripts/preInstall.sh +++ /dev/null @@ -1,7 +0,0 @@ -# create the rcloudgistservice user and rcloudgistservice -/usr/sbin/useradd -c "RCloud Gist Service" \ - -r -d /opt/rcloud-gist-service rcloudgistservice 2>/dev/null || : - -LOG_DIR=/var/log/rcloud-gist-service -mkdir -p $LOG_DIR -chown rcloudgistservice:root $LOG_DIR From f80bdbf4de3c607e6b68a7d7ba18984602dd9c50 Mon Sep 17 00:00:00 2001 From: Andrew Dunn Date: Thu, 26 Jan 2017 10:09:50 +0000 Subject: [PATCH 08/12] Added test for the HeaderUrlRewritingFilter --- .../filters/HeaderUrlRewritingFilter.java | 6 +- .../filters/HeaderUrlRewritingFilterTest.java | 55 +++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 rcloud-gist-service/src/test/java/com/mangosolutions/rcloud/gists/filters/HeaderUrlRewritingFilterTest.java diff --git a/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/HeaderUrlRewritingFilter.java b/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/HeaderUrlRewritingFilter.java index 033f385..406da6c 100644 --- a/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/HeaderUrlRewritingFilter.java +++ b/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/HeaderUrlRewritingFilter.java @@ -21,7 +21,7 @@ public final class HeaderUrlRewritingFilter extends ZuulFilter { private static final Logger log = LoggerFactory.getLogger(HeaderUrlRewritingFilter.class); - private static final ImmutableSet DEFAULT_WHITELIST = ImmutableSet.of("Link", "Location"); + public static final ImmutableSet DEFAULT_WHITELIST = ImmutableSet.of("Link", "Location"); private final ImmutableSet whitelist = DEFAULT_WHITELIST; @@ -61,6 +61,10 @@ public Object run() { } return null; } + + public ImmutableSet getWhitelist() { + return this.whitelist; + } private static void rewriteHeaders(final RequestContext context, final Collection whitelist) { assert context != null; diff --git a/rcloud-gist-service/src/test/java/com/mangosolutions/rcloud/gists/filters/HeaderUrlRewritingFilterTest.java b/rcloud-gist-service/src/test/java/com/mangosolutions/rcloud/gists/filters/HeaderUrlRewritingFilterTest.java new file mode 100644 index 0000000..5b64694 --- /dev/null +++ b/rcloud-gist-service/src/test/java/com/mangosolutions/rcloud/gists/filters/HeaderUrlRewritingFilterTest.java @@ -0,0 +1,55 @@ +package com.mangosolutions.rcloud.gists.filters; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.List; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.netflix.util.Pair; +import com.netflix.zuul.context.RequestContext; + +public class HeaderUrlRewritingFilterTest { + + + @Before + public void setupHeaders() throws MalformedURLException { + RequestContext context = RequestContext.getCurrentContext(); + //add link header + context.addZuulResponseHeader("Link", "; rel=\"next\", ; rel=\"last\""); + //add location header + context.addZuulResponseHeader("Location", "https://api.github.com"); + //add another header + context.addZuulResponseHeader("Content-Type", "application/json"); + + context.addZuulResponseHeader("Blacklist-Header", "https://api.github.com"); + + context.addZuulRequestHeader("x-forwarded-host", "localhost:8080"); + context.addZuulRequestHeader("x-forwarded-proto", "http"); + context.addZuulRequestHeader("x-forwarded-port", "8080"); + context.setRouteHost(new URL("https://api.github.com")); + } + + @Test + public void filterHeaders() { + RequestContext context = RequestContext.getCurrentContext(); + Assert.assertNotNull(context); + HeaderUrlRewritingFilter filter = new HeaderUrlRewritingFilter(1); + filter.run(); + List> headers = context.getZuulResponseHeaders(); + for(Pair header: headers) { + if(header.first().startsWith("Blacklist")) { + Assert.assertTrue(header.second().contains("https://api.github.com")); + } else if(filter.getWhitelist().contains(header.first())) { + Assert.assertTrue(header.second().contains("http://localhost:8080")); + Assert.assertFalse(header.second().contains("https://api.github.com")); + } else { + Assert.assertFalse(header.second().contains("https://api.github.com")); + } + } + + } + +} From c279c65019b6a4df182b2c8f694ee05accf4fb45 Mon Sep 17 00:00:00 2001 From: Andrew Dunn Date: Thu, 26 Jan 2017 12:31:23 +0000 Subject: [PATCH 09/12] Updated the unit tests for the content url rewriting.. --- .../JsonContentUrlRewritingFilter.java | 9 +- .../JsonContentUrlRewritingFilterTest.java | 99 +++++++++++++++++++ 2 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 rcloud-gist-service/src/test/java/com/mangosolutions/rcloud/gists/filters/JsonContentUrlRewritingFilterTest.java diff --git a/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/JsonContentUrlRewritingFilter.java b/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/JsonContentUrlRewritingFilter.java index 943ee32..adf0d19 100644 --- a/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/JsonContentUrlRewritingFilter.java +++ b/rcloud-gist-service/src/main/java/com/mangosolutions/rcloud/gists/filters/JsonContentUrlRewritingFilter.java @@ -24,6 +24,8 @@ public class JsonContentUrlRewritingFilter extends ZuulFilter { private static final Logger logger = LoggerFactory.getLogger(JsonContentUrlRewritingFilter.class); + private static final MimeType[] JSON_MIME_TYPES = {MimeTypeUtils.APPLICATION_JSON, MimeTypeUtils.parseMimeType("application/*+json")}; + private int order = 100; public JsonContentUrlRewritingFilter(int order) { @@ -59,7 +61,12 @@ private boolean isJsonResponseContentType(List> zuulRespons String value = header.second(); try { MimeType mimeType = MimeTypeUtils.parseMimeType(value); - return mimeType.isCompatibleWith(MimeTypeUtils.APPLICATION_JSON); + for(MimeType jsonMimeType: JSON_MIME_TYPES) { + if(jsonMimeType.isCompatibleWith(mimeType)) { + return true; + } + + } } catch (InvalidMimeTypeException e) { logger.warn("Could not parse {} as a valid mimetype", value); } diff --git a/rcloud-gist-service/src/test/java/com/mangosolutions/rcloud/gists/filters/JsonContentUrlRewritingFilterTest.java b/rcloud-gist-service/src/test/java/com/mangosolutions/rcloud/gists/filters/JsonContentUrlRewritingFilterTest.java new file mode 100644 index 0000000..46ada3f --- /dev/null +++ b/rcloud-gist-service/src/test/java/com/mangosolutions/rcloud/gists/filters/JsonContentUrlRewritingFilterTest.java @@ -0,0 +1,99 @@ +package com.mangosolutions.rcloud.gists.filters; + +import java.net.MalformedURLException; +import java.net.URL; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.springframework.http.MediaType; + +import com.netflix.zuul.context.RequestContext; + +public class JsonContentUrlRewritingFilterTest { + + private static final String REMOTE_URL = "https://api.github.com"; + private static final String LOCAL_URL = "http://localhost:8080"; + + @Before + public void setup() throws MalformedURLException { + setup(MediaType.APPLICATION_JSON_UTF8_VALUE); + } + + public void setup(String contentType) throws MalformedURLException { + RequestContext context = RequestContext.getCurrentContext(); + context.getZuulResponseHeaders().clear(); + context.addZuulRequestHeader("x-forwarded-host", "localhost:8080"); + context.addZuulRequestHeader("x-forwarded-proto", "http"); + context.addZuulRequestHeader("x-forwarded-port", "8080"); + context.addZuulResponseHeader("Content-Type", contentType); + context.setRouteHost(new URL("https://api.github.com")); + } + + public void setContent(String content) { + RequestContext.getCurrentContext().setResponseBody(content); + } + + @Test + public void testShouldFilterJsonContent() { + Assert.assertTrue(new JsonContentUrlRewritingFilter(1).shouldFilter()); + } + + @Test + public void testShouldNotFilterXmlContent() throws MalformedURLException { + setup(MediaType.APPLICATION_XML_VALUE); + Assert.assertFalse(new JsonContentUrlRewritingFilter(1).shouldFilter()); + } + + @Test + public void testShouldFilterJsonExtensionContent() throws MalformedURLException { + setup("application/hal+json"); + Assert.assertTrue(new JsonContentUrlRewritingFilter(1).shouldFilter()); + } + + @Test + public void replaceUrlInJsonArray() throws MalformedURLException { + String jsonArray = "[ \"https://api.github.com\" ]"; + setContent(jsonArray); + JsonContentUrlRewritingFilter filter = new JsonContentUrlRewritingFilter(1); + filter.run(); + String responseBody = RequestContext.getCurrentContext().getResponseBody(); + Assert.assertFalse(responseBody.contains(REMOTE_URL)); + Assert.assertTrue(responseBody.contains(LOCAL_URL)); + } + + @Test + public void replaceUrlInJsonObject() { + String jsonArray = "{ \"url\": \"https://api.github.com\" }"; + setContent(jsonArray); + JsonContentUrlRewritingFilter filter = new JsonContentUrlRewritingFilter(1); + filter.run(); + String responseBody = RequestContext.getCurrentContext().getResponseBody(); + Assert.assertFalse(responseBody.contains(REMOTE_URL)); + Assert.assertTrue(responseBody.contains(LOCAL_URL)); + } + + @Test + public void replaceUrlInNestedJsonObject() { + String jsonArray = "{ \"obj\": { \"url\": \"https://api.github.com\" } }"; + setContent(jsonArray); + JsonContentUrlRewritingFilter filter = new JsonContentUrlRewritingFilter(1); + filter.run(); + String responseBody = RequestContext.getCurrentContext().getResponseBody(); + Assert.assertFalse(responseBody.contains(REMOTE_URL)); + Assert.assertTrue(responseBody.contains(LOCAL_URL)); + } + + @Test + public void replaceUrlInSpecificJsonObject() { + String jsonArray = "{ \"obj\": { \"url\": \"https://api.github.com\", \"noUrl\": \"I am not a url\" } }"; + setContent(jsonArray); + JsonContentUrlRewritingFilter filter = new JsonContentUrlRewritingFilter(1); + filter.run(); + String responseBody = RequestContext.getCurrentContext().getResponseBody(); + Assert.assertFalse(responseBody.contains(REMOTE_URL)); + Assert.assertTrue(responseBody.contains(LOCAL_URL)); + Assert.assertTrue(responseBody.contains("I am not a url")); + } + +} From c73d905660dc8595eda468ad61c9f221bc26e586 Mon Sep 17 00:00:00 2001 From: Andrew Dunn Date: Thu, 26 Jan 2017 14:46:56 +0000 Subject: [PATCH 10/12] Updated the build to upload release back to GitHub --- .gitattributes | 3 +++ .travis.yml | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..aea7be0 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +gradlew text eol=lf +*.sh text eol=lf +*.conf text eol=lf diff --git a/.travis.yml b/.travis.yml index e358d25..fc0b7a5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,16 @@ language: java jdk: - - oraclejdk7 +- oraclejdk7 +deploy: + provider: releases + api_key: + secure: HYh0fPSByuuJB/r7YKc+iHRBzo13lScGz1Kign/Uj3GnAcDL+NpnYCvEl1JCymd+v8vPyFDxWmFJ6mJSyAlQzhxIRoeHpOQhECVJXuuB9IUk+o2hb+MhUuPpOwGtlUDW2Rry9qnz73fX5boEBzrYHF3vwbbqdfqc+sugkPcPJubtwlsH49CWHA0kMila/esZyuB7KvXuEmVCy3BCHBPK9KZ9Wo8LlB3pQhUwJWkSJID5BJl/zYDqOrKo4kjXpu6p5xcwAi+HtpCmV3gAMgcHPTO8sXy2b9JKcko0C52/+nyHpQ39gmwS0y+q0lnGZJy38IF52JDzXLAq1DxIXm2hPvI/zMNlhYJ6ovCuT7xTXypKnktwetQJ5dT6hvtNmtBMxkXww43svAb/k9nr2bo9z65iOqKpqdFQYzFB/wltToMbX50xC9Yi+fYv219H3Fc6vioJw5bkRCoapC7ImmkhVjsv5YDEGbqv3dyEbGOWyI0phnJBd6+UjMFCSxX9pinu+RUhLU0uga9/765kiMYsQAH7svzyo5iYcN0HB4X1amAKjaLmp2so2AqKk4FKznVHhtY0Mm74Xgq6tmQTk7iuCwbvf2M1pCjhV4zDBio2XODHGGxJeIZAQTx0CIXz0F20o6K9XC2vDM6BWHt2YCN6qjR0buYP/NdsOPcjeSnCVMk= + file_glob: True + file: + - "./rcloud-gist-service/build/distributions/*.deb" + - "./rcloud-gist-service/build/distributions/*.rpm" + - "./rcloud-gist-service/build/libs/*.jar" + skip_cleanup: true + on: + repo: MangoTheCat/rcloud-gist-services + tags: true From 87431ad805848ebfbeda3234903572ba7324c74f Mon Sep 17 00:00:00 2001 From: Andrew Dunn Date: Thu, 26 Jan 2017 14:51:14 +0000 Subject: [PATCH 11/12] Corrected spelling on the travis property. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fc0b7a5..b32eff0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ deploy: provider: releases api_key: secure: HYh0fPSByuuJB/r7YKc+iHRBzo13lScGz1Kign/Uj3GnAcDL+NpnYCvEl1JCymd+v8vPyFDxWmFJ6mJSyAlQzhxIRoeHpOQhECVJXuuB9IUk+o2hb+MhUuPpOwGtlUDW2Rry9qnz73fX5boEBzrYHF3vwbbqdfqc+sugkPcPJubtwlsH49CWHA0kMila/esZyuB7KvXuEmVCy3BCHBPK9KZ9Wo8LlB3pQhUwJWkSJID5BJl/zYDqOrKo4kjXpu6p5xcwAi+HtpCmV3gAMgcHPTO8sXy2b9JKcko0C52/+nyHpQ39gmwS0y+q0lnGZJy38IF52JDzXLAq1DxIXm2hPvI/zMNlhYJ6ovCuT7xTXypKnktwetQJ5dT6hvtNmtBMxkXww43svAb/k9nr2bo9z65iOqKpqdFQYzFB/wltToMbX50xC9Yi+fYv219H3Fc6vioJw5bkRCoapC7ImmkhVjsv5YDEGbqv3dyEbGOWyI0phnJBd6+UjMFCSxX9pinu+RUhLU0uga9/765kiMYsQAH7svzyo5iYcN0HB4X1amAKjaLmp2so2AqKk4FKznVHhtY0Mm74Xgq6tmQTk7iuCwbvf2M1pCjhV4zDBio2XODHGGxJeIZAQTx0CIXz0F20o6K9XC2vDM6BWHt2YCN6qjR0buYP/NdsOPcjeSnCVMk= - file_glob: True + file_glob: true file: - "./rcloud-gist-service/build/distributions/*.deb" - "./rcloud-gist-service/build/distributions/*.rpm" From 3e3750d52a5f1540dac627786eb1c9c6f8d1b730 Mon Sep 17 00:00:00 2001 From: Andrew Dunn Date: Thu, 26 Jan 2017 16:57:41 +0000 Subject: [PATCH 12/12] Updated the documentation for the release and added on security over the service management port. --- Vagrantfile | 3 +- rcloud-gist-service/README.md | 72 ++++++++++++++++++- rcloud-gist-service/build.gradle | 1 + .../src/main/pkg/home/application.yml | 13 ++++ .../src/main/resources/application.yml | 10 ++- 5 files changed, 96 insertions(+), 3 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index c175ac0..d581ce3 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -27,7 +27,8 @@ Vagrant.configure(2) do |config| # within the machine from a port on the host machine. In the example below, # accessing "localhost:8080" will access port 80 on the guest machine. config.vm.network "forwarded_port", guest: 80, host: 80 - config.vm.network "forwarded_port", guest: 8080, host: 8080 + config.vm.network "forwarded_port", guest: 13010, host: 13110 + config.vm.network "forwarded_port", guest: 13011, host: 13111 #config.ssh.forward_agent = true diff --git a/rcloud-gist-service/README.md b/rcloud-gist-service/README.md index e7f76dc..47c0d4b 100644 --- a/rcloud-gist-service/README.md +++ b/rcloud-gist-service/README.md @@ -1 +1,71 @@ -# Implementation of the RCLoud Gist Service. +# Implementation of the RCloud Gist Service. +A Java based service to provide GitHub gist functionality to RCloud. + +## Building +The code uses the Gradle build system and includes the gradle wrapper in the root +of the project. The only requirement for building is a Java JDK version 7 or greater +and an internet connection, all dependencies will be downloaded for the build. + +To build the software run the following command which runs the tests and generates +the artifacts. + +`./gradlew build` + +### Outputs +The build generates 3 artifacts +1. An executable jar file created here: `./rcloud=gist-service/build/libs` +2. An rpm install file created here: `./rcloud=gist-service/build/distributions` +3. An deb install file created here: `./rcloud=gist-service/build/distributions` + +## Installation + +The redhat and debian install archives will install the application to +`/opt/rcloud-gist-service` and create an entry in the `/etc/init.d/` folder +which can be used to start and stop the service. + +### User and Groups +The installation creates a user and group for the service called `rcloudgistservice` which the service runs as. + +### Debian based systems +To install the service on debian based systems the following command can be used, you will need to use the correct name for the deb file for the version you are installing. + +`sudo dpkg -i ./rcloud-gist-service_0.1.0-20170126123855_all.deb` + +## Default ports +The service uses two ports, one for the gist api and the other for the service management functionality, these can be controlled in configuration. The management port is secured using basic auth. + +## Service Configuration + +Configuration is held within the `/opt/rcloud-gist-service/application.yml` file. +The following parameters are configurable: + +| Property | Description | Default | +|----------|-------------|---------| +| `github.api.url` | The URL to the root of the GitHub installation that this should use | `https://api.github.com` | +| `service.port` | The port that the gist api is accessible over | `13010` | +| `management.port` | The port that the service management api is accessible over | `13011` | +| `security.user.name` | The username that is required for basic auth access to the management port | `admin` | +| `security.user.password` | The username that is required for basic auth access to the management port | If not specified the password is generated at service startup and can be identified in the `/var/log/rcloud-gist-service/rcloud-gist-service-file.log` file. The following command can be used to find the password. `cat /var/log/rcloud-gist-service/rcloud-gist-service-file.log | grep "Using default security"`. More information can be found on the [spring boot documentation.](http://docs.spring.io/spring-boot/docs/current/reference/html/production-ready-monitoring.html) | + +### Java Configuration +The startup parameters for the JVM are stored in the conf file in the installation directory, this must have the same name as the jar file. + + +## RCloud Configuration +To configure RCloud edit the `rcloud.conf` and change the `api.github.url` value to the URL of this service if running with the defaults then this would be `http://localhost:13010` e.g. : + +`github.api.url: http://localhost:13010` + +## Starting and stopping the service + +An System V startup script is installed as part of the installation, thissupports the following commands: + +| Command | Example | +|---------|--------------------------------------| +| start | `service rcloud-gist-service start` | +| stop | `service rcloud-gist-service stop` | +| status | `service rcloud-gist-service status` | + +## Logging +The service uses [Logback](https://logback.qos.ch/), this is controlled by the +configuration file in the installation directory `/opt/rcloud-gist-service/logback.xml`, this configuration file can be updated and the changes will be reloaded to alter the log output. The service writes log files to `/var/log/rcloud-gist-service/` access to this folder is restricted to `root` group and the `rcloudgistservice` user/group. diff --git a/rcloud-gist-service/build.gradle b/rcloud-gist-service/build.gradle index 718b518..eb84b1e 100644 --- a/rcloud-gist-service/build.gradle +++ b/rcloud-gist-service/build.gradle @@ -4,6 +4,7 @@ project.description = "RCloud Gist service providing gist access api." dependencies { compile 'org.springframework.boot:spring-boot-starter' compile 'org.springframework.boot:spring-boot-starter-actuator' + compile 'org.springframework.boot:spring-boot-starter-security' compile 'org.springframework.boot:spring-boot-starter-web' compile 'org.springframework.cloud:spring-cloud-starter' compile 'org.springframework.cloud:spring-cloud-starter-zuul' diff --git a/rcloud-gist-service/src/main/pkg/home/application.yml b/rcloud-gist-service/src/main/pkg/home/application.yml index 54fc32a..7e41ce4 100644 --- a/rcloud-gist-service/src/main/pkg/home/application.yml +++ b/rcloud-gist-service/src/main/pkg/home/application.yml @@ -1,3 +1,9 @@ +server: + port: 13010 + +management: + port: 13011 + logging: config: logback.xml file: /var/log/${spring.application.name}/${spring.application.name}-file.log @@ -5,3 +11,10 @@ logging: github: api: url: https://api.github.com + + + +#security: +# user: +# name: admin +# password: secret diff --git a/rcloud-gist-service/src/main/resources/application.yml b/rcloud-gist-service/src/main/resources/application.yml index 4dce046..841e22a 100644 --- a/rcloud-gist-service/src/main/resources/application.yml +++ b/rcloud-gist-service/src/main/resources/application.yml @@ -2,7 +2,9 @@ server: port: 13010 management: - port: 13011 + port: 13011 + security: + enabled: true info: version: 1.0.0 @@ -15,6 +17,12 @@ spring: config: enabled: false +security: + user: + name: admin + basic: + enabled: false + zuul: routes: