wip - new plugin api

README.md

@@ -1,5 +1,4 @@
-Strider-simple-runner
-=====================
+## Strider-simple-runner
 
 [![Build Status](https://hosted.stridercd.com/51f050cf04568a0c00000008/strider-cd/strider-simple-worker/badge)](https://hosted.stridercd.com/strider-cd/strider-simple-worker/)
 
@@ -8,6 +7,119 @@ Strider-simple-runner
 Easy-to-configure in-process runner implementation for Strider Continous
 Deployment. This runner comes bundled with Strider.
 
+## Spec
+
+`require('strider-simple-runner').create(emitter, config, done)`
+
+### Config
+
+All options are optional.
+
+```
+pty(false):              use 'pty' for running commands. Currently disabled
+logger(console):         .log, .warn, .error
+io(new EventEmitter):    the channel of internal communication with the job worker
+processJob(core.process):function to run a job. (task, config, ondone)
+pluginDirs:              the directories in which to look for plugins
+dataDir($HOME/.strider): the directory in which to clone/test/etc
+```
+
+### Events
+
+#### Expected to consume
+
+- 'job.new'
+```
+{
+  id: Oid,
+  type: 'TEST_ONLY' | 'TEST_AND_DEPLOY',
+  user_id: Oid,
+  trigger: {
+  },
+  ref: {
+    branch: String,
+    id: String // commit id
+  },
+  // this stuff is not part of the "job" document in mongo, but is added
+  project: {
+    // project config straight from db, includes plugin config and
+    // project level provider config
+  },
+  userProvider: { // user-level config for the provider. Things like a github
+  }               // OAuth token. Retrieved from user.providers.[providerid]
+}
+```
+
+Ex: github provider config
+```js
+{
+  id: 'github',
+  user: {
+    token: '1234',
+    username: 'hacker'
+  },
+  project: {
+    url: 'example.com/repo.git',
+    display_url: 'http://example.com/repo',
+    auth: {
+      method: 'https',
+      username: null, // use user's gh auth
+      password: null // use user's gh token
+    }
+  }
+}
+```
+
+Ex: git provider config
+```js
+{
+  id: 'git',
+  user: {},
+  project: {
+    url: 'example.com/repo.git',
+    method: 'ssh',
+    privkey: null, // use repo-level ssh keys
+    pubkey: null
+  }
+}
+```
+
+Project config looks like:
+
+```js
+{
+  name: 'owner/name',
+  public: false,
+  active: true,
+  deploy_on_green: true,
+  secret: // what's this for?
+  runner: {
+    id: 'simple', // or docker, etc.
+    pty: false
+    // other config for the runner
+  },
+  privkey: '',
+  pubkey: '',
+  provider: {}, // provider config
+  // owner is implicit, as it's embedded ...
+  collaborators: [],
+  plugins: [{
+    id: 'heroku',
+    // plugin config
+  }, ...]
+}
+```
+
+Plugins needed:
+
+- heroku
+- webhooks
+
+Providers:
+
+- git
+- github
+
 Tests
 =====
 

index.js

@@ -1,19 +1,12 @@
 
 var Runner = require('./lib')
+  , Loader = require('strider-extension-loader')
 
 var create = function(emitter, opts, cb){
-  console.log(">>>>> RUNNER:: CREATE", arguments)
+  console.log(">>>>> RUNNER:: CREATED")
+  var loader = new Loader()
   var runner = new Runner(emitter, opts)
-
-  // TEMP - assume all the extensions are loaded. This
-  // will be replaced when we pass the extension config
-  // through in the run events.
-
-  var pth = require.main.paths
-  loader.initWorkerExtensions(pth, runner.buildContext({}, pth), function(){
-    console.log("!!!!! RUNNER CREATE>", arguments)
-    cb(null)
-  })
+  runner.loadExtensions(require.main.paths, cb)
 }
 
 module.exports = {
@@ -25,9 +18,14 @@ module.exports = {
     // --> 'job.new' { repo: {...}, job: {...} }
     // --> 'job.cancel' jobid
     // Events it is expected to emit
-    // --> 'browser.update' eventname, data
-    // --> 'job.queued' jobid, time
-    // --> 'job.done' { id: jobid, commands: [...], times: {...}, test_status: int, deploy_status: int }
+    // <-- 'browser.update' eventname, data
+    // <-- 'job.queued' jobid, time
+    // <-- 'job.done' { id: jobid, commands: [...], times: {...}, test_status: int, deploy_status: int }
+
+    // Backwards compat: TODO remove
+    // --> queue.new_job
+    // <-- queue.job_update
+    // <-- queue.job_complete
 
   // We expose these for other modules / unit testing
   Runner : Runner

lib/consts.js

@@ -6,13 +6,19 @@ var phases = ['env', 'prepare', 'test', 'deploy', 'cleanup']
 var skels = {
   job: {
     id: null,
+    data: null,
     phases: {},
     phase: phases[0],
     queued: null,
     started: null,
     finished: null,
     test_status: -1,
-    deploy_status: -1
+    deploy_status: -1,
+    std: {
+      out: '',
+      err: '',
+      merged: ''
+    }
   },
   command: {
     out: '',

lib/index.js

@@ -3,6 +3,7 @@ var _ = require('lodash')
   , path = require('path')
   , async = require('async')
   , EventEmitter2 = require('eventemitter2').EventEmitter2
+  , Loader = require('strider-extension-loader')
 
   , core = require('strider-runner-core')
 
@@ -34,10 +35,26 @@ function Runner(emitter, config) {
   this.io = this.config.io
   this.hooks = []
   this.jobdata = new JobData(this.io)
+  this.attach()
 }
 
 Runner.prototype = {
   // public API
+  loadExtensions: function (dirs, done) {
+    var self = this
+      , loader = new Loader()
+    loader.collectExtensions(require.main.paths, function (err) {
+      if (err) return done(err)
+      loader.initWorkerExtensions(self.getContext(), function (err, extensions) {
+        if (err) return done(err)
+        self.extensions = extensions
+        done(null)
+      })
+    })
+  },
+  getContext: function () {
+    return {}
+  },
 
   // private API
   attach: function () {
@@ -53,18 +70,54 @@ Runner.prototype = {
       // our running job was cancelled, so we need to start the next one
       async.setImmediate(self.queue.process)
     })
+    // proxy up plugin events
+    this.io.on('plugin.*', function () {
+      self.emitter.emit.apply(self.emitter, [this.event].concat([].slice.call(arguments)))
+    })
+
+    // Backwards compat: TODO remove
+    this.io.on('job.status.std*', function (id, text) {
+      var which = this.event.split('.')[2]
+        , data = {
+            jobId: id,
+            stdout: '',
+            stderr: ''
+          }
+        , job = self.jobdata.get(id)
+      data.userId = job.data.user_id
+      // # seconds since it started
+      data.timeElapsed = (new Date().getTime() - job.started.getTime())/1000
+      data[which] = text
+      data.repoUrl = job.provider.project.display_url
+      self.emitter.emit('queue.job_update', data)
+    })
+    this.io.on('job.done', function (job) {
+      self.emitter.emit('queue.job_complete', {
+        stdout: job.std.out,
+        stderr: job.std.err,
+        stdmerged: job.std.merged,
+        testExitCode: job.test_status,
+        deployExitCode: job.deploy_status,
+        tasks: [] // what are these??
+      });
+    });
+    /*
+    this.emitter.on('queue.new_job', function (old, more) {
+      this.emitter.emit('job.new', oldJobToNew(old, more));
+    });
+    */
   },
 
-  queueJob: function (task) {
+  queueJob: function (job) {
     var now = new Date()
-    this.jobdata.add(task.job.id)
+    this.jobdata.add(job.id)
     this.log('Got new job')
-    this.emitter.emit('browser.update', 'job.status.queued', {id: task.job.id, time: now})
-    this.queue.push(task)
+    this.emitter.emit('browser.update', 'job.status.queued', {id: job.id, time: now})
+    this.queue.push(job)
   },
   cancelJob: function (id) {
     for (var i=0; i<this.queue.tasks.length; i++) {
-      if (this.queue.tasks[i].data.job.id === id) {
+      if (this.queue.tasks[i].data.id === id) {
         this.queue.tasks.splice(i, 1)
         this.emitter.emit('job.cancelled', id, this.jobdata.pop(id))
         return
@@ -73,21 +126,21 @@ Runner.prototype = {
     // if it wasn't found in the queue, fire to the worker in case it is in process
     this.io.emit('job.cancel', id)
   },
-  processJob: function (task, next) {
+  processJob: function (job, next) {
     var self = this
       , now = new Date()
-    this.jobdata.get(task.job.id).started = now
-    this.config.processJob(task, this.config, function (err) {
-      var job = self.jobdata.pop(task.job.id)
+    this.jobdata.get(job.id).started = now
+    this.config.processJob(job, this.config, function (err) {
+      var jobdata = self.jobdata.pop(job.id)
       if (err) {
-        job.errored = true
-        job.error = {
+        jobdata.errored = true
+        jobdata.error = {
           message: err.message,
           stack: err.stack
         }
-        self.emitter.emit('browser.update', 'job.status.errored', [task.job.id])
+        self.emitter.emit('browser.update', 'job.status.errored', [job.id])
       }
-      self.emitter.emit('job.done', job)
+      self.emitter.emit('job.done', jobdata)
       next(null)
     });
   }

lib/jobdata.js

@@ -16,9 +16,10 @@ function JobData(io) {
 
 JobData.prototype = {
   // public api
-  add: function (id) {
-    var job = this.jobs[id] = _.cloneDeep(consts.skels.job)
-    job.id = id
+  add: function (data) {
+    var job = this.jobs[data.id] = _.cloneDeep(consts.skels.job)
+    job.id = data.id
+    job.data = data
     job.queued = new Date()
     for (var i=0; i<consts.phases.length; i++) {
       job.phases[consts.phases[i]] = _.cloneDeep(consts.skels.phase)
@@ -77,11 +78,15 @@ JobData.prototype = {
       var command = utils.ensureCommand(this.phases[this.phase])
       command.out += text
       command.merged += text
+      this.std.out += text
+      this.std.merged += text
     },
     'stderr': function (text) {
       var command = utils.ensureCommand(this.phases[this.phase])
       command.err += text
       command.merged += text
+      this.std.err += text
+      this.std.merged += text
     },
   }
 }

package.json

@@ -27,7 +27,9 @@
     "test": "./node_modules/mocha/bin/mocha -R tap test"
   },
   "strider": {
-    "runner": "index.js"
+    "id": "simple-runner",
+    "type": "runner",
+    "webapp": "index.js"
   },
   "author": "Niall O'Higgins <[email protected]> (http://frozenridge.co)",
   "contributors": [