pull-dependencies

#!/usr/bin/env ruby

# SEMPRE depends on several library/data files into |lib|.  Run this script to
# copy those dependencies to your local directory.  This allows you to run
# SEMPRE from anywhere.  This file consists of a set of modules (which loosely
# correspond to the code modules).
#
# The master copy of these dependencies are stored on the Stanford NLP machines.
#
# Usage:
#   ./pull-dependencies <module-1> ... <module-n>
#
# For developers with ssh access to NLP machines, there are two more local commands:
# - Copy or link |sourcePath| into lib/|dir|.
#   ./pull-dependencies -l <module-1> ... <module-n>
# - Deploy the dependencies to the NLP machines's public www.
#   ./pull-dependencies -l -r <module-1> ... <module-n>

# Specify the version of the dependencies
# (To developer: Update this before releasing a new version!)
$version = '2.0'

$isLocal = ARGV.index('-l')
$isRelease = ARGV.index('-r')
if $isRelease and not $isLocal
  puts "ERROR: To release, must use both -l and -r"
  exit 1
end
ARGV.delete_if { |x| x == '-l' or x == '-r' }

def isZip(name)
  # Directories are zipped
  name.end_with?('.exec') or name !~ /\./
end

def pull(sourcePath, dir=nil, opts={})
  puts sourcePath
  destDir = 'lib' + (dir ? '/' + dir : '')
  system "mkdir -p #{destDir}"

  name = File.basename(sourcePath)
  ext = isZip(name) ? '.zip' : ''

  if not $isLocal and not $isRelease
    # Download url => localPath
    if sourcePath.start_with?('http://') || sourcePath.start_with?('https://')
      url = sourcePath
    else
      url = 'http://nlp.stanford.edu/software/sempre/dependencies-' + $version + sourcePath + ext
    end
    localPath = destDir + '/' + name + ext
    system "mkdir -p #{File.dirname(localPath)}" or exit 1
    system "wget -c '#{url}' -O #{localPath}" or exit 1
    # Unzip localPath to destDir if it's a zip file
    if isZip(name)
      system "cd #{File.dirname(localPath)} && unzip #{File.basename(localPath)}" or exit 1
      system "rm #{localPath}" or exit 1
    end
  else
    rsyncOpts = '-rlptDzi'  # Preserve everything except groups and permissions
    if $isRelease
      # Copy sourcePath to cluster
      baseDeployPath = '/u/apache/htdocs/static/software/sempre/dependencies-' + $version
      deployPath = baseDeployPath + sourcePath + ext
      system "mkdir -p #{File.dirname(deployPath)}" or exit 1
      if File.exists?(sourcePath)
        if isZip(name)
          system "cd #{File.dirname(sourcePath)} && zip -r #{deployPath} #{File.basename(sourcePath)}" or exit 1
        else
          if opts[:symlink]
            system "ln -sf #{File.expand_path(sourcePath)} #{deployPath}" or exit 1
          else
            system "rsync #{rsyncOpts} #{sourcePath} #{deployPath}" or exit 1
          end
        end
      else
        system "rsync #{rsyncOpts} jamie.stanford.edu:#{sourcePath} #{deployPath}" or exit 1
      end
      system "chmod -R og=u #{baseDeployPath}" #or exit 1
    else
      # Download sourcePath from cluster to destDir
      if File.exists?(sourcePath)
        if opts[:symlink]
          system "ln -sf #{File.expand_path(sourcePath)} #{destDir}" or exit 1
        else
          system "rsync #{rsyncOpts} #{sourcePath} #{destDir}" or exit 1
        end
      else
        system "rsync #{rsyncOpts} jamie.stanford.edu:#{sourcePath} #{destDir}" or exit 1
      end
    end
  end
end

# source: path to master git repository
def updateGit(source)
  dir = File.basename(source.sub(/\.git$/, ''))
  if File.exists?(dir)
    system 'cd '+dir+' && git pull' or exit 1
  else
    system 'git clone ' + source or exit 1
  end
end

def downloadExec(path)
  ['options.map', 'params', 'grammar'].each { |file|
    if File.exists?(path+'/'+file)
      pull(path+'/'+file, 'models/'+File.basename(path))
    end
  }
end

$modules = []
def addModule(name, description, func)
  $modules << [name, description, func]
end

############################################################

addModule('core', 'Core utilities (need to compile)', lambda {
  # fig: options parsing, experiment management, utils
  updateGit('https://github.com/percyliang/fig')
  system 'make -C fig' or exit 1
  system 'mkdir -p lib && cd lib && ln -sf ../fig/fig.jar' or exit 1

  # Google libraries
  pull('/u/nlp/data/semparse/resources/guava-14.0.1.jar')

  # TestNG -- testing framework
  pull('/u/nlp/data/semparse/resources/testng-6.8.5.jar')
  pull('/u/nlp/data/semparse/resources/jcommander-1.30.jar')

  # Checkstyle: make sure code looks fine
  pull('/u/nlp/data/semparse/resources/checkstyle')

  # JSON
  pull('/u/nlp/data/semparse/resources/jackson-core-2.2.0.jar')
  pull('/u/nlp/data/semparse/resources/jackson-annotations-2.2.0.jar')
  pull('/u/nlp/data/semparse/resources/jackson-databind-2.2.0.jar')

  # jLine from maven central
  pull('http://central.maven.org/maven2/jline/jline/2.14.2/jline-2.14.2.jar')
})

addModule('corenlp', 'Stanford CoreNLP 3.6.0', lambda {
  pull('/u/nlp/data/semparse/resources/stanford-corenlp-full-2015-12-09.zip', '', {:symlink => true})
  if not File.exists?('lib/stanford-corenlp-full-2015-12-09')
    system "cd lib && unzip stanford-corenlp-full-2015-12-09.zip" or exit 1
  end
  pull('/u/nlp/data/semparse/resources/stanford-corenlp-caseless-2015-04-20-models.jar',
       'stanford-corenlp-full-2015-12-09', {:symlink => true})
  # Remove old file (for backward compatibility)
  if Dir.glob('lib/stanford-corenlp*.jar').any?
    system 'rm -v lib/stanford-corenlp*.jar' or exit 1
  end
  {'stanford-corenlp-3.6.0.jar' => 'stanford-corenlp.jar',
   'stanford-corenlp-3.6.0-models.jar' => 'stanford-corenlp-models.jar',
   'stanford-corenlp-caseless-2015-04-20-models.jar' => 'stanford-corenlp-caseless-models.jar',
   'joda-time.jar' => 'joda-time.jar',
   'jollyday.jar' => 'jollyday.jar',
   'ejml-0.23.jar' => 'ejml.jar',
   'slf4j-api.jar' => 'slf4j-api.jar',
   'slf4j-simple.jar' => 'slf4j-simple.jar',
  }.each { |key, value|
    system "ln -sfv stanford-corenlp-full-2015-12-09/#{key} lib/#{value}" or exit 1
  }
})

addModule('corenlp-3.2.0', 'Stanford CoreNLP 3.2.0 (for backward reproducibility)', lambda {
  pull('/u/nlp/data/semparse/resources/stanford-corenlp-full-2013-06-20.zip', '', {:symlink => true})
  if not File.exists?('lib/stanford-corenlp-full-2013-06-20')
    system "cd lib && unzip stanford-corenlp-full-2013-06-20.zip" or exit 1
  end
  pull('/u/nlp/data/semparse/resources/stanford-corenlp-caseless-2013-06-07-models.jar',
       'stanford-corenlp-full-2013-06-20', {:symlink => true})
  # Remove old file (for backward compatibility)
  if Dir.glob('lib/stanford-corenlp*.jar').any?
    system 'rm -v lib/stanford-corenlp*.jar' or exit 1
  end
  {'stanford-corenlp-3.2.0.jar' => 'stanford-corenlp.jar',
   'stanford-corenlp-3.2.0-models.jar' => 'stanford-corenlp-models.jar',
   'stanford-corenlp-caseless-2013-06-07-models.jar' => 'stanford-corenlp-caseless-models.jar',
   'joda-time.jar' => 'joda-time.jar',
   'jollyday.jar' => 'jollyday.jar'
  }.each { |key, value|
    system "ln -sfv stanford-corenlp-full-2013-06-20/#{key} lib/#{value}" or exit 1
  }
})

addModule('virtuoso', 'Virtuoso: if want to run own SPARQL server locally', lambda {
  updateGit('https://github.com/openlink/virtuoso-opensource')
  # Run this command to compile:
  #system "cd virtuoso-opensource && ./autogen.sh && ./configure --prefix=$PWD/install && make && make install" or exit 1
})

addModule('esslli_2016', 'Data for ESSLLI 2016 semantic parsing class', lambda {
  pull('/u/nlp/data/semparse/esslli_2016', 'data/esslli_2016/', {:symlink => true})
})

############################################################

if ARGV.size == 0
  puts "#{$0} <module-1> ... <module-n>"
  puts
  puts "Modules:"
  $modules.each { |name,description,func|
    puts "  #{name}: #{description}"
  }
  puts
  puts "Internal use (Stanford NLP only):"
  puts "  #{$0} -l <module-1> ...: Get the files from the local Stanford NLP server instead"
  puts "  #{$0} -l -r <module-1> ...: Release to the public www directory on the server"
end

$modules.each { |name,description,func|
  if ARGV.index(name)
    puts "===== Downloading #{name}: #{description}"
    func.call
  end
}