chore: add release and continuous/post kokoro jobs (googleapis#836)

Author: Graham Paye

.kokoro/continuous/linux.cfg

@@ -3,10 +3,10 @@
 build_file: "google-api-ruby-client/.kokoro/trampoline.sh"
 
 # Configure the docker image for kokoro-trampoline.
-# Dockerfile is maintained at https://github.com/googleapis/google-cloud-ruby/tree/master/.kokoro/docker/ruby-multi
+# Dockerfile is maintained at https://github.com/googleapis/google-cloud-ruby/tree/master/.kokoro/docker/multi
 env_vars: {
     key: "TRAMPOLINE_IMAGE"
-    value: "gcr.io/cloud-devrel-kokoro-resources/yoshi-ruby/ruby-multi"
+    value: "gcr.io/cloud-devrel-kokoro-resources/yoshi-ruby/multi"
 }
 
 env_vars: {

.kokoro/presubmit/linux.cfg

@@ -5,7 +5,7 @@ build_file: "google-api-ruby-client/.kokoro/trampoline.sh"
 # Configure the docker image for kokoro-trampoline.
 env_vars: {
     key: "TRAMPOLINE_IMAGE"
-    value: "gcr.io/cloud-devrel-kokoro-resources/yoshi-ruby/ruby-multi"
+    value: "gcr.io/cloud-devrel-kokoro-resources/yoshi-ruby/multi"
 }
 
 env_vars: {

.kokoro/release.cfg

@@ -0,0 +1,94 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Build logs will be here
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+  }
+}
+
+# Fetch the token needed for reporting release status to GitHub
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 73713
+      keyname: "yoshi-automation-github-key"
+    }
+  }
+}
+
+# Fetch magictoken to use with Magic Github Proxy 
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 73713
+      keyname: "releasetool-magictoken"
+      backend_type: FASTCONFIGPUSH
+    }
+  }
+}
+
+# Fetch api key to use with Magic Github Proxy 
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 73713
+      keyname: "magic-github-proxy-api-key"
+      backend_type: FASTCONFIGPUSH
+    }
+  }
+}
+
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 73713
+      keyname: "docuploader_service_account"
+    }
+  }
+}
+
+# Download resources for system tests (service account key, etc.)
+gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-ruby"
+
+# Download trampoline resources.
+gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
+
+# Use the trampoline script to run in docker.
+build_file: "google-api-ruby-client/.kokoro/trampoline.sh"
+
+# Configure the docker image for kokoro-trampoline.
+env_vars: {
+    key: "TRAMPOLINE_IMAGE"
+    value: "gcr.io/cloud-devrel-kokoro-resources/yoshi-ruby/release"
+}
+
+env_vars: {
+    key: "TRAMPOLINE_BUILD_FILE"
+    value: "github/google-api-ruby-client/.kokoro/build.sh"
+}
+
+env_vars: {
+    key: "TRAMPOLINE_SCRIPT"
+    value: "trampoline_v1.py"
+}
+
+env_vars: {
+    key: "JOB_TYPE"
+    value: "release"
+}
+
+env_vars: {
+    key: "OS"
+    value: "linux"
+}
+
+env_vars: {
+    key: "REPO_DIR"
+    value: "github/google-api-ruby-client"
+}
+
+env_vars: {
+    key: "PACKAGE"
+    value: "google-api-client"
+}

.repo-metadata.json

@@ -0,0 +1,5 @@
+{
+    "name": "google-api-client",
+    "language": "ruby",
+    "distribution-name": "google-api-client"
+}

.rubocop.yml

@@ -5,3 +5,8 @@ Metrics/LineLength:
 
 Style/FormatString:
   EnforcedStyle: sprintf
+
+AllCops:
+  Exclude:
+    - "Rakefile"
+    - "rakelib/**/*"

Gemfile

@@ -22,6 +22,7 @@ group :development do
   gem 'redis', '~> 3.2'
   gem 'logging', '~> 2.2'
   gem 'opencensus', '~> 0.4'
+  gem 'httparty'
 end
 
 platforms :jruby do

Rakefile

@@ -1,4 +1,51 @@
 require "bundler/gem_tasks"
+require "json"
+
+task :release_gem, :tag do |_t, args|
+  tag = args[:tag]
+  raise "You must provide a tag to release." if tag.nil?
+
+  # Verify the tag format "vVERSION"
+  m = tag.match(/v(?<version>\S*)/)
+  raise "Tag #{tag} does not match the expected format." if m.nil?
+
+  version = m[:version]
+  raise "You must provide a version." if version.nil?
+
+  api_token = ENV["RUBYGEMS_API_TOKEN"]
+
+  require "gems"
+  if api_token
+    ::Gems.configure do |config|
+      config.key = api_token
+    end
+  end
+
+  Bundler.with_clean_env do
+    sh "rm -rf pkg"
+    sh "bundle update"
+    sh "bundle exec rake build"
+  end
+
+  path_to_be_pushed = "pkg/google-api-client-#{version}.gem"
+  gem_was_published = nil
+  if File.file? path_to_be_pushed
+    begin
+      response = ::Gems.push File.new(path_to_be_pushed)
+      puts response
+      raise unless response.include? "Successfully registered gem:"
+      gem_was_published = true
+      puts "Successfully built and pushed google-api-client for version #{version}"
+    rescue StandardError => e
+      gem_was_published = false
+      puts "Error while releasing google-api-client version #{version}: #{e.message}"
+    end
+  else
+    raise "Cannot build google-api-client for version #{version}"
+  end
+
+  Rake::Task["kokoro:publish_docs"].invoke if gem_was_published
+end
 
 task default: :spec
 
@@ -22,13 +69,35 @@ namespace :kokoro do
   task :nightly do
     Rake::Task["spec"].invoke
   end
+
+  task :post do
+    require_relative "rakelib/link_checker.rb"
+
+    link_checker = LinkChecker.new
+    link_checker.run
+    exit link_checker.exit_status
+  end
+
+  task :release do
+    # Until code generation process is updated and release-please is set up, just publish docs
+    require_relative "rakelib/devsite/devsite_builder.rb"
+
+    DevsiteBuilder.new.publish_if_missing ENV["DOCS_BUILD_TAG"]
+  end
+
+  desc "Publish docs for the latest git tag"
+  task :publish_docs do
+    require_relative "rakelib/devsite/devsite_builder.rb"
+
+    DevsiteBuilder.new.publish ENV["DOCS_BUILD_TAG"]
+  end
 end
 
 def header str, token = "#"
   line_length = str.length + 8
-  puts ""
+  puts
   puts token * line_length
   puts "#{token * 3} #{str} #{token * 3}"
   puts token * line_length
-  puts ""
+  puts
 end

rakelib/devsite/devsite_builder.rb

@@ -0,0 +1,126 @@
+require "pathname"
+require "tmpdir"
+
+require_relative "repo_metadata.rb"
+
+class DevsiteBuilder
+  def initialize build_tag = nil
+    @build_tag = build_tag || latest_tag
+    @output_dir = "doc"
+  end
+
+  def build tag
+    checkout_tag tag
+    doc_path = tmp_dir + @output_dir
+    FileUtils.remove_dir doc_path if Dir.exist? doc_path
+    markup = "--markup markdown --markup-provider redcarpet"
+
+    Dir.chdir tmp_dir do
+      cmds = ["-o #{@output_dir}", markup]
+      cmd "yard --verbose #{cmds.join ' '}"
+    end
+    metadata.build doc_path
+  end
+
+  def upload
+    Dir.chdir tmp_dir + @output_dir do
+      opts = [
+        "--credentials=#{ENV['KOKORO_KEYSTORE_DIR']}/73713_docuploader_service_account",
+        "--staging-bucket=#{ENV.fetch 'STAGING_BUCKET', 'docs-staging'}",
+        "--metadata-file=./docs.metadata"
+      ]
+      cmd "python3 -m docuploader upload . #{opts.join ' '}"
+    end
+  end
+
+  def publish tag = nil
+    build(tag || @build_tag)
+    upload
+  end
+
+  def publish_if_missing tag = nil
+    tag ||= @build_tag
+    puts tag
+    puts missing? tag
+    publish tag if missing? tag
+  end
+
+  def missing? tag
+    require "httparty"
+
+    url = "https://googleapis.dev/ruby/google-api-client/v#{version tag}/index.html"
+    response = HTTParty.get url
+    response.code != 200
+  rescue StandardError
+    true
+  end
+
+  def cmd line
+    puts line
+    output = `#{line}`
+    puts output
+    output
+  end
+
+  def metadata
+    return @metadata if @metadata
+
+    metadata_json = "#{tmp_dir}/.repo-metadata.json"
+    @metadata = RepoMetadata.from_source metadata_json if File.file? metadata_json
+    @metadata ||= RepoMetadata.from_source "name"              => "google-api-client",
+                                           "distribution-name" => "google-api-client",
+                                           "language"          => "ruby"
+    @metadata["version"] = version @build_tag
+    @metadata
+  end
+
+  def checkout_tag git_tag
+    Dir.chdir tmp_dir do
+      `git checkout tags/#{git_tag} -b v#{version git_tag}`
+    end
+  end
+
+  def version git_tag
+    m = git_tag.match(/(\d+\.\d+\.\d+)/)
+    return m if m.nil?
+    m[0]
+  end
+
+  def versions
+    Dir.chdir tmp_dir do
+      tags.map { |t| version t }.reject(&:nil?).sort_by { |v| Gem::Version.new v }.reverse
+    end
+  end
+
+  def tags
+    Dir.chdir tmp_dir do
+      `git tag`.split "\n"
+    end
+  end
+
+  def latest_version
+    @latest_version ||= versions.first
+  end
+
+  def latest_tag
+    @latest_tag ||= tags.select { |t| t.include? latest_version }.min_by(&:size)
+  end
+
+  def tmp_dir
+    return @tmp_dir if @tmp_dir
+
+    tmp = Dir.tmpdir
+    dir_name = "google-api-ruby-client"
+    @tmp_dir = Pathname.new(tmp) + dir_name
+    FileUtils.remove_dir @tmp_dir if Dir.exist? @tmp_dir
+
+    Dir.chdir tmp do
+      `git clone https://github.com/googleapis/google-api-ruby-client.git`
+    end
+    Dir.chdir @tmp_dir do
+      `git fetch`
+    end
+
+    @tmp_dir
+  end
+end

rakelib/devsite/link_checker.rb

@@ -0,0 +1,64 @@
+require "open3"
+
+class LinkChecker
+  def initialize
+    @failed = false
+  end
+
+  def run
+    job_info
+    git_commit = ENV.fetch "KOKORO_GITHUB_COMMIT", "master"
+
+    markdown_files = Dir.glob "**/*.md"
+    broken_markdown_links = check_links(markdown_files,
+                                        "https://github.com/googleapis/google-api-ruby-client/tree/#{git_commit}",
+                                        " --skip '^(?!(\\Wruby.*google|.*google.*\\Wruby|.*cloud\\.google\\.com))'")
+
+    broken_devsite_links = check_links(["google-api-client"],
+                                       "https://googleapis.dev/ruby",
+                                       "/latest/ --recurse --skip https:.*github.*")
+
+    puts_broken_links broken_markdown_links
+    puts_broken_links broken_devsite_links
+  end
+
+  def check_links location_list, base, tail
+    broken_links = Hash.new { |h, k| h[k] = [] }
+    location_list.each do |location|
+      out, err, st = Open3.capture3 "npx linkinator #{base}/#{location}#{tail}"
+      puts out
+      unless st.to_i.zero?
+        @failed = true
+        puts err
+      end
+      checked_links = out.split "\n"
+      checked_links.select! { |link| link =~ /\[\d+\]/ && !link.include?("[200]") }
+      unless checked_links.empty?
+        @failed = true
+        broken_links[location] += checked_links
+      end
+    end
+    broken_links
+  end
+
+  def puts_broken_links link_hash
+    link_hash.each do |location, links|
+      puts "#{location} contains the following broken links:"
+      links.each { |link| puts "  #{link}" }
+      puts ""
+    end
+  end
+
+  def job_info
+    line_length = "Using Ruby - #{RUBY_VERSION}".length + 8
+    puts
+    puts "#" * line_length
+    puts "### Using Ruby - #{RUBY_VERSION} ###"
+    puts "#" * line_length
+    puts
+  end
+
+  def exit_status
+    @failed ? 1 : 0
+  end
+end

rakelib/devsite/repo_metadata.rb

@@ -0,0 +1,56 @@
+require "json"
+
+class RepoMetadata
+  attr_accessor :data
+
+  def initialize data
+    @data = data
+    normalize_data!
+  end
+
+  def allowed_fields
+    [
+      "name", "version", "language", "distribution-name",
+      "product-page", "github-repository", "issue-tracker"
+    ]
+  end
+
+  def build output_directory
+    fields = @data.to_a.map { |kv| "--#{kv[0]} #{kv[1]}" }
+    Dir.chdir output_directory do
+      cmd "python3 -m docuploader create-metadata #{fields.join ' '}"
+    end
+  end
+
+  def normalize_data!
+    @data.delete_if { |k, _| !allowed_fields.include?(k) }
+  end
+
+  def [] key
+    data[key]
+  end
+
+  def []= key, value
+    @data[key] = value
+  end
+
+  def cmd line
+    puts line
+    output = `#{line}`
+    puts output
+    output
+  end
+
+  def self.from_source source
+    if source.is_a? RepoMetadata
+      data = source.data
+    elsif source.is_a? Hash
+      data = source
+    elsif File.file? source
+      data = JSON.parse File.read(source)
+    else
+      raise "Source must be a path, hash, or RepoMetadata instance"
+    end
+    RepoMetadata.new data
+  end
+end