diff --git a/.github/workflows/rspec.yml b/.github/workflows/rspec.yml new file mode 100644 index 0000000..4efdd11 --- /dev/null +++ b/.github/workflows/rspec.yml @@ -0,0 +1,32 @@ +name: Ruby + +on: + push: + branches: [main] + pull_request: + +jobs: + rspec: + runs-on: ubuntu-latest + env: + BUNDLE_JOBS: 4 + BUNDLE_RETRY: 3 + strategy: + fail-fast: false + matrix: + ruby: ["3.0", "3.1", "3.2"] + steps: + - uses: actions/checkout@v4 + - uses: actions/cache@v3 + with: + path: /home/runner/bundle + key: bundle-${{ matrix.ruby }}-${{ hashFiles('**/*.gemspec') }} + restore-keys: | + bundle-${{ matrix.ruby }}- + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby }} + bundler-cache: true + - name: Run rspec + run: bundle exec rspec diff --git a/.rspec b/.rspec new file mode 100644 index 0000000..c99d2e7 --- /dev/null +++ b/.rspec @@ -0,0 +1 @@ +--require spec_helper diff --git a/Gemfile.lock b/Gemfile.lock index 72ea205..aa4a19f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,9 @@ PATH remote: . specs: - sg_fargate_rails (0.1.5) + sg_fargate_rails (0.1.8) + aws-sdk-ec2 (~> 1.413) + aws-sdk-scheduler (~> 1.10) lograge (~> 0.12) puma rack-attack (~> 6.6) @@ -27,12 +29,29 @@ GEM i18n (>= 1.6, < 2) minitest (>= 5.1) tzinfo (~> 2.0) + aws-eventstream (1.2.0) + aws-partitions (1.835.0) + aws-sdk-core (3.185.1) + aws-eventstream (~> 1, >= 1.0.2) + aws-partitions (~> 1, >= 1.651.0) + aws-sigv4 (~> 1.5) + jmespath (~> 1, >= 1.6.1) + aws-sdk-ec2 (1.413.0) + aws-sdk-core (~> 3, >= 3.184.0) + aws-sigv4 (~> 1.1) + aws-sdk-scheduler (1.10.0) + aws-sdk-core (~> 3, >= 3.184.0) + aws-sigv4 (~> 1.1) + aws-sigv4 (1.6.0) + aws-eventstream (~> 1, >= 1.0.2) builder (3.2.4) concurrent-ruby (1.2.0) crass (1.0.6) + diff-lcs (1.5.0) erubi (1.12.0) i18n (1.12.0) concurrent-ruby (~> 1.0) + jmespath (1.6.2) lograge (0.12.0) actionpack (>= 4) activesupport (>= 4) @@ -71,6 +90,19 @@ GEM rake (13.0.6) request_store (1.5.1) rack (>= 1.4) + rspec (3.12.0) + rspec-core (~> 3.12.0) + rspec-expectations (~> 3.12.0) + rspec-mocks (~> 3.12.0) + rspec-core (3.12.2) + rspec-support (~> 3.12.0) + rspec-expectations (3.12.3) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-mocks (3.12.6) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-support (3.12.1) thor (1.2.1) tzinfo (2.0.6) concurrent-ruby (~> 1.0) @@ -82,6 +114,7 @@ PLATFORMS DEPENDENCIES rake (~> 13.0) + rspec sg_fargate_rails! BUNDLED WITH diff --git a/lib/sg_fargate_rails.rb b/lib/sg_fargate_rails.rb index b27d53b..cd9144d 100644 --- a/lib/sg_fargate_rails.rb +++ b/lib/sg_fargate_rails.rb @@ -2,6 +2,8 @@ require_relative "sg_fargate_rails/version" require_relative "sg_fargate_rails/config" +require_relative "sg_fargate_rails/current_ecs_task" +require_relative "sg_fargate_rails/event_bridge_schedule" require 'lograge' if defined?(::Rails::Railtie) diff --git a/lib/sg_fargate_rails/current_ecs_task.rb b/lib/sg_fargate_rails/current_ecs_task.rb new file mode 100644 index 0000000..46e9dfd --- /dev/null +++ b/lib/sg_fargate_rails/current_ecs_task.rb @@ -0,0 +1,82 @@ +require 'net/http' +require 'json' +require 'aws-sdk-ec2' + +module SgFargateRails + class CurrentEcsTask + def cluster_arn + metadata[:Cluster] + end + + def task_definition_arn + "#{cluster_arn.split(":cluster/")[0]}:task-definition/#{metadata[:Family]}:#{metadata[:Revision]}" + end + + def cfn_stack_name + "#{ENV['COPILOT_APPLICATION_NAME']}-#{ENV['COPILOT_ENVIRONMENT_NAME']}" + end + + def security_group_ids + @security_group_ids ||= fetch_security_group_ids + end + + def public_subnet_ids + @public_subnet_ids ||= fetch_public_subnet_ids + end + + private + + def metadata + @metadata ||= begin + response = Net::HTTP.get(URI.parse("#{ENV['ECS_CONTAINER_METADATA_URI']}/task")) + JSON.parse(response, symbolize_names: true) + end + end + + def region + ENV['AWS_REGION'] || 'ap-northeast-1' + end + + def ec2_client + @ec2_client ||= Aws::EC2::Client.new(region: region, credentials: credentials) + end + + def credentials + @credentials ||= Aws::ECSCredentials.new(retries: 3) + end + + def fetch_security_group_ids + security_group_params = { + filters: [ + { + name: 'tag:aws:cloudformation:logical-id', + values: ['EnvironmentSecurityGroup'], + }, + { + name: 'tag:aws:cloudformation:stack-name', + values: [cfn_stack_name], + } + ], + } + resp = ec2_client.describe_security_groups(security_group_params) + resp.to_h[:security_groups].map { |group| group[:group_id] } + end + + def fetch_public_subnet_ids + subnet_params = { + filters: [ + { + name: 'tag:aws:cloudformation:logical-id', + values: %w[PublicSubnet1 PublicSubnet2], + }, + { + name: 'tag:aws:cloudformation:stack-name', + values: [cfn_stack_name], + }, + ], + } + resp = ec2_client.describe_subnets(subnet_params) + resp.to_h[:subnets].map { |subnet| subnet[:subnet_id] } + end + end +end diff --git a/lib/sg_fargate_rails/event_bridge_schedule.rb b/lib/sg_fargate_rails/event_bridge_schedule.rb new file mode 100644 index 0000000..78e9b22 --- /dev/null +++ b/lib/sg_fargate_rails/event_bridge_schedule.rb @@ -0,0 +1,125 @@ +require "aws-sdk-scheduler" + +module SgFargateRails + class EventBridgeSchedule + CONTAINER_TYPES = { + 'small' => { cpu: '512', memory: '1024', }, + 'medium' => { cpu: '1024', memory: '2048', }, + 'large' => { cpu: '2048', memory: '4096', }, + }.freeze + + attr_reader :name + + def initialize(name, cron, command, container_type) + @name = name + @cron = cron + @command = command + @container_type = container_type + end + + def create_run_task(group_name:, cluster_arn:, task_definition_arn:, network_configuration:) + params = { + name: @name, + state: 'ENABLED', + flexible_time_window: { mode: 'OFF' }, + group_name: group_name, + schedule_expression: @cron, + schedule_expression_timezone: timezone, + target: { + arn: cluster_arn, + ecs_parameters: { + task_count: 1, + task_definition_arn: task_definition_arn, + launch_type: 'FARGATE', + network_configuration: network_configuration + }, + input: input_overrides_json, + retry_policy: { + maximum_event_age_in_seconds: 120, + maximum_retry_attempts: 2, + }, + role_arn: role_arn_for(group_name, cluster_arn), + }, + } + client.create_schedule(params) + end + + def input_overrides_json + type = convert_container_type + if type + { + **type, + "containerOverrides": [ + { + "name": "rails", + **type, + "command": container_command, + } + ] + }.to_json + else + { + "containerOverrides": [ + { + "name": "rails", + "command": container_command, + } + ] + }.to_json + end + end + + def convert_container_type + CONTAINER_TYPES[@container_type] + end + + def container_command + %w[bundle exec] + @command.split(' ') + end + + private + + def timezone + ENV['TZ'] || 'Asia/Tokyo' + end + + def role_arn_for(group_name, cluster_arn) + account_id = cluster_arn.split(':')[4] + "arn:aws:iam::#{account_id}:role/#{group_name}-eventbridge-scheduler-role" + end + + def client + self.class.client + end + + class << self + def parse(filename) + schedules = YAML.unsafe_load(File.open(filename))[environment] + schedules.map { |name, info| EventBridgeSchedule.new(name, info['cron'], info['command'], info['container_type']) if name != '<<' } + end + + def delete_all!(group_name) + client.list_schedules(group_name: group_name, max_results: 100).schedules.each do |schedule| + client.delete_schedule(name: schedule.name, group_name: group_name) + Rails.logger.info "[EventBridgeSchedule] Deleted #{group_name}/#{schedule.name}" + end + end + + def client + @client ||= Aws::Scheduler::Client.new(region: region, credentials: credentials) + end + + def environment + ENV['RAILS_ENV'] + end + + def region + ENV['AWS_REGION'] || 'ap-northeast-1' + end + + def credentials + Aws::ECSCredentials.new(retries: 3) + end + end + end +end diff --git a/lib/sg_fargate_rails/railtie.rb b/lib/sg_fargate_rails/railtie.rb index eb5ea83..7dedfba 100644 --- a/lib/sg_fargate_rails/railtie.rb +++ b/lib/sg_fargate_rails/railtie.rb @@ -6,6 +6,10 @@ module SgFargateRails class Railtie < ::Rails::Railtie + rake_tasks do + load File.expand_path('../tasks/sg_fargate_rails.rake', __dir__) + end + initializer :initialize_sg_fargate_rails do |app| unless ::Rails.env.in?(%w[development test]) SgFargateRails::RackAttack.setup diff --git a/lib/sg_fargate_rails/version.rb b/lib/sg_fargate_rails/version.rb index a3b67fb..cade099 100644 --- a/lib/sg_fargate_rails/version.rb +++ b/lib/sg_fargate_rails/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SgFargateRails - VERSION = "0.1.8" + VERSION = "0.1.9" end diff --git a/lib/tasks/sg_fargate_rails.rake b/lib/tasks/sg_fargate_rails.rake new file mode 100644 index 0000000..2baa491 --- /dev/null +++ b/lib/tasks/sg_fargate_rails.rake @@ -0,0 +1,33 @@ +namespace :sg_fargate_rails do + require 'sg_fargate_rails' + + desc 'EventBridge Schedules' + task recreate_schedules: :environment do + ecs_task = SgFargateRails::CurrentEcsTask.new + Rails.logger.info "[INFO] security_group_ids: #{ecs_task.security_group_ids}" + Rails.logger.info "[INFO] subnet_ids: #{ecs_task.public_subnet_ids}" + + group_name = ecs_task.cfn_stack_name + Rails.logger.info "[EventBridgeSchedule] Clear all schedules in #{group_name}" + SgFargateRails::EventBridgeSchedule.delete_all!(group_name) + + Rails.logger.info "[EventBridgeSchedule] Register schedules in #{group_name}" + config_file = Rails.root.join('config/eventbridge_schedules.yml') + SgFargateRails::EventBridgeSchedule.parse(config_file).each do |schedule| + Rails.logger.info "[EventBridgeSchedule] Register schedule #{schedule.name} in #{group_name}" + # TODO: この辺で AWS の API Limit などのエラーが発生するとスケジュールが消えたままとなるので、エラーの内容に応じてリトライなどのエラー処理が必要 + schedule.create_run_task( + group_name: group_name, + cluster_arn: ecs_task.cluster_arn, + task_definition_arn: ecs_task.task_definition_arn, + network_configuration: { + awsvpc_configuration: { + assign_public_ip: 'ENABLED', + security_groups: ecs_task.security_group_ids, + subnets: ecs_task.public_subnet_ids, + }, + } + ) + end + end +end diff --git a/sg_fargate_rails.gemspec b/sg_fargate_rails.gemspec index babe328..d8a2fbe 100644 --- a/sg_fargate_rails.gemspec +++ b/sg_fargate_rails.gemspec @@ -1,5 +1,4 @@ # frozen_string_literal: true - require_relative "lib/sg_fargate_rails/version" Gem::Specification.new do |spec| @@ -40,4 +39,8 @@ Gem::Specification.new do |spec| spec.add_dependency 'puma' spec.add_dependency 'lograge', '~> 0.12' spec.add_dependency 'rack-attack', '~> 6.6' + spec.add_dependency 'aws-sdk-ec2', '~> 1.413' + spec.add_dependency 'aws-sdk-scheduler', '~> 1.10' + + spec.add_development_dependency 'rspec' end diff --git a/spec/fixtures/event_bridge_schedule/blank_schedule.yml b/spec/fixtures/event_bridge_schedule/blank_schedule.yml new file mode 100644 index 0000000..a518dd8 --- /dev/null +++ b/spec/fixtures/event_bridge_schedule/blank_schedule.yml @@ -0,0 +1,7 @@ +common: &common + +staging: + <<: *common + +production: + <<: *common \ No newline at end of file diff --git a/spec/fixtures/event_bridge_schedule/schedule.yml b/spec/fixtures/event_bridge_schedule/schedule.yml new file mode 100644 index 0000000..640b0b8 --- /dev/null +++ b/spec/fixtures/event_bridge_schedule/schedule.yml @@ -0,0 +1,11 @@ +common: &common + daily_backup_to_s3: + command: 'jobmon --estimate-time=3000 sg_tiny_backup:backup' + cron: 'cron(30 1 * * ? *)' + container_type: medium + +staging: + <<: *common + +production: + <<: *common \ No newline at end of file diff --git a/spec/sg_fargate_rails/event_bridge_schedule_spec.rb b/spec/sg_fargate_rails/event_bridge_schedule_spec.rb new file mode 100644 index 0000000..e048dea --- /dev/null +++ b/spec/sg_fargate_rails/event_bridge_schedule_spec.rb @@ -0,0 +1,76 @@ +require 'spec_helper' + +describe SgFargateRails::EventBridgeSchedule do + + describe '.input_overrides_json' do + let(:cron) { 'cron(30 16 * * ? *)' } + let(:command) { 'jobmon --estimate-time=3000 stat' } + let(:schedule) { SgFargateRails::EventBridgeSchedule.new('name', cron, command, container_type) } + + subject { schedule.input_overrides_json } + + context 'container_typeが指定されている場合' do + let(:container_type) { 'small' } + + it 'cpuやmemoryの情報が補完されること' do + is_expected.to eq( + { + "cpu": "512", + "memory": "1024", + "containerOverrides": [ + { + "name": "rails", + "cpu": "512", + "memory": "1024", + "command": %w[bundle exec jobmon --estimate-time=3000 stat], + } + ] + }.to_json + ) + end + end + + context 'container_typeが指定されていない場合' do + let(:container_type) { nil } + + it do + is_expected.to eq( + { + "containerOverrides": [ + { + "name": "rails", + "command": %w[bundle exec jobmon --estimate-time=3000 stat], + } + ] + }.to_json + ) + end + end + end + + describe '#parse' do + context 'scheduleの登録がない場合' do + let(:filename) { 'spec/fixtures/event_bridge_schedule/blank_schedule.yml' } + + it do + allow(ENV).to receive(:[]).and_call_original + allow(ENV).to receive(:[]).with('RAILS_ENV').and_return('staging') + + expect(SgFargateRails::EventBridgeSchedule.parse(filename)).to eq [nil] + end + end + + context 'scheduleの登録が複数存在する場合' do + let(:filename) { 'spec/fixtures/event_bridge_schedule/schedule.yml' } + + it do + allow(ENV).to receive(:[]).and_call_original + allow(ENV).to receive(:[]).with('RAILS_ENV').and_return('staging') + + results = SgFargateRails::EventBridgeSchedule.parse(filename) + expect(results.size).to eq 1 + expect(results.first.name).to eq 'daily_backup_to_s3' + end + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 0000000..84092eb --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,4 @@ +require 'sg_fargate_rails' +require 'rubygems' +require 'rspec' +require 'yaml' \ No newline at end of file