From ee8d05639aa9c225d58350e320e2a94f84213f5a Mon Sep 17 00:00:00 2001 From: Jonathan Feist Date: Wed, 11 Aug 2021 21:03:09 +0100 Subject: [PATCH 1/9] refactor: global stats habitat methods - reduce number of database hits --- app/models/habitat.rb | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/app/models/habitat.rb b/app/models/habitat.rb index f3b6ee13..03af309c 100644 --- a/app/models/habitat.rb +++ b/app/models/habitat.rb @@ -41,10 +41,11 @@ def calculate_global_cover_change }) end - def global_stats + def global_stats + stats = geo_entity_stats.country_stats.to_a # reduce hits to database { - total_habitat_cover: geo_entity_stats.country_stats.pluck(:total_value).compact.reduce(&:+), - protected_habitat_cover: geo_entity_stats.country_stats.pluck(:protected_value).compact.reduce(&:+) + total_habitat_cover: stats.pluck(:total_value).compact.reduce(&:+), + protected_habitat_cover: stats.pluck(:protected_value).compact.reduce(&:+) } end @@ -87,8 +88,10 @@ def type def global_protection stats = { 'name' => name, 'total_value' => 0, 'protected_value' => 0 } - stats['total_value'] = global_stats[:total_habitat_cover] - stats['protected_value'] = global_stats[:protected_habitat_cover] + global_stats_data = global_stats # reduce hits to database + + stats['total_value'] = global_stats_data[:total_habitat_cover] + stats['protected_value'] = global_stats_data[:protected_habitat_cover] protected_value = stats['protected_value'] > 0 ? stats['protected_value'] : 1 stats.merge({'protected_percentage' => protected_value / stats['total_value'] * 100}) From 222d233592ed832ff6c93f4856926ffd3e272187 Mon Sep 17 00:00:00 2001 From: Jonathan Feist Date: Wed, 11 Aug 2021 21:16:20 +0100 Subject: [PATCH 2/9] refactor: format habitats model --- app/models/habitat.rb | 122 +++++++++++++++++++++++++++++------------- 1 file changed, 86 insertions(+), 36 deletions(-) diff --git a/app/models/habitat.rb b/app/models/habitat.rb index 03af309c..08da2c8b 100644 --- a/app/models/habitat.rb +++ b/app/models/habitat.rb @@ -13,40 +13,75 @@ def protected_title end def calculate_country_cover_change(country_name) - country_cover_change = { change_km: 0, change_percentage: 0 } + country_cover_change = { + change_km: 0, + change_percentage: 0 + } + + # We only have mangroves data at the moment + return country_cover_change unless name == 'mangroves' - # We only got mangroves data at the moment - return country_cover_change unless name == "mangroves" geo_entity_id = GeoEntity.find_by(name: country_name).id - habitat_base_year = ChangeStat.find_by(habitat_id: id, geo_entity_id: geo_entity_id)&.send("total_value_#{baseline_year}".to_sym) - habitat_last_year = ChangeStat.find_by(habitat_id: id, geo_entity_id: geo_entity_id)&.send(latest_year) - return country_cover_change if (habitat_base_year.nil? || habitat_last_year.nil?) + habitat_base_year = ChangeStat.find_by( + habitat_id: id, + geo_entity_id: geo_entity_id + ) + &.send("total_value_#{baseline_year}".to_sym) + + habitat_last_year = ChangeStat.find_by( + habitat_id: id, + geo_entity_id: geo_entity_id + ) + &.send(latest_year) + + return country_cover_change if habitat_base_year.nil? || habitat_last_year.nil? + change_km = habitat_last_year - habitat_base_year - change_percentage = (change_km/habitat_base_year) * 100 + change_percentage = (change_km / habitat_base_year) * 100 - country_cover_change.merge!({change_km: ActiveSupport::NumberHelper.number_to_delimited(change_km.round(2)), change_percentage: change_percentage.round(2)}) + country_cover_change.merge!( + { + change_km: ActiveSupport::NumberHelper.number_to_delimited(change_km.round(2)), + change_percentage: change_percentage.round(2) + } + ) end def calculate_global_cover_change - global_cover_change = { change_km: 0, change_percentage: 0, baseline_year: baseline_year, original_total: 0 } - return global_cover_change unless name == "mangroves" - habitat_base_year = ChangeStat.includes(:geo_entity).where.not(geo_entities: { iso3: nil }).where(habitat_id: id).pluck("total_value_#{baseline_year}".to_sym).inject(0) { |sum, x| sum + x } - habitat_last_year = ChangeStat.includes(:geo_entity).where.not(geo_entities: { iso3: nil }).where(habitat_id: id).pluck(latest_year).inject(0) { |sum, x| sum + x } + global_cover_change = { + change_km: 0, + change_percentage: 0, + baseline_year: baseline_year, + original_total: 0 + } + + return global_cover_change unless name == 'mangroves' + + habitat_base_year = ChangeStat.includes(:geo_entity) + .where + .not(geo_entities: { iso3: nil }) + .where(habitat_id: id) + .pluck("total_value_#{baseline_year}".to_sym) + .inject(0) { |sum, x| sum + x } + + habitat_last_year = ChangeStat.includes(:geo_entity) + .where + .not(geo_entities: { iso3: nil }) + .where(habitat_id: id) + .pluck(latest_year) + .inject(0) { |sum, x| sum + x } + total_value_change = habitat_last_year - habitat_base_year total_value_change_percentage = (total_value_change / habitat_base_year) * 100 - global_cover_change.merge!({ - change_km: total_value_change.round(2), change_percentage: total_value_change_percentage.round(2), - baseline_year: baseline_year, original_total: habitat_base_year.round(2) - }) - end - - def global_stats - stats = geo_entity_stats.country_stats.to_a # reduce hits to database - { - total_habitat_cover: stats.pluck(:total_value).compact.reduce(&:+), - protected_habitat_cover: stats.pluck(:protected_value).compact.reduce(&:+) - } + global_cover_change.merge!( + { + change_km: total_value_change.round(2), + change_percentage: total_value_change_percentage.round(2), + baseline_year: baseline_year, + original_total: habitat_base_year.round(2) + } + ) end def calculate_global_protection @@ -71,7 +106,7 @@ def total_value_by_country c = Carto.new(name) total_value_by_country = 0 - if name == "coldcorals" + if name == 'coldcorals' total_value_by_country = c.total_points_by_country total_value_by_country = sort_country_count(total_value_by_country) else @@ -82,19 +117,32 @@ def total_value_by_country end def type - name == "coldcorals" ? "points" : "area" + name == 'coldcorals' ? 'points' : 'area' + end + + def global_stats + stats = geo_entity_stats.country_stats.to_a # reduce hits to database + { + total_habitat_cover: stats.pluck(:total_value).compact.reduce(&:+), + protected_habitat_cover: stats.pluck(:protected_value).compact.reduce(&:+) + } end def global_protection - stats = { 'name' => name, 'total_value' => 0, 'protected_value' => 0 } + stats = { + 'name' => name, + 'total_value' => 0, + 'protected_value' => 0 + } global_stats_data = global_stats # reduce hits to database stats['total_value'] = global_stats_data[:total_habitat_cover] stats['protected_value'] = global_stats_data[:protected_habitat_cover] - protected_value = stats['protected_value'] > 0 ? stats['protected_value'] : 1 - stats.merge({'protected_percentage' => protected_value / stats['total_value'] * 100}) + protected_value = stats['protected_value'].positive? ? stats['protected_value'] : 1 + + stats.merge({ 'protected_percentage' => protected_value / stats['total_value'] * 100 }) end def self.global_protection @@ -103,7 +151,7 @@ def self.global_protection def self.global_protection_by_id hash = {} - self.global_protection.each do |habitat_stats| + global_protection.each do |habitat_stats| hash[habitat_stats['name']] = habitat_stats.except('name') end hash @@ -114,10 +162,11 @@ def self.global_protection_by_id def sum_country_areas(total_area_by_country) country_total_area = {} total_area_by_country.flatten.each do |country_data| - next if country_data["iso3"].include? "/" #remove areas which have multiple iso - next if country_data["iso3"].include? "ABNJ" #remove ABNJ - country_total_area[country_data["iso3"]] ||= 0 - country_total_area[country_data["iso3"]] += country_data["sum"] + next if country_data['iso3'].include? '/' # remove areas which have multiple iso + next if country_data['iso3'].include? 'ABNJ' # remove ABNJ + + country_total_area[country_data['iso3']] ||= 0 + country_total_area[country_data['iso3']] += country_data['sum'] end country_total_area end @@ -125,8 +174,9 @@ def sum_country_areas(total_area_by_country) def sort_country_count(total_value_by_country) country_total_points = {} total_value_by_country.each do |total_value| - next if total_value["iso3"].include? "ABNJ" - country_total_points[total_value["iso3"]] = total_value["count"] + next if total_value['iso3'].include? 'ABNJ' + + country_total_points[total_value['iso3']] = total_value['count'] end country_total_points end From 8e20f36fc68e437e45ab3c913741a93f2319e40d Mon Sep 17 00:00:00 2001 From: Jonathan Feist Date: Thu, 12 Aug 2021 01:44:19 +0100 Subject: [PATCH 3/9] fix: add GBL check to country_stats scope - fix countries scope in GeoEntity model that would never check for NULL --- app/models/geo_entity.rb | 2 +- app/models/geo_entity_stat.rb | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/app/models/geo_entity.rb b/app/models/geo_entity.rb index 35068733..303c62f5 100644 --- a/app/models/geo_entity.rb +++ b/app/models/geo_entity.rb @@ -15,7 +15,7 @@ class GeoEntity < ApplicationRecord has_many :country_citations, foreign_key: 'country_id' - scope :countries, -> { where.not(iso3: nil || 'GBL') } + scope :countries, -> { where.not(iso3: nil).where.not(iso3: 'GBL') } scope :regions, -> { where(iso3: nil) } # Only allowing actual countries to be considered for the 'Next country' button diff --git a/app/models/geo_entity_stat.rb b/app/models/geo_entity_stat.rb index 463885a0..6d7c21c8 100644 --- a/app/models/geo_entity_stat.rb +++ b/app/models/geo_entity_stat.rb @@ -4,10 +4,13 @@ class GeoEntityStat < ApplicationRecord has_many :sources, through: :geo_entity_stats_sources has_many :geo_entity_stats_sources, class_name: 'GeoEntityStatsSources', dependent: :destroy - scope :country_stats, -> { joins(:geo_entity).where('geo_entities.iso3 IS NOT NULL') } + scope :country_stats, lambda { + joins(:geo_entity).where.not('geo_entities.iso3': nil) + .where.not('geo_entities.iso3': 'GBL') + } - # TODO - consider adding :present-but-unknown to this list which will require some thinking - enum occurrence: [:absent, :unknown, :present] + # TODO: - consider adding :present-but-unknown to this list which will require some thinking + enum occurrence: %i[absent unknown present] BASE_OCCURRENCES = { 'coralreefs' => 'unknown', @@ -15,5 +18,5 @@ class GeoEntityStat < ApplicationRecord 'mangroves' => 'unknown', 'seagrasses' => 'unknown', 'coldcorals' => 'unknown' - } + }.freeze end From d3cf5eadc1034d57b20b3d77df55563486a3a18a Mon Sep 17 00:00:00 2001 From: Jonathan Feist Date: Thu, 12 Aug 2021 01:58:08 +0100 Subject: [PATCH 4/9] feat: add experimental methods that replace Habitat.global_protection Replacing the existing methods would entail checking where they are used and where their related/dependent methods are also used, e.g. if used in tests, then the tests will need to be updated --- app/models/habitat.rb | 58 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/app/models/habitat.rb b/app/models/habitat.rb index 08da2c8b..706cabc5 100644 --- a/app/models/habitat.rb +++ b/app/models/habitat.rb @@ -157,6 +157,64 @@ def self.global_protection_by_id hash end + # experimental alternative to self.global_protection_by_id: not currently used + def self.global_protection_by_id_v2 + hash = {} + global_protection_v2.each do |habitat_stats| + hash[habitat_stats['name']] = habitat_stats.except('name') + end + hash + end + + # experimental alternative to self.global_protection: not currently used + # TODO: be brave, use this method instead of the original, it's cool and its + # 2x faster than the original + # TODO: check codebase for how other related methods are used e.g. + # global_stats and global_protection methods. e.g. they are in habitat_spec.rb + def self.global_protection_v2 + geo_entities = GeoEntity.arel_table + geo_entity_stats = GeoEntityStat.arel_table + habitats = Habitat.arel_table + + # get all the data we need in a single query + # @see scuttle.io for sql->arel help + query = GeoEntityStat.select( + [ + habitats[:name], + geo_entity_stats[:total_value].sum.as('total_value'), + geo_entity_stats[:protected_value].sum.as('protected_value') + ] + ) + .where( + geo_entities[:iso3].not_eq(nil).and(geo_entities[:iso3].not_eq('GBL')) + ) + .joins( + geo_entity_stats.join(geo_entities).on( + geo_entities[:id].eq(geo_entity_stats[:geo_entity_id]) + ).join_sources + ) + .joins( + geo_entity_stats.join(habitats).on( + habitats[:id].eq(geo_entity_stats[:habitat_id]) + ).join_sources + ) + .group(habitats[:name]) + + results = ActiveRecord::Base.connection.execute(query.to_sql) + + results.map do |row| + total_value = row['total_value'].to_f + protected_value = row['protected_value'].to_f + protected_percentage = protected_value / total_value * 100 + { + name: row['name'], + total_value: total_value.round(2), + protected_value: protected_value.round(2), + protected_percentage: protected_percentage.round(2) + }.stringify_keys + end + end + private def sum_country_areas(total_area_by_country) From c5f3a6e0e9ea8833bcdd19e487589490632b3136 Mon Sep 17 00:00:00 2001 From: Jonathan Feist Date: Fri, 13 Aug 2021 16:45:50 +0100 Subject: [PATCH 5/9] fix: change global stats to use pre-calculated stats --- app/models/habitat.rb | 62 +++++++++++++++---- config/habitats.yml | 10 +++ ...t_coverage_protection_stats_to_habitats.rb | 6 ++ db/schema.rb | 4 +- lib/tasks/import_habitats.rake | 17 ++++- 5 files changed, 84 insertions(+), 15 deletions(-) create mode 100644 db/migrate/20210812135252_add_global_habitat_coverage_protection_stats_to_habitats.rb diff --git a/app/models/habitat.rb b/app/models/habitat.rb index 706cabc5..3142e28d 100644 --- a/app/models/habitat.rb +++ b/app/models/habitat.rb @@ -4,6 +4,26 @@ class Habitat < ApplicationRecord has_many :change_stats has_many :species + validates_presence_of :name, + :title, + :theme, + :total_area, # see NOTE(1) + :protected_area # see NOTE(1) + + validates_numericality_of :total_area, { # see NOTE(1) + greater_than_or_equal_to: ->(habitat) { habitat.protected_area } + } + + validates_numericality_of :protected_area, { # see NOTE(1) + greater_than_or_equal_to: 0, + less_than_or_equal_to: ->(habitat) { habitat.total_area } + } + + # NOTE(1): if stats are ever generated automatically again - this could be removed + def percent_protected + protected_area / total_area * 100 + end + def global_coverage_title(habitat_type) habitat_type == 'points' ? "Total number of #{title.downcase} records globally" : "Total global recorded coverage of #{title.downcase}" end @@ -121,28 +141,48 @@ def type end def global_stats - stats = geo_entity_stats.country_stats.to_a # reduce hits to database { - total_habitat_cover: stats.pluck(:total_value).compact.reduce(&:+), - protected_habitat_cover: stats.pluck(:protected_value).compact.reduce(&:+) + total_habitat_cover: total_area, + protected_habitat_cover: protected_area } + + # TODO: maybe re-implement this code if stats gets generated by the app again + # otherwise, allow this to remain commented for the indefinite future + + # stats = geo_entity_stats.country_stats.to_a # reduce hits to database + # { + # total_habitat_cover: stats.pluck(:total_value).compact.reduce(&:+), + # protected_habitat_cover: stats.pluck(:protected_value).compact.reduce(&:+) + # } end def global_protection - stats = { + global_stats_data = global_stats + + { 'name' => name, - 'total_value' => 0, - 'protected_value' => 0 + 'protected_percentage' => percent_protected, + 'total_value' => global_stats_data[:total_habitat_cover], + 'protected_value' => global_stats_data[:protected_habitat_cover] } - global_stats_data = global_stats # reduce hits to database + # TODO: see global_stats comments - we can now use a computed property if the stats + # are pre-calculated - this logic below could be restored if stats are generated + # automatically again + + # stats = { + # 'name' => name, + # 'total_value' => 0, + # 'protected_value' => 0 + # } - stats['total_value'] = global_stats_data[:total_habitat_cover] - stats['protected_value'] = global_stats_data[:protected_habitat_cover] + # global_stats_data = global_stats # reduce hits to database - protected_value = stats['protected_value'].positive? ? stats['protected_value'] : 1 + # stats['total_value'] = global_stats_data[:total_habitat_cover] + # stats['protected_value'] = global_stats_data[:protected_habitat_cover] - stats.merge({ 'protected_percentage' => protected_value / stats['total_value'] * 100 }) + # protected_value = stats['protected_value'].positive? ? stats['protected_value'] : 1 + # stats.merge({ 'protected_percentage' => protected_value / stats['total_value'] * 100 }) end def self.global_protection diff --git a/config/habitats.yml b/config/habitats.yml index 257452f9..3d6e2d08 100644 --- a/config/habitats.yml +++ b/config/habitats.yml @@ -6,6 +6,8 @@ habitats: poly_table: 'wcmc008_coralreef2010_py_v4' point_table: 'wcmc008_coralreef2010_pt_v4' wms_url: 'https://gis.unep-wcmc.org/arcgis/rest/services/marine/WCMC_008_CoralReefs_WMS/MapServer/export?transparent=true&format=png32&bbox={bbox-epsg-3857}&bboxSR=EPSG:3857&imageSR=EPSG:3857&size=256,256&f=image' + total_area: 149886.974126303 + protected_area: 67431.4170347656 saltmarshes: name: 'saltmarshes' title: 'Saltmarshes' @@ -13,6 +15,8 @@ habitats: poly_table: 'wcmc027_saltmarshes_py_v6' point_table: 'wcmc027_saltmarshes_pt_v6' wms_url: 'https://gis.unep-wcmc.org/arcgis/rest/services/marine/WCMC_027_Saltmarsh_WMS/MapServer/export?transparent=true&format=png32&bbox={bbox-epsg-3857}&bboxSR=EPSG:3857&imageSR=EPSG:3857&size=256,256&f=image' + total_area: 224435.075089843 + protected_area: 111636.441534896 mangroves: name: 'mangroves' title: 'Mangroves' @@ -20,6 +24,8 @@ habitats: poly_table: 'wcmc011_atlasmangrove2010_py_v3' point_table: wms_url: 'https://gis.unep-wcmc.org/arcgis/rest/services/marine/WCMC_011_WorldAtlasMangroves_WMS/MapServer/export?transparent=true&format=png32&bbox={bbox-epsg-3857}&bboxSR=EPSG:3857&imageSR=EPSG:3857&size=256,256&f=image' + total_area: 135869.552269994 + protected_area: 58109.098834161 seagrasses: name: 'seagrasses' title: 'Seagrasses' @@ -27,6 +33,8 @@ habitats: poly_table: 'wcmc_013_014_seagrassespy_v6' point_table: 'wcmc_013_014_seagrassespt_v6' wms_url: 'https://gis.unep-wcmc.org/arcgis/rest/services/marine/WCMC_013_014_Seagrass_WMS/MapServer/export?transparent=true&format=png32&bbox={bbox-epsg-3857}&bboxSR=EPSG:3857&imageSR=EPSG:3857&size=256,256&f=image' + total_area: 314001.940552758 + protected_area: 83606.1196777014 coldcorals: name: 'coldcorals' title: 'Cold-water corals' @@ -34,4 +42,6 @@ habitats: poly_table: 'wcmc001_coldcorals2017_py_v5' point_table: 'wcmc001_coldcorals2017_pt_v5' wms_url: 'https://gis.unep-wcmc.org/arcgis/rest/services/marine/WCMC_001_ColdCorals2017_WMS/MapServer/export?transparent=true&format=png32&bbox={bbox-epsg-3857}&bboxSR=EPSG:3857&imageSR=EPSG:3857&size=256,256&f=image' + total_area: 15336.9752787556 + protected_area: 4505.64215821009 diff --git a/db/migrate/20210812135252_add_global_habitat_coverage_protection_stats_to_habitats.rb b/db/migrate/20210812135252_add_global_habitat_coverage_protection_stats_to_habitats.rb new file mode 100644 index 00000000..2a530946 --- /dev/null +++ b/db/migrate/20210812135252_add_global_habitat_coverage_protection_stats_to_habitats.rb @@ -0,0 +1,6 @@ +class AddGlobalHabitatCoverageProtectionStatsToHabitats < ActiveRecord::Migration[5.1] + def change + add_column :habitats, :total_area, :float, null: true + add_column :habitats, :protected_area, :float, null: true + end +end diff --git a/db/schema.rb b/db/schema.rb index 26b89f96..5518d7ca 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20210525133211) do +ActiveRecord::Schema.define(version: 20210812135252) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -125,6 +125,8 @@ t.datetime "created_at", null: false t.datetime "updated_at", null: false t.text "wms_url" + t.float "total_area" + t.float "protected_area" end create_table "sources", force: :cascade do |t| diff --git a/lib/tasks/import_habitats.rake b/lib/tasks/import_habitats.rake index 773260e2..808dd07e 100644 --- a/lib/tasks/import_habitats.rake +++ b/lib/tasks/import_habitats.rake @@ -1,12 +1,23 @@ +# NOTE(1): if the global stats are ever generated by the app again, we might +# no-longer need to have the total_area and protected_area attributes namespace :import do desc "import CSV data into database" task :habitats, [:csv_file] => [:environment] do habitats_config = YAML.load(File.open("#{Rails.root}/config/habitats.yml", 'r')) habitats_config['habitats'].each do |name, data| - Habitat.where(name: data['name']).first_or_create do |habitat| - habitat.update_attributes(data) - end + attributes = { + title: data['title'], + theme: data['theme'], + poly_table: data['poly_table'], + point_table: data['point_table'], + wms_url: data['wms_url'], + total_area: data['total_area'].to_f, # see NOTE(1) + protected_area: data['protected_area'].to_f # see NOTE(1) + } + + Habitat.find_or_create_by(name: data['name']).update_attributes(attributes) + Rails.logger.info "#{name.capitalize} habitat created!" end end From 6a1ce6971db901820ed2f562304cfb6b45c4d096 Mon Sep 17 00:00:00 2001 From: Jonathan Feist Date: Fri, 13 Aug 2021 23:27:08 +0100 Subject: [PATCH 6/9] feat: refresh records after server updates a release --- config/deploy.rb | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/config/deploy.rb b/config/deploy.rb index 4b4ca0d1..085dc7e6 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -59,3 +59,14 @@ set :keep_releases, 5 set :passenger_restart_with_touch, false + +namespace :deploy do + desc 'Delete and recreate records' + task :import_refresh do + on roles(:app) do + invoke "import:refresh" + end + end +end + +before 'deploy:publishing', 'deploy:import_refresh' From 2940bbb9a1ceea8fe159b345e12431a8c5b4402d Mon Sep 17 00:00:00 2001 From: Jonathan Feist Date: Sat, 14 Aug 2021 01:58:08 +0100 Subject: [PATCH 7/9] style: format deploy.rb --- config/deploy.rb | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/config/deploy.rb b/config/deploy.rb index 085dc7e6..ad60626b 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # config valid only for current version of Capistrano # lock "3.11.0" @@ -7,51 +9,43 @@ set :nvm_type, :user # or :system, depends on your nvm setup set :nvm_node, 'v10.15.1' -set :nvm_map_bins, %w{node npm yarn} +set :nvm_map_bins, %w[node npm yarn] set :deploy_user, 'wcmc' - set :backup_path, "/home/#{fetch(:deploy_user)}/Backup" - - # Default deploy_to directory is /var/www/my_app_name set :deploy_to, "/home/#{fetch(:deploy_user)}/#{fetch(:application)}" # Default value for :scm is :git set :scm, :git -set :scm_username, "unepwcmc-read" - +set :scm_username, 'unepwcmc-read' set :rvm_type, :user set :rvm_ruby_version, '2.5.0' - - set :ssh_options, { - forward_agent: true, + forward_agent: true } - # Default value for :format is :pretty # set :format, :pretty # Default value for :log_level is :debug -#set :log_level, :debug +# set :log_level, :debug # Default value for :pty is false set :pty, true # Default value for :linked_files is [] -#set :linked_files, fetch(:linked_files, []).push('config/database.yml', 'config/secrets.yml') +# set :linked_files, fetch(:linked_files, []).push('config/database.yml', 'config/secrets.yml') -set :linked_files, %w{config/database.yml .env} +set :linked_files, %w[config/database.yml .env] # Default value for linked_dirs is [] set :linked_dirs, fetch(:linked_dirs, []).push('log', 'tmp/pids', 'tmp/cache', 'tmp/sockets', 'vendor/bundle', 'public/system', 'public/downloads', 'node_modules', 'client/node_modules') - # Default value for default_env is {} # set :default_env, { path: "/opt/ruby/bin:$PATH" } @@ -64,7 +58,7 @@ desc 'Delete and recreate records' task :import_refresh do on roles(:app) do - invoke "import:refresh" + invoke 'import:refresh' end end end From 83c52000f1cb76aaf71b3f3724d55e0e7a0225b1 Mon Sep 17 00:00:00 2001 From: Jonathan Feist Date: Sat, 14 Aug 2021 02:00:52 +0100 Subject: [PATCH 8/9] fix: change how to execute the rake task in deploy.rb - new option: TASK= if on staging --- config/deploy.rb | 12 ++++++++---- config/deploy/staging.rb | 24 ++++++------------------ lib/tasks/import_habitats.rake | 4 ++-- 3 files changed, 16 insertions(+), 24 deletions(-) diff --git a/config/deploy.rb b/config/deploy.rb index ad60626b..0d4edc45 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -55,12 +55,16 @@ set :passenger_restart_with_touch, false namespace :deploy do - desc 'Delete and recreate records' - task :import_refresh do + desc 'Run any rake task if specified (e.g. cap staging deploy TASK=import:refresh)' + task :run_task do on roles(:app) do - invoke 'import:refresh' + within release_path do + with rails_env: fetch(:rails_env) do + rake ENV['TASK'] + end + end end end end -before 'deploy:publishing', 'deploy:import_refresh' +before 'deploy:publishing', 'deploy:run_task' if ENV['TASK'] \ No newline at end of file diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index b17e39f0..988b887f 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -1,18 +1,12 @@ set :stage, :staging -set :branch, :develop - - - -server "web-supported-staging.linode.unep-wcmc.org", user: 'wcmc', roles: %w{app web db} - -set :application, "ocean-plus-habitats" -set :server_name, "ocean-plus-habitats.web-supported-staging.linode.unep-wcmc.org" -set :sudo_user, "wcmc" -set :app_port, "80" - - +set :branch, ENV['BRANCH'] || :develop +server 'web-supported-staging.linode.unep-wcmc.org', user: 'wcmc', roles: %w[app web db] +set :application, 'ocean-plus-habitats' +set :server_name, 'ocean-plus-habitats.web-supported-staging.linode.unep-wcmc.org' +set :sudo_user, 'wcmc' +set :app_port, '80' # server-based syntax # ====================== @@ -23,8 +17,6 @@ # server "example.com", user: "deploy", roles: %w{app web}, other_property: :other_value # server "db.example.com", user: "deploy", roles: %w{db} - - # role-based syntax # ================== @@ -37,8 +29,6 @@ # role :web, %w{user1@primary.com user2@additional.com}, other_property: :other_value # role :db, %w{deploy@example.com} - - # Configuration # ============= # You can set any configuration variable like in config/deploy.rb @@ -47,8 +37,6 @@ # http://capistranorb.com/documentation/getting-started/configuration/ # Feel free to add new variables to customise your setup. - - # Custom SSH Options # ================== # You may pass any option but keep in mind that net/ssh understands a diff --git a/lib/tasks/import_habitats.rake b/lib/tasks/import_habitats.rake index 808dd07e..6e8617b1 100644 --- a/lib/tasks/import_habitats.rake +++ b/lib/tasks/import_habitats.rake @@ -1,7 +1,7 @@ # NOTE(1): if the global stats are ever generated by the app again, we might # no-longer need to have the total_area and protected_area attributes namespace :import do - desc "import CSV data into database" + desc 'import CSV data into database' task :habitats, [:csv_file] => [:environment] do habitats_config = YAML.load(File.open("#{Rails.root}/config/habitats.yml", 'r')) @@ -21,4 +21,4 @@ namespace :import do Rails.logger.info "#{name.capitalize} habitat created!" end end -end \ No newline at end of file +end From 92494e9c1bf73d8f26f65f8c74af40b8dc988cd5 Mon Sep 17 00:00:00 2001 From: Jonathan Feist Date: Sat, 14 Aug 2021 23:51:53 +0100 Subject: [PATCH 9/9] chore: merge develop and update changelog for release 1.3.1 --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eac468c6..16f77820 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,4 +42,8 @@ opened by Dependabot. * refactor: raise more informative exception in Esri class if the response is not what was expected * feat: add utilities/files.rb with `latest_file_by_glob` method to help select the latest filename-timestamped CSV that is used in the _habitat coverage protection_ imports * refactor: add more informative error when a CSV is missing an expected header -* fix: wrap _import:refresh_ within a DB transaction so that it doesn't commit anything on failure \ No newline at end of file +* fix: wrap _import:refresh_ within a DB transaction so that it doesn't commit anything on failure + +## 1.3.1 + +* correctly merge develop into release and re-release