diff --git a/.gitignore b/.gitignore index 4d5f77ca..7d282ef1 100644 --- a/.gitignore +++ b/.gitignore @@ -30,4 +30,5 @@ yarn-debug.log* .env previous_failed_specs.txt -coverage \ No newline at end of file +coverage +/TAGS diff --git a/CHANGELOG.md b/CHANGELOG.md index fcc46ff3..eac468c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,4 +27,19 @@ system specs. Also included is a FactoryBot linter for linting factories. A couple of redundant methods have also been removed from the models. * Merged in a number of PRs fixing vulnerabilities on the frontend and backend that were -opened by Dependabot. \ No newline at end of file +opened by Dependabot. + +## 1.3.0 + +* chore: add `/TAGS` to .gitignore +* chore: upgrade puma from 3.7 ~> 4.3 +* fix: add missing "`WHERE geo_entities.iso3 IS NOT NULL`" to *countries* scope in `geo_entity.rb` and "`WHERE geo_entities.iso3 <> 'GBL'`" to *country_stats* scope in `geo_entity_stat.rb` +* fix: change global protection stats to be static within habitats table, as there was a problem with the automatic generation of global stats not including non-member state data yet. +* refactor: partially automate country last-updated by using the geo_entity's `updated_at` property +* refactor: set Red List last updated date (`@red_list_last_updated`) in site & countries controller +* feat: add rake invoke task to deploy.rb so that you can run any specified rake task before the deploy is published e.g. `bundle exec cap staging deploy TASK=import:refresh` +* feat: add `BRANCH=` option `cap staging deploy` so you can specify the branch to deploy to staging (default to develop) +* refactor: raise more informative exception in Esri class if the response is not what was expected +* feat: add utilities/files.rb with `latest_file_by_glob` method to help select the latest filename-timestamped CSV that is used in the _habitat coverage protection_ imports +* refactor: add more informative error when a CSV is missing an expected header +* fix: wrap _import:refresh_ within a DB transaction so that it doesn't commit anything on failure \ No newline at end of file diff --git a/app/controllers/countries_controller.rb b/app/controllers/countries_controller.rb index 73474733..b9e78f3b 100644 --- a/app/controllers/countries_controller.rb +++ b/app/controllers/countries_controller.rb @@ -6,7 +6,7 @@ def show return redirect_to(action: 'not_found', controller: 'errors') unless @country - # TODO - work out how to integrate i18n with this for country names - we may want to + # TODO: - work out how to integrate i18n with this for country names - we may want to # have a list of country names in each language which is then dynamically fetched from # a CSV/yml depending on the language selected @iso3 = @country&.iso3 @@ -26,8 +26,10 @@ def show @example_species_common = Serializers::SpeciesImagesSerializer.new(@country.all_species).to_json @example_species_threatened = Serializers::SpeciesImagesSerializer.new(@country.all_species, true).to_json - @example_species_select = habitats.reject { |habitat| habitat['data'].nil? }.sort { |h1, h2| h2['data'].last[1] <=> h1['data'].last[1] }. - map { |habitat| { id: habitat[:id], name: habitat[:title] }} + @example_species_select = habitats + .reject { |habitat| habitat['data'].nil? } + .sort { |h1, h2| h2['data'].last[1] <=> h1['data'].last[1] } + .map { |habitat| { id: habitat[:id], name: habitat[:title] } } @habitat_change = Serializers::HabitatCountryChangeSerializer.new(@country, habitats_present_status).serialize @@ -38,15 +40,17 @@ def show @next_country_url = country_link_path(@next_country.actual_name) @next_country_name = @next_country.actual_name @next_country_flag = helpers.if_country_get_flag(@next_country.iso3) + + @red_list_last_updated = Date.parse('2021-08-01').strftime('%b, %Y') # TODO: automate based on date imported end private def next_country - following_countries = GeoEntity.permitted_countries.select { |geo_entity| geo_entity.name > @country.name } - + following_countries = GeoEntity.permitted_countries.select { |geo_entity| geo_entity.name > @country.name } + return GeoEntity.permitted_countries.first if following_countries.blank? - - following_countries.min_by { |geo_entity| geo_entity.name } + + following_countries.min_by(&:name) end end diff --git a/app/controllers/site_controller.rb b/app/controllers/site_controller.rb index ae2dcad9..210c355b 100644 --- a/app/controllers/site_controller.rb +++ b/app/controllers/site_controller.rb @@ -12,7 +12,7 @@ def index red_list_data = Species.count_species @red_list_data = @habitats.each { |habitat| habitat['data'] = red_list_data[habitat[:id]] } - + @habitat_cover = Serializers::HabitatCoverSerializer.new.serialize doughnut_chart = I18n.t('home.sdg.doughnut_chart_data') @@ -20,19 +20,19 @@ def index doughnut_chart.each do |item| @doughnut_chart.push({ - 'title': item[:title], - 'colour': item[:colour], - 'icon': ActionController::Base.helpers.image_url(item[:icon]), - 'description': item[:description], - 'url': item[:url], - 'source': item[:source] - }) + 'title': item[:title], + 'colour': item[:colour], + 'icon': ActionController::Base.helpers.image_url(item[:icon]), + 'description': item[:description], + 'url': item[:url], + 'source': item[:source] + }) end - end - def about + @red_list_last_updated = Date.parse('2021-08-01').strftime('%b, %Y') # TODO: automate based on date imported end - def legal - end + def about; end + + def legal; end end diff --git a/app/models/geo_entity.rb b/app/models/geo_entity.rb index 35068733..303c62f5 100644 --- a/app/models/geo_entity.rb +++ b/app/models/geo_entity.rb @@ -15,7 +15,7 @@ class GeoEntity < ApplicationRecord has_many :country_citations, foreign_key: 'country_id' - scope :countries, -> { where.not(iso3: nil || 'GBL') } + scope :countries, -> { where.not(iso3: nil).where.not(iso3: 'GBL') } scope :regions, -> { where(iso3: nil) } # Only allowing actual countries to be considered for the 'Next country' button diff --git a/app/models/geo_entity_stat.rb b/app/models/geo_entity_stat.rb index 463885a0..6d7c21c8 100644 --- a/app/models/geo_entity_stat.rb +++ b/app/models/geo_entity_stat.rb @@ -4,10 +4,13 @@ class GeoEntityStat < ApplicationRecord has_many :sources, through: :geo_entity_stats_sources has_many :geo_entity_stats_sources, class_name: 'GeoEntityStatsSources', dependent: :destroy - scope :country_stats, -> { joins(:geo_entity).where('geo_entities.iso3 IS NOT NULL') } + scope :country_stats, lambda { + joins(:geo_entity).where.not('geo_entities.iso3': nil) + .where.not('geo_entities.iso3': 'GBL') + } - # TODO - consider adding :present-but-unknown to this list which will require some thinking - enum occurrence: [:absent, :unknown, :present] + # TODO: - consider adding :present-but-unknown to this list which will require some thinking + enum occurrence: %i[absent unknown present] BASE_OCCURRENCES = { 'coralreefs' => 'unknown', @@ -15,5 +18,5 @@ class GeoEntityStat < ApplicationRecord 'mangroves' => 'unknown', 'seagrasses' => 'unknown', 'coldcorals' => 'unknown' - } + }.freeze end diff --git a/app/models/habitat.rb b/app/models/habitat.rb index f3b6ee13..706cabc5 100644 --- a/app/models/habitat.rb +++ b/app/models/habitat.rb @@ -13,39 +13,75 @@ def protected_title end def calculate_country_cover_change(country_name) - country_cover_change = { change_km: 0, change_percentage: 0 } + country_cover_change = { + change_km: 0, + change_percentage: 0 + } + + # We only have mangroves data at the moment + return country_cover_change unless name == 'mangroves' - # We only got mangroves data at the moment - return country_cover_change unless name == "mangroves" geo_entity_id = GeoEntity.find_by(name: country_name).id - habitat_base_year = ChangeStat.find_by(habitat_id: id, geo_entity_id: geo_entity_id)&.send("total_value_#{baseline_year}".to_sym) - habitat_last_year = ChangeStat.find_by(habitat_id: id, geo_entity_id: geo_entity_id)&.send(latest_year) - return country_cover_change if (habitat_base_year.nil? || habitat_last_year.nil?) + habitat_base_year = ChangeStat.find_by( + habitat_id: id, + geo_entity_id: geo_entity_id + ) + &.send("total_value_#{baseline_year}".to_sym) + + habitat_last_year = ChangeStat.find_by( + habitat_id: id, + geo_entity_id: geo_entity_id + ) + &.send(latest_year) + + return country_cover_change if habitat_base_year.nil? || habitat_last_year.nil? + change_km = habitat_last_year - habitat_base_year - change_percentage = (change_km/habitat_base_year) * 100 + change_percentage = (change_km / habitat_base_year) * 100 - country_cover_change.merge!({change_km: ActiveSupport::NumberHelper.number_to_delimited(change_km.round(2)), change_percentage: change_percentage.round(2)}) + country_cover_change.merge!( + { + change_km: ActiveSupport::NumberHelper.number_to_delimited(change_km.round(2)), + change_percentage: change_percentage.round(2) + } + ) end def calculate_global_cover_change - global_cover_change = { change_km: 0, change_percentage: 0, baseline_year: baseline_year, original_total: 0 } - return global_cover_change unless name == "mangroves" - habitat_base_year = ChangeStat.includes(:geo_entity).where.not(geo_entities: { iso3: nil }).where(habitat_id: id).pluck("total_value_#{baseline_year}".to_sym).inject(0) { |sum, x| sum + x } - habitat_last_year = ChangeStat.includes(:geo_entity).where.not(geo_entities: { iso3: nil }).where(habitat_id: id).pluck(latest_year).inject(0) { |sum, x| sum + x } + global_cover_change = { + change_km: 0, + change_percentage: 0, + baseline_year: baseline_year, + original_total: 0 + } + + return global_cover_change unless name == 'mangroves' + + habitat_base_year = ChangeStat.includes(:geo_entity) + .where + .not(geo_entities: { iso3: nil }) + .where(habitat_id: id) + .pluck("total_value_#{baseline_year}".to_sym) + .inject(0) { |sum, x| sum + x } + + habitat_last_year = ChangeStat.includes(:geo_entity) + .where + .not(geo_entities: { iso3: nil }) + .where(habitat_id: id) + .pluck(latest_year) + .inject(0) { |sum, x| sum + x } + total_value_change = habitat_last_year - habitat_base_year total_value_change_percentage = (total_value_change / habitat_base_year) * 100 - global_cover_change.merge!({ - change_km: total_value_change.round(2), change_percentage: total_value_change_percentage.round(2), - baseline_year: baseline_year, original_total: habitat_base_year.round(2) - }) - end - - def global_stats - { - total_habitat_cover: geo_entity_stats.country_stats.pluck(:total_value).compact.reduce(&:+), - protected_habitat_cover: geo_entity_stats.country_stats.pluck(:protected_value).compact.reduce(&:+) - } + global_cover_change.merge!( + { + change_km: total_value_change.round(2), + change_percentage: total_value_change_percentage.round(2), + baseline_year: baseline_year, + original_total: habitat_base_year.round(2) + } + ) end def calculate_global_protection @@ -70,7 +106,7 @@ def total_value_by_country c = Carto.new(name) total_value_by_country = 0 - if name == "coldcorals" + if name == 'coldcorals' total_value_by_country = c.total_points_by_country total_value_by_country = sort_country_count(total_value_by_country) else @@ -81,17 +117,32 @@ def total_value_by_country end def type - name == "coldcorals" ? "points" : "area" + name == 'coldcorals' ? 'points' : 'area' + end + + def global_stats + stats = geo_entity_stats.country_stats.to_a # reduce hits to database + { + total_habitat_cover: stats.pluck(:total_value).compact.reduce(&:+), + protected_habitat_cover: stats.pluck(:protected_value).compact.reduce(&:+) + } end def global_protection - stats = { 'name' => name, 'total_value' => 0, 'protected_value' => 0 } + stats = { + 'name' => name, + 'total_value' => 0, + 'protected_value' => 0 + } + + global_stats_data = global_stats # reduce hits to database - stats['total_value'] = global_stats[:total_habitat_cover] - stats['protected_value'] = global_stats[:protected_habitat_cover] + stats['total_value'] = global_stats_data[:total_habitat_cover] + stats['protected_value'] = global_stats_data[:protected_habitat_cover] - protected_value = stats['protected_value'] > 0 ? stats['protected_value'] : 1 - stats.merge({'protected_percentage' => protected_value / stats['total_value'] * 100}) + protected_value = stats['protected_value'].positive? ? stats['protected_value'] : 1 + + stats.merge({ 'protected_percentage' => protected_value / stats['total_value'] * 100 }) end def self.global_protection @@ -100,21 +151,80 @@ def self.global_protection def self.global_protection_by_id hash = {} - self.global_protection.each do |habitat_stats| + global_protection.each do |habitat_stats| + hash[habitat_stats['name']] = habitat_stats.except('name') + end + hash + end + + # experimental alternative to self.global_protection_by_id: not currently used + def self.global_protection_by_id_v2 + hash = {} + global_protection_v2.each do |habitat_stats| hash[habitat_stats['name']] = habitat_stats.except('name') end hash end + # experimental alternative to self.global_protection: not currently used + # TODO: be brave, use this method instead of the original, it's cool and its + # 2x faster than the original + # TODO: check codebase for how other related methods are used e.g. + # global_stats and global_protection methods. e.g. they are in habitat_spec.rb + def self.global_protection_v2 + geo_entities = GeoEntity.arel_table + geo_entity_stats = GeoEntityStat.arel_table + habitats = Habitat.arel_table + + # get all the data we need in a single query + # @see scuttle.io for sql->arel help + query = GeoEntityStat.select( + [ + habitats[:name], + geo_entity_stats[:total_value].sum.as('total_value'), + geo_entity_stats[:protected_value].sum.as('protected_value') + ] + ) + .where( + geo_entities[:iso3].not_eq(nil).and(geo_entities[:iso3].not_eq('GBL')) + ) + .joins( + geo_entity_stats.join(geo_entities).on( + geo_entities[:id].eq(geo_entity_stats[:geo_entity_id]) + ).join_sources + ) + .joins( + geo_entity_stats.join(habitats).on( + habitats[:id].eq(geo_entity_stats[:habitat_id]) + ).join_sources + ) + .group(habitats[:name]) + + results = ActiveRecord::Base.connection.execute(query.to_sql) + + results.map do |row| + total_value = row['total_value'].to_f + protected_value = row['protected_value'].to_f + protected_percentage = protected_value / total_value * 100 + { + name: row['name'], + total_value: total_value.round(2), + protected_value: protected_value.round(2), + protected_percentage: protected_percentage.round(2) + }.stringify_keys + end + end + private def sum_country_areas(total_area_by_country) country_total_area = {} total_area_by_country.flatten.each do |country_data| - next if country_data["iso3"].include? "/" #remove areas which have multiple iso - next if country_data["iso3"].include? "ABNJ" #remove ABNJ - country_total_area[country_data["iso3"]] ||= 0 - country_total_area[country_data["iso3"]] += country_data["sum"] + next if country_data['iso3'].include? '/' # remove areas which have multiple iso + next if country_data['iso3'].include? 'ABNJ' # remove ABNJ + + country_total_area[country_data['iso3']] ||= 0 + country_total_area[country_data['iso3']] += country_data['sum'] end country_total_area end @@ -122,8 +232,9 @@ def sum_country_areas(total_area_by_country) def sort_country_count(total_value_by_country) country_total_points = {} total_value_by_country.each do |total_value| - next if total_value["iso3"].include? "ABNJ" - country_total_points[total_value["iso3"]] = total_value["count"] + next if total_value['iso3'].include? 'ABNJ' + + country_total_points[total_value['iso3']] = total_value['count'] end country_total_points end diff --git a/app/views/countries/partials/_example-species.html.erb b/app/views/countries/partials/_example-species.html.erb index 87b5d65f..434eb80c 100644 --- a/app/views/countries/partials/_example-species.html.erb +++ b/app/views/countries/partials/_example-species.html.erb @@ -37,7 +37,7 @@ <% end %>

- <%= t('countries.shared.example_species.last_updated').html_safe %> + <%= t('countries.shared.example_species.last_updated', { last_updated: last_updated }).html_safe %>

diff --git a/app/views/countries/partials/_proportion-protected.html.erb b/app/views/countries/partials/_proportion-protected.html.erb index 6927702d..da1bd997 100644 --- a/app/views/countries/partials/_proportion-protected.html.erb +++ b/app/views/countries/partials/_proportion-protected.html.erb @@ -21,7 +21,7 @@

- <%= t('shared.proportion_protected.last_updated').html_safe %> + <%= t('shared.proportion_protected.last_updated', { last_updated: last_updated }).html_safe %>

diff --git a/app/views/countries/partials/_red-list.html.erb b/app/views/countries/partials/_red-list.html.erb index b6c72dd7..72982f62 100644 --- a/app/views/countries/partials/_red-list.html.erb +++ b/app/views/countries/partials/_red-list.html.erb @@ -26,7 +26,7 @@ <% end %>

- <%= t('countries.shared.red_list.smalltext_3').html_safe %> + <%= t('countries.shared.red_list.smalltext_3', { last_updated: last_updated }).html_safe %>

diff --git a/app/views/countries/show.html.erb b/app/views/countries/show.html.erb index 7356e7d0..18e7269d 100644 --- a/app/views/countries/show.html.erb +++ b/app/views/countries/show.html.erb @@ -29,14 +29,16 @@ id: "section-#{t('countries.nav_sticky')[1][:id]}", map_datasets: @map_datasets_habitats, habitats_present: @habitats_present, - modal_content: habitats_protected_modal + modal_content: habitats_protected_modal, + last_updated: @country.updated_at.strftime('%b, %Y') } %> <%= render partial: '/countries/partials/red-list', locals: { id: "section-#{t('countries.nav_sticky')[2][:id]}", - modal_content: country_red_list_modal + modal_content: country_red_list_modal, + last_updated: @red_list_last_updated } %> @@ -46,7 +48,8 @@ example_species_select: @example_species_select, example_species_selected: preferential_selection(@example_species_select), example_species_common: @example_species_common, - example_species_threatened: @example_species_threatened + example_species_threatened: @example_species_threatened, + last_updated: @country.updated_at.strftime('%b, %Y') } %> diff --git a/app/views/site/index.html.erb b/app/views/site/index.html.erb index 02f50484..5d3e8567 100644 --- a/app/views/site/index.html.erb +++ b/app/views/site/index.html.erb @@ -34,7 +34,8 @@ <%= render partial: '/site/partials/red-list', locals: { id: "section-#{t('home.nav_sticky')[3][:id]}", - modal_content: red_list_modal + modal_content: red_list_modal, + last_updated: @red_list_last_updated } %> diff --git a/app/views/site/partials/_red-list.html.erb b/app/views/site/partials/_red-list.html.erb index 2abb284b..d7af2c89 100644 --- a/app/views/site/partials/_red-list.html.erb +++ b/app/views/site/partials/_red-list.html.erb @@ -24,7 +24,7 @@

- <%= t('home.red_list.smallprint_3').html_safe %> + <%= t('home.red_list.smallprint_3', { last_updated: last_updated }).html_safe %>

diff --git a/config/locales/countries/shared/example-species/en.yml b/config/locales/countries/shared/example-species/en.yml index 6d6919be..bc2e4059 100644 --- a/config/locales/countries/shared/example-species/en.yml +++ b/config/locales/countries/shared/example-species/en.yml @@ -9,5 +9,5 @@ en: link_text: IUCN Species page total_species_count: 'Total number of species' smallprint: 'Sources: Species images, where available, were drawn from the World Atlas of Mangroves (Spalding et al., 2010) and the World Atlas of Seagrasses (Green and Short, 2003). Saltmarshes, cold-water corals and warm-water corals do not currently have detailed species images. Species Red List statuses were drawn from the IUCN Red List of Threatened Species, version 2021-1.' - last_updated: 'Last updated: May, 2021' + last_updated: 'Last updated: %{last_updated}' diff --git a/config/locales/countries/shared/proportion_protected/en.yml b/config/locales/countries/shared/proportion_protected/en.yml index 21603eae..8e782fed 100644 --- a/config/locales/countries/shared/proportion_protected/en.yml +++ b/config/locales/countries/shared/proportion_protected/en.yml @@ -8,7 +8,7 @@ en: title_absent: Confirmed Absence title_unknown: Data Deficient note: 'Note: This outlines the proportion of each habitat that is known to occur within protected or conserved areas. However, the level of protection and whether these habitats are explicitly and effectively protected is uncertain.' - last_updated: 'Last updated: May 2021' + last_updated: 'Last updated: %{last_updated}' # NB - these citations come from the OceanPlus_Static_Citations.csv citations: - "UNEP-WCMC, WorldFish Centre, WRI, TNC (2018). Global distribution of warm-water coral reefs, compiled from multiple sources including the Millennium Coral Reef Mapping Project. Version 4.0. Includes contributions from IMaRS-USF and IRD (2005), IMaRS-USF (2005) and Spalding et al. (2001). Cambridge (UK): UN Environment World Conservation Monitoring Centre. URL: http://data.unep-wcmc.org/datasets/1" diff --git a/config/locales/countries/shared/red_list/en.yml b/config/locales/countries/shared/red_list/en.yml index 4df9f7e4..d1cb4156 100644 --- a/config/locales/countries/shared/red_list/en.yml +++ b/config/locales/countries/shared/red_list/en.yml @@ -10,4 +10,4 @@ en: total: Total smalltext_1: 'Disclaimer: hese statuses derive from global assessments, and do not necessarily reflect each species’ status by country. We will endeavour to strengthen alignment with national Red List assessments, where possible. Please contact us at oceanplus@unep-wcmc.org if you would like to provide further information.' smalltext_2: 'Source: Adapted from data available through the IUCN Red List of Threatened Species' - smalltext_3: 'Last updated: May, 2021' + smalltext_3: 'Last updated: %{last_updated}' diff --git a/config/locales/home/red_list/en.yml b/config/locales/home/red_list/en.yml index 23e1dafd..e20f32e1 100644 --- a/config/locales/home/red_list/en.yml +++ b/config/locales/home/red_list/en.yml @@ -10,7 +10,7 @@ en: - IUCN 2021. The IUCN Red List of Threatened Species. Version 2021-1. https://www.iucnredlist.org. smallprint_1: 'Disclaimer: These statuses derive from global assessments, and do not necessarily reflect each species’ status by country.' smallprint_2: 'Source: Data adapted from the IUCN Red List of Threatened Species, version 2021-1.' - smallprint_3: 'Last updated: May 2021' + smallprint_3: 'Last updated: %{last_updated}' categories: - title: Critically endangered diff --git a/config/locales/shared/proportion_protected/en.yml b/config/locales/shared/proportion_protected/en.yml index 17297303..83301ef5 100644 --- a/config/locales/shared/proportion_protected/en.yml +++ b/config/locales/shared/proportion_protected/en.yml @@ -8,6 +8,6 @@ en: title_absent: Confirmed Absence title_unknown: Data Deficient note: 'Note: This outlines the proportion of each habitat that is known to occur within protected areas. However, the level of protection and whether or not these habitats are explicitly and effectively protected is unknown.' - last_updated: 'Last updated: May 2021' + last_updated: 'Last updated: %{last_updated}' card: percentage: '%{percentage}% of %{area} km2' \ No newline at end of file diff --git a/lib/data/habitat_coverage_protection/coldcorals_country_output_old.csv b/lib/data/habitat_coverage_protection/country/coldcorals_country_output_2021-05-01.csv similarity index 100% rename from lib/data/habitat_coverage_protection/coldcorals_country_output_old.csv rename to lib/data/habitat_coverage_protection/country/coldcorals_country_output_2021-05-01.csv diff --git a/lib/data/habitat_coverage_protection/coldcorals_country_output.csv b/lib/data/habitat_coverage_protection/country/coldcorals_country_output_2021-06-01.csv similarity index 100% rename from lib/data/habitat_coverage_protection/coldcorals_country_output.csv rename to lib/data/habitat_coverage_protection/country/coldcorals_country_output_2021-06-01.csv diff --git a/lib/data/habitat_coverage_protection/country/coldcorals_country_output_2021-08-01.csv b/lib/data/habitat_coverage_protection/country/coldcorals_country_output_2021-08-01.csv new file mode 100644 index 00000000..32758a0d --- /dev/null +++ b/lib/data/habitat_coverage_protection/country/coldcorals_country_output_2021-08-01.csv @@ -0,0 +1,134 @@ +"iso3","total_area","protected_area","percent_protected" +"ABNJ",1874.98221419641,421.629372626203,22.4871131807994 +"ABW",3.54881962864481,0,0 +"AGO",3.39567053773998,0,0 +"AIA",3.38364589335382,0,0 +"ALB",5.98647948252716,0,0 +"ARG",61.8263440651753,6.98422602063557,11.2965211290401 +"ASM",0.998440794849897,0,0 +"ATA",18.8890674198804,5.41915114173775,28.6893525301 +"ATF",14.3193147158519,14.3193147158519,100 +"ATG",0.997746538545076,0,0 +"AUS",333.428727950404,122.413497377955,36.7135423904335 +"BEN",0.997746527205306,0,0 +"BES",9.6108429947203,9.6108429947203,100 +"BHS",412.761934786788,32.6122202202251,7.9009757130514 +"BLM",0.997746564489533,0.997746564489533,100 +"BLZ",2.99323973953098,1.99549312134955,66.6666653858546 +"BMU",26.3981335788759,0,0 +"BRA",233.165254915499,1.99549316229226,0.855827838935714 +"BRB",6.37970478883903,0,0 +"CAN",204.280433116829,22.9600988252007,11.239499777284 +"CHL",86.8581116896839,46.2352596925966,53.2307907611213 +"CHN",0.997746571639701,0,0 +"CIV",9.97746576754958,0,0 +"COK",2.3826012700284,2.3826012700284,100 +"COL",56.1477557221478,10.89431447625,19.4029384365094 +"CPV",34.3151290222723,0,0 +"CRI",8.97971910900959,2.99323973787266,33.3333337216468 +"CUB",56.277985296135,0,0 +"CUW",4.9887328907093,0,0 +"CYM",1.99549313274619,0,0 +"CYP",0.998101946907064,0,0 +"DMA",7.3651650429386,0,0 +"DNK",2.99365873047323,0,0 +"DOM",20.0701086865526,17.4317996988043,86.8545356233371 +"DPT",229.89902396618,185.243639172779,80.5760877001503 +"DZA",2.5290385636979,0,0 +"ECU",52.6278347129635,50.6323415510359,96.208293248599 +"ERI",2.74578579390164,0,0 +"ESP",1469.083604719,1257.62536809466,85.6061128212789 +"FJI",2.99323971956311,0,0 +"FRA",239.896192978977,178.91284914885,74.57927819827 +"FRO",287.549263288492,0,0 +"FSM",1.99549317053256,0,0 +"GBR",352.975252551642,202.843836698487,57.4668720348348 +"GHA",7.61553161242364,0,0 +"GIN",0.998146567593262,0.998146567593262,100 +"GLP",9.19477485048201,9.1698670627569,99.7291093242614 +"GNB",0.997746560532099,0,0 +"GNQ",4.39760340599616,3.26008430454921,74.1332040107132 +"GRC",17.0600645525478,0.997746578259555,5.84843378046038 +"GRD",14.9650777683986,0.278272351153,1.85947814945955 +"GRL",18.3319350570199,0,0 +"GTM",0.997746563836964,0,0 +"GUF",5.98647933098521,0,0 +"GUY",15.4553640726882,0,0 +"HMD",1.99549314057558,0.997746560926959,49.9999995309014 +"HND",12.3844405176936,0.997746580674196,8.05645260477223 +"HRV",72.3828884378215,0.997746571876348,1.37842878808772 +"HTI",3.99098626516811,0,0 +"IDN",144.255161624918,3.9909863193429,2.76661595632881 +"IND",10.9752122615403,0,0 +"IRL",474.374394242633,187.917398643326,39.6137314585345 +"ISL",528.361101493694,316.867529204908,59.9717746649238 +"ITA",166.128989497245,45.6543445503437,27.4812630164712 +"JAM",19.5994634664317,0.997746576420656,5.09068311043061 +"JPN",95.2004617816563,9.62331959682966,10.1084799555919 +"KEN",3.99098631850294,0,0 +"KIR",1.99549316032311,0,0 +"KNA",6.51111490902298,0,0 +"LBR",6.93362618358551,0,0 +"LCA",12.3844174162346,1.76116507923105,14.220814916351 +"LKA",3.99098634737099,0,0 +"MAR",152.069298267823,0,0 +"MDG",9.37406314085495,0.997746585254377,10.6436938845217 +"MEX",117.738383967179,45.9973331117806,39.0674065346462 +"MHL",8.97971910029726,0,0 +"MLT",4.34711451271544,3.33789040240339,76.78404589159 +"MMR",5.98647941695625,0,0 +"MNE",1.99549313801732,0,0 +"MOZ",11.9729587680157,0,0 +"MRT",39.6204336641054,0,0 +"MSR",6.96977023996479,0,0 +"MTQ",6.54682430758551,6.54682430758551,100 +"MUS",0.997746579673261,0,0 +"MYS",7.47166374201478,0.100966641788051,1.35132743220621 +"NAM",1.99549315598508,0,0 +"NCL",69.5987054118583,61.6450588823612,88.5721343774565 +"NFK",41.7871319935976,41.6952069307061,99.7800158601322 +"NGA",9.19148440299485,0,0 +"NIC",5.49799999620016,0.422557530579863,7.68565898275566 +"NOR",3017.02068872933,235.201815547869,7.79583038414391 +"NZL",831.974151997951,236.767279429134,28.4584898293472 +"OMN",4.98873283503172,0,0 +"PAN",32.9256369568588,0.121064862520771,0.367691785824518 +"PER",8.97971908075518,0.997746571348176,11.1111111870581 +"PHL",174.352255369976,112.645239820787,64.6078478203528 +"PLW",3.24242093570336,3.24242093570336,100 +"PNG",1.99549314958219,0,0 +"PRI",33.814589651525,6.47154170631325,19.1383121102621 +"PRT",369.793066177503,105.47058183024,28.5215141864271 +"RUS",17.9482763207811,0,0 +"SAU",6.9842260911645,0,0 +"SEN",4.19373899705194,0,0 +"SGS",1.24353842074005,0.16011082865611,12.8754227441421 +"SHN",14.3647412205723,8.28364217293744,57.6664907897826 +"SJM",2.99323969393461,0,0 +"SOM",8.97971912156534,0,0 +"SPM",2.9932397982855,0,0 +"STP",0.997746554697471,0,0 +"SUR",34.4225685734462,0,0 +"SWE",31.0529245436098,19.3690018890172,62.3741633797356 +"SYC",3.44559793005424,0,0 +"TCA",2.99323972468268,0,0 +"THA",3.99098637555205,0,0 +"TLS",6.03496586934398,0,0 +"TON",1.99549321591097,0,0 +"TTO",14.0099836042785,0,0 +"TUN",11.2526376432774,0,0 +"TUV",0.997746564827538,0,0 +"TWN",4.12520435586714,0,0 +"UMI",9.38785685362166,9.38785685362166,100 +"URY",22.2218651326868,0,0 +"USA",2137.24104821964,399.877658986044,18.7099933963532 +"VCT",10.272776718187,0,0 +"VEN",68.0550161904896,1.99549307838309,2.93217633333243 +"VGB",7.98201000193008,0,0 +"VIR",7.98197261387586,0.0425599107840693,0.533200411012225 +"VNM",1.31862966938088,0,0 +"VUT",18.1009321578198,0,0 +"WLF",33.1123345937815,0,0 +"WSM",1.99549315802455,0,0 +"YEM",0.997746569395683,0,0 +"ZAF",44.6205635802445,24.2220065340606,54.2844029535852 diff --git a/lib/data/habitat_coverage_protection/coralreefs_country_output_old.csv b/lib/data/habitat_coverage_protection/country/coralreefs_country_output_2021-05-01.csv similarity index 100% rename from lib/data/habitat_coverage_protection/coralreefs_country_output_old.csv rename to lib/data/habitat_coverage_protection/country/coralreefs_country_output_2021-05-01.csv diff --git a/lib/data/habitat_coverage_protection/coralreefs_country_output.csv b/lib/data/habitat_coverage_protection/country/coralreefs_country_output_2021-06-01.csv similarity index 100% rename from lib/data/habitat_coverage_protection/coralreefs_country_output.csv rename to lib/data/habitat_coverage_protection/country/coralreefs_country_output_2021-06-01.csv diff --git a/lib/data/habitat_coverage_protection/country/coralreefs_country_output_2021-08-01.csv b/lib/data/habitat_coverage_protection/country/coralreefs_country_output_2021-08-01.csv new file mode 100644 index 00000000..1ed55c4d --- /dev/null +++ b/lib/data/habitat_coverage_protection/country/coralreefs_country_output_2021-08-01.csv @@ -0,0 +1,113 @@ +"iso3","total_area","protected_area","percent_protected" +"ABW",66.5257434761535,0.494249113192906,0.742944140669502 +"AIA",24.3337905218337,11.061888498537,45.4589616385972 +"ARE",126.761555170089,59.9976094380869,47.3310771215939 +"ASM",45.1590904217466,12.8077880948817,28.3614837572416 +"ATF",137.509204423328,43.6709723592565,31.7585812109082 +"ATG",54.8946288086163,34.1644476245246,62.2364124979056 +"AUS",31683.868469488,27424.7238569564,86.5573718795319 +"BES",200.532585281889,197.646076492077,98.5605786781464 +"BGD",4.53097443625609,0,0 +"BHR",207.846294391492,0.0000125028931051369,0.00000601545153438572 +"BHS",2226.52344364776,365.619343884382,16.4210866464258 +"BLM",10.7130787461384,10.6083276848866,99.0222132802904 +"BLZ",894.571592598028,368.999393410442,41.2487269284718 +"BMU",528.915295492159,0.0782259351129308,0.0147898795477527 +"BRA",697.508463347012,570.816769287793,81.836536656303 +"BRB",31.2227004063655,0.284625165569483,0.911596889010456 +"BRN",62.736094142188,0,0 +"CCK",115.152796279885,7.12050230933185,6.18352531537758 +"CHN",568.75681592756,0.00000313751090256278,0.000000551643657658142 +"COK",253.267917975378,110.19062107485,43.5075322432122 +"COL",934.807894159329,831.509304825238,88.9497521384341 +"COM",218.976991976775,130.270773249896,59.4906213999475 +"CPT",3.8410151537575,1.93210347260396,50.3018966408884 +"CRI",69.8620497617106,24.5026517879331,35.0729070668669 +"CUB",2649.28141294029,741.759521405158,27.9985175520452 +"CUW",38.6732272155731,5.55119799671472,14.354111090267 +"CXR",5.03441231725536,0.669507706766619,13.2986268222786 +"CYM",136.703454098431,51.3959800696223,37.5966945448325 +"DJI",247.190961159525,0,0 +"DMA",16.3589633919245,0.727007963536417,4.44409554639202 +"DOM",350.604976049933,229.384886688664,65.4254509656461 +"DPT",1959.22408113993,1183.76610137837,60.4201486074844 +"ECU",125.413568309819,125.413568309819,100 +"EGY",2207.10880252002,1157.47481502015,52.4430337869421 +"ERI",1583.18776808474,0,0 +"FJI",3366.66410826214,1172.48941002786,34.8264445850255 +"FSM",3171.83567923052,55.6613510967024,1.75486238020394 +"GLP",113.331772065992,111.919088138377,98.7534970098299 +"GRD",45.4431923245749,2.99456602496455,6.58969115456516 +"GTM",7.06314739238198,5.08667552604251,72.0171227281593 +"GUM",136.868368586213,9.50686020793446,6.94598781744526 +"HND",717.914337311414,243.944372290704,33.9795933320227 +"HTI",325.602719796727,114.121495993399,35.0493067332622 +"IDN",20103.4759073885,6788.4191055991,33.7673899621716 +"IND",2036.5686996224,0,0 +"IRN",117.604362425968,1.72289630901478,1.46499353720773 +"ISR",0.793356959935676,0.605763184239616,76.3544299515227 +"JAM",412.681124524885,72.331114739451,17.5271197156703 +"JOR",2.86070435046009,0.319393218706919,11.1648454219168 +"JPN",1038.59789808939,743.951750309036,71.6303924432749 +"KEN",410.17083033528,124.792828045584,30.4245984395273 +"KHM",47.4636924358226,8.03641425423907,16.9317089375324 +"KIR",1960.04450246774,72.8100194054221,3.71471256462557 +"KNA",41.8207486356309,41.1408408655975,98.3742333836316 +"KWT",72.7455221277148,0,0 +"LCA",29.4758166276421,3.74404106316117,12.7020774706885 +"LKA",108.902758257978,0.989204930757828,0.908337811255904 +"MAF",6.20538653993903,6.18148557370006,99.6148351744869 +"MDG",2401.19262887085,818.073432929061,34.0694629449099 +"MDV",2696.11221556477,119.539344364491,4.43376739567384 +"MEX",1560.48415487727,1089.46073446789,69.8155589124567 +"MHL",1992.45189889501,358.465514298801,17.9911753200969 +"MMR",605.711660810521,38.0741145358433,6.28584803615884 +"MNP",81.4012004309254,11.153187890086,13.7015029643822 +"MOZ",1675.05525056339,162.539866054386,9.70355252459389 +"MSR",2.42337577584969,0,0 +"MTQ",72.0061818384689,71.8422069167881,99.7722766052939 +"MUS",2551.10918368597,1839.17136891162,72.093008824275 +"MYS",2148.12186462979,669.353382409515,31.1599352639555 +"MYT",166.236988064711,165.093311363693,99.3120203185026 +"NCL",4574.81550401555,3618.37344435656,79.0933195268865 +"NIC",460.096382117654,280.52987262467,60.9719796824948 +"NIU",15.3818564031818,0.379728862595909,2.46868032468019 +"NRU",5.91543097709155,0,0 +"OMN",266.191499792615,46.4368681546685,17.4449102209675 +"PAN",630.693599060019,68.3134740214335,10.8314836432853 +"PCN",39.4042857499816,35.3632698545728,89.7447299995518 +"PHL",13485.4703070914,6299.95124493616,46.7165853431401 +"PLW",506.39938482966,506.25325487833,99.971143339485 +"PNG",7255.24898314781,297.170348345126,4.09593590840757 +"PRI",158.139999715516,33.6710879388224,21.2919489056498 +"PYF",3000.33979819127,0.367032616837495,0.0122330349735306 +"QAT",155.314399820548,0.567754192429643,0.365551547754512 +"REU",12.1231525821093,10.1330737157616,83.5844772812269 +"SAU",3419.26971724347,438.95143793615,12.8375786128396 +"SDN",613.065587203154,121.622739084942,19.8384547467088 +"SGP",3.85049928877091,0,0 +"SLB",2804.80799679427,145.125246825435,5.17415976392341 +"SOM",398.799681938048,0,0 +"SXM",1.70293351871973,0.0192108490204567,1.12810328819527 +"SYC",1493.61105483546,1402.4911810269,93.899357298303 +"TCA",191.625656606749,16.9497448753368,8.84523772832819 +"THA",183.931960517407,115.985274368077,63.0587930677227 +"TKL",96.5721017293588,0,0 +"TLS",35.0497470582209,9.55487220181076,27.2608877488879 +"TON",992.045050511086,164.60043727206,16.5920325077233 +"TTO",32.025848022524,4.99522785364437,15.5974881605982 +"TUV",885.565367715921,66.2450348595456,7.48053585591394 +"TWN",375.201257705391,193.245426175168,51.5044718551836 +"TZA",1814.32607411626,833.504853153585,45.9401904125517 +"UMI",172.330311243945,172.330311243945,100 +"USA",4089.96838696817,3573.66325857112,87.3763051557524 +"VCT",38.7063756302732,18.7420668764878,48.421136237383 +"VEN",344.413780196915,186.450531328625,54.1356188541653 +"VGB",137.132429967879,0.576480826087225,0.420382564665599 +"VIR",33.5206997104829,11.6844457275109,34.8574040173059 +"VNM",478.142638619609,105.454052995411,22.0549360123698 +"VUT",706.934537986607,2.40955292919088,0.34084526921735 +"WLF",411.163036132112,0,0 +"WSM",199.995942811373,20.0732116749996,10.0368094436454 +"YEM",652.240702383273,1.65589449712139,0.253877823182575 +"ZAF",1.39881332507099,1.39881332507099,100 diff --git a/lib/data/habitat_coverage_protection/mangroves_country_output_old.csv b/lib/data/habitat_coverage_protection/country/mangroves_country_output_2021-05-01.csv similarity index 100% rename from lib/data/habitat_coverage_protection/mangroves_country_output_old.csv rename to lib/data/habitat_coverage_protection/country/mangroves_country_output_2021-05-01.csv diff --git a/lib/data/habitat_coverage_protection/mangroves_country_output.csv b/lib/data/habitat_coverage_protection/country/mangroves_country_output_2021-06-01.csv similarity index 100% rename from lib/data/habitat_coverage_protection/mangroves_country_output.csv rename to lib/data/habitat_coverage_protection/country/mangroves_country_output_2021-06-01.csv diff --git a/lib/data/habitat_coverage_protection/country/mangroves_country_output_2021-08-01.csv b/lib/data/habitat_coverage_protection/country/mangroves_country_output_2021-08-01.csv new file mode 100644 index 00000000..f1b9025e --- /dev/null +++ b/lib/data/habitat_coverage_protection/country/mangroves_country_output_2021-08-01.csv @@ -0,0 +1,112 @@ +"iso3","total_area_2016","protected_area","percent_protected" +"ABW",0.337893769827033,0.00685633593436759,2.02913949490023 +"AGO",374.912657045239,1.68759753968406,0.450130852605604 +"AIA",0.00867365899617037,0,0 +"ARE",72.6323292602828,11.2192121209939,15.446581756712 +"ASM",0.187447015030157,0,0 +"ATG",8.86302416134686,4.44475367357458,50.1494026492549 +"AUS",9781.88028304306,4833.06462322561,49.4083395357409 +"BEN",1.40189554889003,0,0 +"BES",1.66389964417757,1.66386676809687,99.9980241548332 +"BGD",4087.12416951906,3752.02959574617,91.8012137661009 +"BHR",0.808425681791958,0.0000125028931051369,0.00154657297346405 +"BHS",969.685321945961,629.313506306957,64.8987348848442 +"BLZ",482.212511262747,157.684986012801,32.7003099940064 +"BRA",10889.8320636586,9402.37059945055,86.3408227462759 +"BRB",0.136871415779438,0,0 +"BRN",113.324340580626,66.7180978889676,58.8735813922517 +"CHN",154.711074653657,10.3484995025761,6.68891966896534 +"CIV",57.7752263873211,4.35936208979259,7.5453829649541 +"CMR",1886.35864892368,1412.96361776458,74.9042934423311 +"COD",255.321092049895,251.144121772571,98.3640324252931 +"COL",2286.66049426079,1067.076477774,46.6652780529608 +"COM",1.04065525060988,0.953654961566152,91.639854890201 +"CRI",361.864426855028,178.553419488889,49.3426284094021 +"CUB",3343.58905781116,1712.0240111793,51.2031826153916 +"CUW",0.0927754125616472,0.03897327458612,42.0081932378616 +"CYM",41.5475025626803,9.25599929652207,22.2781123427529 +"DJI",4.9221930213001,0,0 +"DMA",0.0169957272156196,0.0169957272156196,100 +"DOM",188.147111527773,153.506080232963,81.5883268079317 +"DPT",1.23342702795382,0.858934000488808,69.6380070342492 +"ECU",1379.93417855303,674.012322596956,48.8438023401746 +"EGY",0.983920030073875,0.478465445645617,48.6284892085887 +"ERI",69.8810278047023,0,0 +"FJI",508.850640351845,168.148449693567,33.0447554467654 +"FSM",83.5067583366215,0,0 +"GAB",1746.31123227894,916.042145701946,52.4558354072148 +"GHA",204.235078963438,101.060264489662,49.4823244873395 +"GIN",2209.43003995291,562.816026505302,25.4733581207802 +"GLP",37.0583347124867,35.8941267491431,96.858445010074 +"GMB",597.963203243546,266.382815484905,44.5483625146093 +"GNB",2587.30349761381,965.481448838035,37.3161266054975 +"GNQ",256.59030490496,204.002873442966,79.5052928903638 +"GRD",1.13842574629655,0.335079473957372,29.4335818605148 +"GTM",239.540960102206,111.332886710398,46.4775989304272 +"GUF",500.674797090793,355.74857962599,71.0538221003121 +"GUY",282.60847245255,146.454327517662,51.8223414346688 +"HKG",1.96470556509111,1.13610749340541,57.8258398404205 +"HND",590.176900328398,390.910678298905,66.2361875026602 +"HTI",146.987185557978,67.5439212298656,45.952251533671 +"IDN",26504.9067708686,6475.92589218146,24.4329321667304 +"IND",3488.69339904678,716.7985918624,20.5463338239541 +"IRN",77.9639793037399,76.6106547156143,98.2641668624261 +"JAM",95.3388276176158,54.4928389018161,57.1570264324789 +"JPN",9.81601723532788,7.3675111588001,75.0560128631844 +"KEN",529.166695235685,84.8905457006611,16.0423069828406 +"KHM",585.753360326428,393.880633419294,67.243427028706 +"KNA",0.288188935547014,0.0011635686761729,0.40375202953726 +"LBR",189.218822516554,21.8564777258492,11.5509003994236 +"LCA",1.65699083821273,1.08017088061554,65.1887056768905 +"LKA",197.173612452624,9.4242054402154,4.77964841389707 +"MAF",0.156224122763322,0.153465992393209,98.2345041717459 +"MDG",2585.11646254292,465.230651835273,17.9965064853456 +"MEX",9661.06565866419,7276.02700456782,75.3128822599666 +"MMR",4953.0462909467,150.536114410037,3.03926322443605 +"MOZ",2985.52070148794,1639.50463498831,54.9151990194275 +"MRT",0.833671509111607,0.404355918626622,48.5030271764378 +"MTQ",16.9270319334204,16.7674222397742,99.0570721773671 +"MUS",7.11337394246802,0.644937754415433,9.06655209794395 +"MYS",5092.22754601493,728.607438228297,14.308226245673 +"MYT",5.84028681307456,5.61735637027014,96.1828853626614 +"NCL",294.69534932015,91.703653294858,31.1181202914856 +"NGA",6881.38508430229,370.398471215843,5.38261507935066 +"NIC",731.399359831335,576.373709926278,78.804240416507 +"NZL",310.312771006744,23.4913057648997,7.57020269861508 +"OMN",1.14993793746886,0.536672927288989,46.6697297134369 +"PAK",641.482281647333,230.745527040935,35.9706781687528 +"PAN",1520.75185668653,631.023288919591,41.4941652804876 +"PER",40.2003447049125,15.889111996273,39.5248153042113 +"PHL",2674.55476877824,2021.06110356148,75.5662634826028 +"PLW",61.4039182818926,50.8194177508308,82.7624998090992 +"PNG",4668.85744611161,209.547877942491,4.48820467022419 +"PRI",88.0497344447596,44.4459121865861,50.4781899307947 +"QAT",4.42535412368615,3.98236548151412,89.989758338186 +"SAU",64.9128211255508,7.50169042087165,11.5565620023852 +"SDN",3.70890663886185,0.116712856865263,3.14682649712313 +"SEN",1246.32774142036,728.084108225575,58.4183504890797 +"SGP",5.18777607976631,0.470087676364917,9.06144885856547 +"SLB",527.672506215791,54.7169843976715,10.3694969423506 +"SLE",1264.32655624945,560.196155385083,44.307869087783 +"SLV",375.289221901237,288.175550541099,76.7875904032589 +"SOM",21.1614854300656,0,0 +"STP",0.00668625157058002,0,0 +"SUR",763.19035505868,528.724117666259,69.278144588922 +"SYC",1.11487858265853,0.401947285322408,36.0530098590582 +"TCA",117.838463837769,92.1759671203486,78.2223088441223 +"THA",2246.97268015352,154.730421125088,6.88617278223946 +"TLS",9.81902953810036,5.51689352338759,56.1857310030551 +"TON",8.65103563169179,1.88227443068215,21.7577930645287 +"TTO",52.7671959720383,17.6015448417371,33.356983477129 +"TWN",1.69335771588026,1.29661491934906,76.5706446540764 +"TZA",1114.34177236526,938.721424548354,84.2399924177537 +"USA",1879.20925713601,1692.5245701751,90.065785050175 +"VCT",1.11025620238207,0.509244834223061,45.8673262198823 +"VEN",2799.11412792065,1883.40348975871,67.2856983919344 +"VGB",0.883081766045256,0,0 +"VIR",2.04965381491605,1.31869677504541,64.3375366829653 +"VNM",1580.41802585484,114.481687703237,7.2437599312571 +"VUT",17.636071456825,0,0 +"WSM",3.04012699243989,0.594677565883473,19.560944900075 +"YEM",16.2628149193848,2.41755243939471,14.8655226747557 +"ZAF",24.9221119576282,8.61156014023855,34.5538939672515 diff --git a/lib/data/habitat_coverage_protection/saltmarshes_country_output_old.csv b/lib/data/habitat_coverage_protection/country/saltmarshes_country_output_2021-05-01.csv similarity index 100% rename from lib/data/habitat_coverage_protection/saltmarshes_country_output_old.csv rename to lib/data/habitat_coverage_protection/country/saltmarshes_country_output_2021-05-01.csv diff --git a/lib/data/habitat_coverage_protection/saltmarshes_country_output.csv b/lib/data/habitat_coverage_protection/country/saltmarshes_country_output_2021-06-01.csv similarity index 100% rename from lib/data/habitat_coverage_protection/saltmarshes_country_output.csv rename to lib/data/habitat_coverage_protection/country/saltmarshes_country_output_2021-06-01.csv diff --git a/lib/data/habitat_coverage_protection/country/saltmarshes_country_output_2021-08-01.csv b/lib/data/habitat_coverage_protection/country/saltmarshes_country_output_2021-08-01.csv new file mode 100644 index 00000000..0d413735 --- /dev/null +++ b/lib/data/habitat_coverage_protection/country/saltmarshes_country_output_2021-08-01.csv @@ -0,0 +1,107 @@ +"iso3","total_area","protected_area","percent_protected" +"AGO",4.1149203726672,0.997746601119674,24.2470451614829 +"ALA",0.758065181942218,0.0223359849498313,2.94644649060453 +"ALB",54.5599588089528,46.2691076755062,84.8041470073724 +"ARE",55.5296555220255,10.4604462768005,18.83758539192 +"ARG",4761.72677014368,2018.00084484683,42.3796018179754 +"ASM",1.99672928573158,0.416946293028692,20.8814633014173 +"ATG",25.9888750497518,22.8824971535496,88.0472783440778 +"AUS",13186.5854733994,4233.86136271241,32.1073364386342 +"AZE",0.997746551912462,0,0 +"BEL",4.40658084853609,3.6369964314632,82.5355657021804 +"BGR",1.31334207952052,0.315595516547506,24.0299554448697 +"BHR",18.825165564829,0.0000250057560612897,0.000132831533274841 +"BHS",325.608363124706,290.799712520345,89.3096570768886 +"BIH",25.5796968205445,2.90105674883903,11.3412475886306 +"BRA",2906.19188544046,959.09195568696,33.0016734439269 +"CAN",68200.0819304657,51092.6446826069,74.9158112957974 +"CHL",656.328812728532,380.463248100537,57.9683903436832 +"CHN",5419.02276794656,243.428453396519,4.49210981058051 +"COK",0.997746574117134,0.0512651389028184,5.1380922002344 +"CRI",5.99285380428272,4.37318783012831,72.9733775084428 +"CUB",7006.41806381695,6247.17944365462,89.1636694635274 +"CYP",19.5672692581967,18.445392559763,94.2665648250137 +"DEU",203.070924334074,199.638514517144,98.3097482674165 +"DJI",0.997746615863708,0,0 +"DNK",304.35344656917,292.344979565201,96.0544337055044 +"DPT",2.20711201408454,1.57327617365865,71.2821172472847 +"DZA",1.99549318096299,1.99549318096299,100 +"ECU",647.623192486668,283.126891384029,43.7178431329662 +"EGY",54.9727062852492,21.5683987515263,39.2347406722338 +"ESH",0.844751589802631,0,0 +"ESP",883.581568353256,850.033364981365,96.2031571760358 +"EST",224.049755459842,208.053199013593,92.8602660541104 +"FIN",132.473784150712,87.679333508208,66.1861771899393 +"FJI",1293.16442383697,160.821479157896,12.436274629387 +"FRA",800.795474470054,770.84125140397,96.2594415152125 +"GBR",562.145102424401,520.383766672782,92.5710754089091 +"GEO",5.99285533754899,0.997746608475603,16.6489353117552 +"GHA",99.9718050598989,0,0 +"GMB",199.970134859195,141.675904269983,70.8485316418581 +"GRC",139.963577422363,111.556040394523,79.7036217914636 +"GRL",3912.02346593456,3585.91323871017,91.6638990010127 +"GUM",0.0799664104189736,0.0071861817836339,8.98650038932951 +"HKG",0.406171444268296,0,0 +"HND",63.2926360295864,44.324921266749,70.0317194026002 +"HRV",94.7787316824984,90.6896734555229,95.685679524945 +"IND",1671.31456030351,4.22141242185356,0.252580365307591 +"IRL",99.0772220269396,87.9841144217723,88.803574244188 +"IRN",18260.2002586378,3577.23063098187,19.5903143465785 +"IRQ",6599.20091699695,93.7330257145621,1.4203693279461 +"ISL",43.2733328494041,11.3714023145026,26.278082980288 +"ITA",439.330971322335,415.500159297889,94.5756585399117 +"JPN",8.97971914745799,5.79177809321453,64.4984325022468 +"KEN",1.99549315523612,0,0 +"KHM",0.224005694915426,0,0 +"KOR",224.946095698903,32.9308196707858,14.6394270896192 +"KWT",11298.539567458,1296.76979656158,11.4773222576175 +"LBY",1.99549318826622,0,0 +"LKA",77.9581897419449,27.9297927070169,35.8266306586509 +"LTU",0.997746588514625,0.997746588514625,100 +"LVA",8.69488754116547,7.75612313920853,89.2032599902827 +"MAC",1.00019403411015,0,0 +"MAR",66.9772220158362,66.9772220158362,100 +"MDG",57.8254575569581,36.2673909264801,62.7187271121143 +"MEX",15313.8077049954,11411.1351580243,74.5153352964066 +"MLT",0.997746560380426,0.63971417991643,64.1158992993688 +"MNE",1.05026221431482,1.00544716088894,95.7329652714288 +"MNP",0.00161727442105768,0.000976257153091943,60.3643475949789 +"MRT",12133.4739510172,2453.47953032382,20.2207507942779 +"NAM",210.001163308001,142.708619541206,67.9561090487392 +"NCL",0.997746576236889,0,0 +"NIC",753.057928004391,476.798593125415,63.314995486328 +"NLD",151.502737116509,147.321871658186,97.2404026898155 +"NOR",48.9701862272995,21.1154170430783,43.1189233078862 +"NZL",196.284991111205,46.9793642550326,23.9342621099422 +"OMN",1.21354406908681,0.997746597181646,82.217582582926 +"PAK",4001.05317745601,3630.53237403778,90.7394181735467 +"PAN",220.964892508235,87.8134211868763,39.7408928586263 +"PER",289.643978234784,47.613377373573,16.4385870073148 +"PHL",1.99549315169896,1.99549315169896,100 +"POL",8.89058028986526,7.60049280434893,85.4892769261985 +"PRI",58.4387258259773,12.9139516523662,22.0982772465338 +"PRK",6.84620989693073,0,0 +"PRT",198.935384038407,184.660769104995,92.8244967568689 +"QAT",3.15292255692798,0.997746562330184,31.6451338183938 +"ROU",54.7215575295328,54.7215575295328,100 +"RUS",9405.84974334912,3326.67668788232,35.3681674559454 +"SAU",420.574017110933,0,0 +"SDN",0.997746578501967,0,0 +"SOM",0.99774657513536,0,0 +"SUR",119.977888036083,119.977888036083,100 +"SVN",3.20913385014855,1.06679732392191,33.2425312790404 +"SWE",355.298512876604,197.632065425066,55.6242309670753 +"TON",2.99472705303861,0.0919033601798586,3.06883928158356 +"TUN",1.99549316768183,0.997746609984215,50.0000013101172 +"TUR",256.178074276283,143.207205181771,55.9014293422023 +"TWN",47.0473490974593,0.82591826828883,1.75550436769121 +"TZA",5967.29902556908,3601.8914213888,60.3604982079023 +"UKR",2.92098603646244,0.925492881184739,31.6842624248074 +"URY",303.75936834445,197.931510120904,65.160627374119 +"USA",18786.3016023926,6099.39971297664,32.4672723885143 +"VEN",2583.0095308693,378.303259333964,14.6458328865185 +"VIR",1.26391901912234,0.412619837232182,32.6460660049807 +"VNM",301.194505284933,0.9977465770424,0.331263206843206 +"VUT",1.99549320695315,0,0 +"YEM",0.997746554760933,0,0 +"ZAF",1041.33114793247,220.143561329218,21.140591229438 diff --git a/lib/data/habitat_coverage_protection/seagrasses_country_output_old.csv b/lib/data/habitat_coverage_protection/country/seagrasses_country_output_2021-05-01.csv similarity index 100% rename from lib/data/habitat_coverage_protection/seagrasses_country_output_old.csv rename to lib/data/habitat_coverage_protection/country/seagrasses_country_output_2021-05-01.csv diff --git a/lib/data/habitat_coverage_protection/seagrasses_country_output.csv b/lib/data/habitat_coverage_protection/country/seagrasses_country_output_2021-06-01.csv similarity index 100% rename from lib/data/habitat_coverage_protection/seagrasses_country_output.csv rename to lib/data/habitat_coverage_protection/country/seagrasses_country_output_2021-06-01.csv diff --git a/lib/data/habitat_coverage_protection/country/seagrasses_country_output_2021-08-01.csv b/lib/data/habitat_coverage_protection/country/seagrasses_country_output_2021-08-01.csv new file mode 100644 index 00000000..37c4e5e1 --- /dev/null +++ b/lib/data/habitat_coverage_protection/country/seagrasses_country_output_2021-08-01.csv @@ -0,0 +1,165 @@ +"iso3","total_area","protected_area","percent_protected" +"ABNJ",6335.72743951258,0,0 +"ABW",33.086922981754,0,0 +"AGO",592.857057860186,0,0 +"AIA",15.0102685292499,1.0747075355794,7.1598155188574 +"ALA",0.0000124106493716223,0,0 +"ALB",44.0192816483413,2.93708153556099,6.67226139450565 +"ARE",1021.68240229299,441.895138006581,43.2517127646346 +"ASM",461.078724953926,17.5012300588758,3.79571407477642 +"ATF",0.00000310267769422565,0,0 +"ATG",249.635488122907,53.2690301307572,21.3387249270146 +"AUS",41008.9779166146,16265.0027792817,39.6620535443581 +"AZE",0.0000620531952133495,0,0 +"BEN",1342.7616814721,0,0 +"BES",4.5123039894294,4.50516861977465,99.8418685959222 +"BGD",0.00000310266441194763,0,0 +"BHR",535.748558798727,0.0000625130716955644,0.0000116683602165414 +"BHS",2256.9561788181,69.8262751128324,3.09382502718322 +"BLM",16.8932512432228,16.398876391915,97.0735363833166 +"BLZ",4901.11585178532,2131.09709250445,43.4818754942951 +"BMU",0.0000124108604061004,0,0 +"BRA",426.481911738778,0.980516292365416,0.229908060664947 +"BRB",89.6293604461783,0,0 +"CAN",301.909141054607,70.0127912039411,23.190020335051 +"CCK",215.180925785894,0,0 +"CHL",0.00000310266053187421,0,0 +"CHN",8029.67976685395,0.316751675119956,0.00394476099068717 +"COL",430.801143928378,189.241337525522,43.9277704325186 +"COM",1265.15003764605,56.6372521354687,4.47672216339246 +"CRI",0.000018616073470288,0.0000155133985561554,83.3333548071527 +"CUB",13906.0575742611,6029.74849054022,43.3605891413881 +"CUW",0.0000403345196719117,0.00000930796225040554,23.0769135869677 +"CXR",158.467423183808,11.3010272244458,7.13145137176719 +"CYM",44.9845224406935,3.05191469356327,6.78436610633544 +"CYP",69.8779803955572,45.5414952095501,65.1728841499913 +"DEU",317.308584883037,317.159627575023,99.9530560107385 +"DMA",513.932058426165,6.60972052279404,1.28610784527341 +"DNK",1182.11135711848,624.757164190439,52.8509569278944 +"DOM",346.692193069346,86.0475941506622,24.8195938272688 +"DPT",7190.43210658938,3232.48120532784,44.9553122456377 +"DZA",40.4832019843335,0.0246895200738721,0.0609870733135849 +"EGY",2438.73178917872,470.022334876497,19.2732278704082 +"ERI",16.3334579223989,0,0 +"ESH",0.00000310260420641431,0,0 +"ESP",3100.36182367319,2051.52301668038,66.1704385925451 +"EST",0.0000310266028657269,0.0000248213312549092,80.000157807569 +"FIN",0.000179954461033304,0.0000465401442519139,25.8621786782495 +"FJI",1736.29059953449,288.132036121437,16.5946896330998 +"FRA",1094.50654842483,1057.25695414185,96.5966768918294 +"FSM",1584.22135036628,0,0 +"GBR",149.274984250854,122.842503637147,82.2927594021462 +"GGY",0.000012410597255493,0,0 +"GHA",2713.71099027373,0.0000125026134211153,0.000000460720152806477 +"GIN",12809.5967374729,533.248943312363,4.16288626598533 +"GLP",758.386403795894,742.848749918019,97.9512219891989 +"GNB",15318.2986187856,2341.02701723416,15.2825524263069 +"GRC",626.43256980222,570.902991406077,91.1355856842382 +"GRD",9.9175033947353,0.69375517799722,6.9952603027643 +"GRL",0.00000930797129828644,0,0 +"GTM",0.183243540221459,0.130607526898978,71.2753785160082 +"GUM",473.274787649748,39.4829588842905,8.34250205474926 +"HKG",45.3679687557244,5.72779045402617,12.6251860312866 +"HND",2739.15446725393,731.868551522999,26.7187761870441 +"HRV",299.617704872247,171.77169426702,57.3302883887523 +"HTI",748.630282120462,10.8896273286377,1.45460684515637 +"IDN",17597.0831463419,3548.05553424727,20.1627480232987 +"IMN",0.000065154858614612,0.000065154858614612,100 +"IND",4229.40928328594,0.00102408768024271,0.0000242134920422521 +"IRL",3.65161555032424,3.64957672952182,99.944166608606 +"IRN",2704.38380957145,290.700718390694,10.7492404503324 +"IRQ",74.8788273838901,0,0 +"ISL",0.0000899774390557103,0.0000372320956844125,41.3793680673218 +"ISR",17.3956693790278,1.93963485364609,11.150101852272 +"ITA",3724.98560816091,1842.90049751798,49.4740300064635 +"JAM",791.696202342602,132.762568248298,16.7693829849705 +"JEY",0.0000403343028342647,0.0000403343028342647,100 +"JOR",1.37053976968828,0,0 +"JPN",473.732775567186,405.092717375302,85.5108065702857 +"KAZ",0.0000744645401148926,0.0000310267274066077,41.6664460140841 +"KEN",116.061684166683,72.6126468133214,62.563840370469 +"KHM",0.000018615799717274,0.00000620524187402819,33.3332006589554 +"KIR",496.23419105688,0,0 +"KNA",60.0867909656262,52.0710341548048,86.6597022706588 +"KOR",0.000319573859459994,0.0000930800055146868,29.1262888873234 +"KWT",1584.36892441357,192.309088388201,12.1378982776616 +"LBN",0.00000930798099701388,0,0 +"LBY",11.5167847063231,0,0 +"LCA",6.61364546350088,4.27094963592619,64.5778437851943 +"LKA",2867.85284295957,4.04768272879596,0.141139833542464 +"LTU",0.00000620528125591186,0.00000310263537378552,49.9999153274413 +"MAF",48.6259551574376,47.0290514467941,96.715943768152 +"MAR",0.0000124107722984007,0.00000620540370592772,50.0001414636169 +"MCO",0.42329230123052,0.423289198544337,99.9992670109581 +"MDG",5764.78344382922,1181.88985058105,20.5018950338225 +"MDV",2400.96702981869,25.5042970350148,1.06225103128304 +"MEX",9758.42351559485,5913.05353224947,60.5943523848896 +"MHL",525.835650366179,2.00398346855514,0.381104527081725 +"MLT",70.4110826708552,64.4070611952355,91.4729027762771 +"MMR",2933.5499800565,9.65628118394353,0.329167092757613 +"MNE",0.00000620546878801474,0,0 +"MNP",0.000012410489345373,0.00000310263193427738,25.0000773372738 +"MOZ",689.833679793626,648.951373518893,94.0735995541761 +"MRT",0.0000186158445684163,0.0000155132159321901,83.3333984669699 +"MSR",12.9999236078229,0,0 +"MTQ",548.774037481027,541.992549021869,98.7642475780584 +"MUS",132.305605833693,0.0000434372776133905,0.0000328310182623636 +"MYS",540.556781995437,98.869254102542,18.290262447096 +"MYT",161.8689675642,148.490142048655,91.7347804728295 +"NCL",1469.29883408473,1173.61032117045,79.8755361363586 +"NFK",0.0000031026923295338,0,0 +"NGA",8829.3557727832,1.14823433879382,0.0130047352076727 +"NIC",5532.42380860499,914.113880974135,16.5228462713277 +"NLD",1.12254186395004,1.12253565863267,99.9994472083791 +"NOR",62.2784176874285,11.984089994415,19.2427656954331 +"NZL",0.000158235875809673,0.0000217184176713694,13.7253436113897 +"OMN",0.0000496431653478609,0.00000620542476706727,12.5000586154901 +"PAN",1791.34520669683,91.3669457641541,5.10046558433208 +"PHL",14945.9847185179,11437.6040389883,76.526266113582 +"PLW",727.432844946245,720.342944408539,99.025353256048 +"PNG",9226.26538898792,1107.79264018027,12.0069453183353 +"POL",2.65949331985595,2.65948711445082,99.9997666696479 +"PRI",600.654599954727,123.386355745485,20.5419813241728 +"PRK",0.0000372317705650399,0,0 +"PRT",0.0380474615849392,0.00172138425379833,4.52430775166279 +"PYF",785.198894938157,0,0 +"QAT",1147.19497644687,87.7127495484882,7.64584498270335 +"REU",0.00000310269003275122,0,0 +"ROU",0.0000155132756476039,0.0000124105988073282,79.9998600504792 +"RUS",680.042558958503,15.4174394456621,2.26712861460821 +"SAU",20983.3620261165,331.531696706045,1.57997415425332 +"SDN",4439.92728983865,0.0112882531088483,0.000254244098426633 +"SEN",1482.05849404729,482.295935197991,32.5423009371857 +"SGP",127.049405533377,0,0 +"SLB",1253.88796907809,167.169076270899,13.332058397036 +"SLE",4411.13243417548,1721.87260347468,39.0347066012887 +"SOM",0.00000930805412606787,0,0 +"STP",0.00000310257128422728,0,0 +"SUR",0.00000620529040585083,0.00000310262899132227,49.9997387454561 +"SVN",0.0851023784746948,0.0000451089387207022,0.0530054970603616 +"SWE",33.8793118855152,18.8201937311653,55.5506965276105 +"SXM",24.1557061015748,0.723838073025293,2.9965510839615 +"SYC",0.0000558478117101556,0.0000186159873687425,33.3334230987555 +"SYR",0.0000465404981237533,0,0 +"TCA",951.957897856821,66.3289744309076,6.96763739029179 +"TGO",189.82220659268,0,0 +"THA",1822.15023371682,1408.50213576259,77.2989026755233 +"TKM",0.0000248212553036939,0.0000031026673381412,12.5000420010162 +"TLS",334.783636380886,4.00572570945793,1.19651179871306 +"TON",3684.5890575222,184.701268590584,5.01280511088505 +"TTO",1.11733615031514,0.582254173558644,52.1109223392104 +"TUN",4988.24578570717,122.90224092209,2.46383691185069 +"TUR",0.26010258273805,0.00000310271928752771,0.00119288292137125 +"TWN",1241.93970954223,136.98976703692,11.030307347803 +"TZA",45.7691502419777,1.6058955593716,3.50868554666488 +"UKR",155.695511020712,93.5768635495279,60.1024801139448 +"USA",13940.229690357,8524.27008147815,61.1487060889297 +"VCT",0.311822713813302,0.0403967156026403,12.9550266267091 +"VEN",1286.25383041703,98.4335056401584,7.65272789183795 +"VGB",40.7416806205067,0.111369109103452,0.273354234305681 +"VIR",83.7528133328088,25.8673764070837,30.8853820877568 +"VNM",233.615874706196,22.9215079465004,9.81162259428101 +"VUT",1237.73314347656,3.17994359592202,0.256916736267573 +"WSM",982.87233273973,47.3731599990906,4.81986911433748 +"YEM",5455.48189980982,0.00000620521784342261,0.000000113742799579977 +"ZAF",405.565779341976,405.565731809458,99.9999882799485 diff --git a/lib/data/habitat_coverage_protection/coldcorals_regions_output.csv b/lib/data/habitat_coverage_protection/regions/coldcorals_regions_output_2020-06-01.csv similarity index 100% rename from lib/data/habitat_coverage_protection/coldcorals_regions_output.csv rename to lib/data/habitat_coverage_protection/regions/coldcorals_regions_output_2020-06-01.csv diff --git a/lib/data/habitat_coverage_protection/coralreefs_regions_output.csv b/lib/data/habitat_coverage_protection/regions/coralreefs_regions_output_2020-06-01.csv similarity index 100% rename from lib/data/habitat_coverage_protection/coralreefs_regions_output.csv rename to lib/data/habitat_coverage_protection/regions/coralreefs_regions_output_2020-06-01.csv diff --git a/lib/data/habitat_coverage_protection/mangroves_regions_output.csv b/lib/data/habitat_coverage_protection/regions/mangroves_regions_output_2020-06-01.csv similarity index 100% rename from lib/data/habitat_coverage_protection/mangroves_regions_output.csv rename to lib/data/habitat_coverage_protection/regions/mangroves_regions_output_2020-06-01.csv diff --git a/lib/data/habitat_coverage_protection/saltmarshes_regions_output.csv b/lib/data/habitat_coverage_protection/regions/saltmarshes_regions_output_2020-06-01.csv similarity index 100% rename from lib/data/habitat_coverage_protection/saltmarshes_regions_output.csv rename to lib/data/habitat_coverage_protection/regions/saltmarshes_regions_output_2020-06-01.csv diff --git a/lib/data/habitat_coverage_protection/seagrasses_regions_output.csv b/lib/data/habitat_coverage_protection/regions/seagrasses_regions_output_2020-06-01.csv similarity index 100% rename from lib/data/habitat_coverage_protection/seagrasses_regions_output.csv rename to lib/data/habitat_coverage_protection/regions/seagrasses_regions_output_2020-06-01.csv diff --git a/lib/modules/esri.rb b/lib/modules/esri.rb index 49e0ad0c..bc3cfcee 100644 --- a/lib/modules/esri.rb +++ b/lib/modules/esri.rb @@ -1,4 +1,4 @@ -class Esri +class Esri include HTTParty def initialize @@ -14,12 +14,14 @@ def fetch_bounding_box(iso3) # ESRI is flaky begin response = self.class.post(@url, query: options) - rescue Net::OpenTimeout + rescue Net::OpenTimeout retries += 1 retry if retries < 2 end # Response returned includes a 'spatialReference' key for the coordinates JSON.parse(response.parsed_response)['extent'].except('spatialReference') + rescue StandardError + raise "Unexpected response: #{response.parsed_response}" end -end \ No newline at end of file +end diff --git a/lib/modules/utilities/files.rb b/lib/modules/utilities/files.rb new file mode 100644 index 00000000..5c5e824b --- /dev/null +++ b/lib/modules/utilities/files.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Utilities + module Files + # + # Get the latest file based on a date within the filename from a directory using a glob + # + # E.g files: + # + # /foo/bar/baz_file_2021-04-01.csv + # /foo/bar/baz_file_2021-05-01.csv + # /foo/bar/baz_file_2021-06-01.csv + # /foo/bar/baz_file_2021-07-01.csv + # /foo/bar/baz_file_2021-08-01.csv + # + # If we want to automatically detect the latest file "baz_file_2021-08-01.csv", + # we will use the following glob: + # + # "foo/bar/baz_file_*.csv" + # + # The file returned will be the absolute path to the latest file. + def self.latest_file_by_glob(glob) + Dir.glob(Rails.root.join(glob)).max_by do |file| + Date.parse(file) + end + end + end +end diff --git a/lib/tasks/import_bounding_boxes.rake b/lib/tasks/import_bounding_boxes.rake index 4a8efcda..6e54c520 100644 --- a/lib/tasks/import_bounding_boxes.rake +++ b/lib/tasks/import_bounding_boxes.rake @@ -1,8 +1,8 @@ require 'csv' namespace :import do - desc "Import bounding boxes for countries and regions" - task :bounding_boxes => [:environment] do + desc 'Import bounding boxes for countries and regions' + task bounding_boxes: [:environment] do import_bounding_boxes_for('regionalseas') import_bounding_boxes_from_esri @@ -12,7 +12,9 @@ namespace :import do filename = "lib/data/#{geo_entity_type}.csv" CSV.foreach(filename, headers: true) do |row| - name, iso3, bounding_box = [row['country_name'] || row['region'], row['alpha-3'], row['bounding-box']] + name = row['country_name'] || row['region'] + iso3 = row['alpha-3'] + bounding_box = row['bounding-box'] if bounding_box.present? && bounding_box.include?(';') geo_entity = GeoEntity.find_by(name: name, iso3: iso3) bounding_box = bounding_box.split(';').map { |bb| bb.split(' ').map(&:to_f) } @@ -22,7 +24,7 @@ namespace :import do end def import_bounding_boxes_from_esri - filename = "lib/data/habitat_presence/habitat_presence_country.csv" + filename = 'lib/data/habitat_presence/habitat_presence_country.csv' CSV.foreach(filename, headers: true) do |row| iso3 = row['ISO3'] @@ -30,7 +32,7 @@ namespace :import do geo_entity = GeoEntity.find_by(iso3: iso3) bounding_box = Esri.new.fetch_bounding_box(iso3) - # ESRI's parsed response still looks like + # ESRI's parsed response still looks like # "extent": { # "xmin": -141.0068664547689, # "ymin": 40.051148997526347, @@ -40,7 +42,7 @@ namespace :import do formatted_bbox = format_bbox(bounding_box) geo_entity.update_attributes!(bounding_box: formatted_bbox) - end + end end # This turns bboxes into separated pairs of coordinates (4326) diff --git a/lib/tasks/import_prebakedstats.rake b/lib/tasks/import_prebakedstats.rake index 389a8335..5f8a9e75 100644 --- a/lib/tasks/import_prebakedstats.rake +++ b/lib/tasks/import_prebakedstats.rake @@ -1,18 +1,19 @@ require 'csv' namespace :import do - desc "import CSV data into database" + desc 'import CSV data into database' task :prebakedstats, [:csv_file] => [:environment] do habitats = Habitat.all - dir = 'habitat_coverage_protection' - geo_entity_types = ["country", "regions"].freeze + dir = 'lib/data/habitat_coverage_protection' + geo_entity_types = %w[country regions].freeze # import habitat data CSVs for each entity type geo_entity_types.each do |geo_entity_type| habitats.each do |habitat| - filename = "#{habitat.name}_#{geo_entity_type}_output.csv" - csv_file = "#{dir}/#{filename}" - import_new_csv_file(habitat.name, csv_file) + filename = "#{habitat.name}_#{geo_entity_type}_output_*.csv" + filepath = "#{dir}/#{geo_entity_type}/#{filename}" + latest_csv_file = ::Utilities::Files.latest_file_by_glob(filepath) + import_new_csv_file(habitat.name, latest_csv_file) end end @@ -22,20 +23,21 @@ namespace :import do end def import_new_csv_file(habitat, csv_file) - filename = "#{Rails.root}/lib/data/#{csv_file}" - CSV.foreach(filename, headers: true, encoding: "utf-8") do |row| - # First we check whether it is a country (using ISO3) or a region depending on the CSV file - # Then we fetch the geo entity. If it can't be found, fetch_geo_entity will return nil - # Hence we can then skip any geo_entity that is nil - name = row['iso3'] || row['region'] - geo_entity = fetch_geo_entity(name) - next unless geo_entity - - if row.select {|k,v| /baseline/i =~ k}.any? - insert_change_stat(habitat, geo_entity, row) - end + ActiveRecord::Base.transaction do + CSV.foreach(csv_file, headers: true, encoding: 'utf-8') do |row| + # First we check whether it is a country (using ISO3) or a region depending on the CSV file + # Then we fetch the geo entity. If it can't be found, fetch_geo_entity will return nil + # Hence we can then skip any geo_entity that is nil + name = row['iso3'] || row['region'] + raise "Missing iso3 or region CSV header: #{csv_file}" unless name + + geo_entity = fetch_geo_entity(name) + next unless geo_entity + + insert_change_stat(habitat, geo_entity, row) if row.select { |k, _v| /baseline/i =~ k }.any? - insert_geo_entity_stat(habitat, geo_entity, row) + insert_geo_entity_stat(habitat, geo_entity, row) + end end end @@ -44,9 +46,9 @@ namespace :import do latest_year = get_latest_year(csv_row.headers) total_value_column = latest_year ? "total_area_#{latest_year}" : 'total_area' - protected_value = coerce_to_value(csv_row["protected_area"]) + protected_value = coerce_to_value(csv_row['protected_area']) total_value = coerce_to_value(csv_row[total_value_column]) - protected_percentage = coerce_to_value(csv_row["percent_protected"]) + protected_percentage = coerce_to_value(csv_row['percent_protected']) GeoEntityStat.find_or_create_by(habitat: habitat, geo_entity: geo_entity) do |stat| stat.protected_value = protected_value @@ -58,12 +60,12 @@ namespace :import do def coerce_to_value(value) # Set value to nil for deficient data return if value == '-' - + value&.strip || 0 end def get_latest_year(columns) - columns.map { |c| c.split('_').last }.select do |c| + columns.map { |c| c.split('_').last }.select do |c| next if c.nil? c.match(/\A\d+\z/) @@ -75,26 +77,24 @@ namespace :import do habitat = Habitat.find_by(name: habitat) ChangeStat.find_or_create_by(habitat: habitat, geo_entity: geo_entity) do |stat| - stat.total_value_1996 = coerce_to_value(csv_row["total_area_1996"]) - stat.total_value_2007 = coerce_to_value(csv_row["total_area_2007"]) - stat.total_value_2008 = coerce_to_value(csv_row["total_area_2008"]) - stat.total_value_2009 = coerce_to_value(csv_row["total_area_2009"]) - stat.total_value_2010 = coerce_to_value(csv_row["total_area_2010baseline"]) - stat.total_value_2015 = coerce_to_value(csv_row["total_area_2015"]) - stat.total_value_2016 = coerce_to_value(csv_row["total_area_2016"]) - stat.protected_value = coerce_to_value(csv_row["protected_area"]) - stat.protected_percentage = coerce_to_value(csv_row["percent_protected"]) + stat.total_value_1996 = coerce_to_value(csv_row['total_area_1996']) + stat.total_value_2007 = coerce_to_value(csv_row['total_area_2007']) + stat.total_value_2008 = coerce_to_value(csv_row['total_area_2008']) + stat.total_value_2009 = coerce_to_value(csv_row['total_area_2009']) + stat.total_value_2010 = coerce_to_value(csv_row['total_area_2010baseline']) + stat.total_value_2015 = coerce_to_value(csv_row['total_area_2015']) + stat.total_value_2016 = coerce_to_value(csv_row['total_area_2016']) + stat.protected_value = coerce_to_value(csv_row['protected_area']) + stat.protected_percentage = coerce_to_value(csv_row['percent_protected']) end end def fetch_geo_entity(name) - if (name.include? "/") || (name.include? "ABNJ") || (name.include? 'DPT') - geo_entity = GeoEntity.find_by(name: "Disputed") + if (name.include? '/') || (name.include? 'ABNJ') || (name.include? 'DPT') + GeoEntity.find_by(name: 'Disputed') else - geo_entity = GeoEntity.find_by(iso3: name) || GeoEntity.find_by(name: name) + GeoEntity.find_by(iso3: name) || GeoEntity.find_by(name: name) end - - geo_entity end def import_habitat_occurrences @@ -102,13 +102,13 @@ namespace :import do filename = 'lib/data/habitat_presence/habitat_presence_country.csv' - CSV.foreach(filename, headers: true, encoding: "utf-8") do |row| + CSV.foreach(filename, headers: true, encoding: 'utf-8') do |row| next unless countries.include?(row['ISO3']) Habitat.all.each do |habitat| stats = GeoEntity.find_by(iso3: row['ISO3']).geo_entity_stats - next unless stats.find { |stat| stat.habitat == habitat} + next unless stats.find { |stat| stat.habitat == habitat } camel_cased_title = habitat.title.gsub(/[\s-]/, '_') diff --git a/lib/tasks/import_refresh.rake b/lib/tasks/import_refresh.rake index 32eb68e3..001cb737 100644 --- a/lib/tasks/import_refresh.rake +++ b/lib/tasks/import_refresh.rake @@ -1,28 +1,52 @@ +# frozen_string_literal: true + +MODELS = %w[ + GeoEntityStat + ChangeStat + GeoRelationship + CoastalStat + GeoEntity + GeoEntitiesSpecies + Species + GeoEntityStatsSources + Source + CountryCitation + GlobalChangeCitation + GlobalChangeStat + Habitat +].freeze + +TASKS = %w[ + habitats + countries + regions + prebakedstats + new_redlist_data + coastalstats + global_change + sources + country_citations + habitat_sources_per_country + bounding_boxes +].freeze + namespace :import do - desc "Delete and recreate records" - task :refresh => [:environment] do - models = %w( - GeoEntityStat ChangeStat GeoRelationship CoastalStat GeoEntity - GeoEntitiesSpecies Species GeoEntityStatsSources Source CountryCitation - GlobalChangeCitation GlobalChangeStat Habitat - ) + desc 'Delete and recreate records' + task refresh: [:environment] do + ActiveRecord::Base.transaction do + MODELS.each do |m| + p "Deleting #{m}..." + m.constantize.send(:delete_all) + end - models.each do |m| - p "Deleting #{m}..." - m.constantize.send(:delete_all) - end + TASKS.each do |t| + task_name = "import:#{t}" - tasks = %w(habitats countries regions prebakedstats new_redlist_data coastalstats - global_change sources country_citations habitat_sources_per_country bounding_boxes - ) - - tasks.each do |t| - task_name = "import:#{t}" + p "Running #{task_name}..." + Rake::Task[task_name].invoke + end - p "Running #{task_name}..." - Rake::Task[task_name].invoke + Rake::Task['generate:national_csvs'].invoke end - - Rake::Task["generate:national_csvs"].invoke end end