Do not merge: Test readin files

db/migrate/20240925161004_replace_trade_compliance_views_for_rails_testing.rb

@@ -0,0 +1,16 @@
+class ReplaceTradeComplianceViewsForRailsTesting < ActiveRecord::Migration[7.1]
+  def up
+    safety_assured do
+      execute "CREATE OR REPLACE VIEW trade_shipments_appendix_i_view AS #{view_sql('20240905153246', 'trade_shipments_appendix_i_view')}"
+      execute "CREATE OR REPLACE VIEW trade_shipments_cites_suspensions_view AS #{view_sql('20240905155630', 'trade_shipments_cites_suspensions_view')}"
+      execute "CREATE OR REPLACE VIEW trade_shipments_mandatory_quotas_view AS #{view_sql('20240905153745', 'trade_shipments_mandatory_quotas_view')}"
+    end
+  end
+  def down
+    safety_assured do
+      execute "CREATE OR REPLACE VIEW trade_shipments_appendix_i_view AS #{view_sql('2023070615508', 'trade_shipments_appendix_i_view')}"
+      execute "CREATE OR REPLACE VIEW trade_shipments_cites_suspensions_view AS #{view_sql('2023070616851', 'trade_shipments_cites_suspensions_view')}"
+      execute "CREATE OR REPLACE VIEW trade_shipments_mandatory_quotas_view AS #{view_sql('2023070615541', 'trade_shipments_mandatory_quotas_view')}"
+    end
+  end
+end

lib/data/quotas.csv

@@ -1,7 +1,7 @@
 quota_type,start_date,end_date,taxon_name,taxon_concept_id,rank,iso_code2,quota,unit,term,source,purpose,Applies_to_import,notes,internal notes
-Listing_annotations,16/02/1995,Present,Melanosuchus niger,10978,SPECIES,EC,0,All,All,All,All,,"**I** Included in Appendix I, except for the population of Brazil, which is included in Appendix II, and the population of Ecuador, which is included in Appendix II and is subject to a zero annual export quota until an annual export quota has been approved by the CITES Secretariat and the IUCN/SSC Crocodile Specialist Group.
+Listing_annotations,16/02/1995,Present,Melanosuchus niger,10978,SPECIES,EC,0,All,All,All,All,,"**I** Included in Appendix I, except for the population of Brazil, which is included in Appendix II, and the population of Ecuador, which is included in Appendix II and is subject to a zero annual export quota until an annual export quota has been approved by the CITES Secretariat and the IUCN/SSC Crocodile Specialist Group.
 **II** Population of Brazil and the population of Ecuador, which is included in Appendix II and is subject to a zero annual export quota until an annual export quota has been approved by the CITES Secretariat and the IUCN/SSC Crocodile Specialist Group.",
-CoP_approved,09/06/1997,Present,Capra falconeri,3780,SPECIES,PK,12,All,TRO,All,H; P,,hunting trophies  Conf. 10.15 (Rev. CoP14),
+CoP_approved,09/06/1997,Present,Capra falconeri,3780,SPECIES,PK,0,All,TRO,All,H; P,,hunting trophies  Conf. 10.15 (Rev. CoP14),
 CoP_approved,09/06/1997,23/02/2023,Panthera pardus,8619,SPECIES,ET,500,All,SKI; TRO,All,H; P,,Resolution Conf. 10.14 (Rev. CoP19),
 CoP_approved,10/06/1997,Present,Panthera pardus,8619,SPECIES,NA,250,All,SKI; TRO,All,H; P,,Resolution Conf. 10.14 (Rev. CoP19),
 CoP_approved,11/06/1997,Present,Panthera pardus,8619,SPECIES,TZ,500,All,SKI; TRO,All,H; P,,Resolution Conf. 10.14 (Rev. CoP19),
@@ -17,7 +17,6 @@ CoP_approved,20/06/1997,Present,Panthera pardus,8619,SPECIES,UG,28,All,SKI; TRO,
 Listing_annotations,18/09/1997,Present,Chaetophractus nationi,5522,SPECIES,All,0,All,All,All,All,,**II** Included in Appendix II. A zero annual export quota has been established. All specimens shall be deemed to be specimens of species included in Appendix I and the trade in them shall be regulated accordingly.,
 Listing_annotations,19/07/2000,Present,Centrochelys sulcata,31025,SPECIES,All,0,All,All,W,T,,"**II** [FAMILY listing Testudinidae spp.] Included in Appendix II, except for the species included in Appendix I. A zero annual export quota has been established for Centrochelys sulcata for specimens removed from the wild and traded for primarily commercial purposes.",
 CoP_approved,14/10/2004,Present,Diceros bicornis,6656,SPECIES,NA,5,All,TRO; SKI; SKU; HOR; BOD; FOO; GEN,All,H; P,,hunting trophies from adult males (Resolution Conf. 13.5 (Rev.CoP18)),
-CoP_approved,14/10/2004,25/11/2019,Diceros bicornis,6656,SPECIES,ZA,5,All,TRO; SKI; SKU; HOR; BOD; FOO; GEN,All,H; P,,hunting trophies from adult males [Note: see Resolution Conf. 13.5 (Rev.CoP18)],
 Listing_annotations,12/06/2013,Present,Chelodina mccordi,7441,SPECIES,All,0,All,All,W,All,,**II** Zero export quota for specimens from the wild.,
 Listing_annotations,12/06/2013,Present,Batagur borneoensis,4927,SPECIES,All,0,All,All,W,T,,**II** Zero quota for wild specimens for commercial purposes.,
 Listing_annotations,12/06/2013,Present,Batagur trivittata,4397,SPECIES,All,0,All,All,W,T,,**II** Zero quota for wild specimens for commercial purposes.,
@@ -26,7 +25,7 @@ Listing_annotations,12/06/2013,25/11/2019,Cuora bourreti,65766,SPECIES,All,0,All
 Listing_annotations,12/06/2013,Present,Cuora flavomarginata,7271,SPECIES,All,0,All,All,W,T,,"**II** [GENUS listing Cuora spp.] Except the species included in Appendix I; zero quota for wild specimens for commercial purposes for Cuora aurocapitata, C. flavomarginata, C. galbinifrons, C. mccordi, C. mouhotii, C. pani, C. trifasciata, C. yunnanensis and C. zhoui.",
 Listing_annotations,12/06/2013,23/02/2023,Cuora galbinifrons,10210,SPECIES,All,0,All,All,W,T,,"**II** [GENUS listing Cuora spp.] Except the species included in Appendix I; zero quota for wild specimens for commercial purposes for Cuora aurocapitata, C. flavomarginata, C. galbinifrons, C. mccordi, C. mouhotii, C. pani, C. trifasciata, C. yunnanensis and C. zhoui.",
 Listing_annotations,12/06/2013,Present,Cuora mccordi,3678,SPECIES,All,0,All,All,W,T,,"**II** [GENUS listing Cuora spp.] Except the species included in Appendix I; zero quota for wild specimens for commercial purposes for Cuora aurocapitata, C. flavomarginata, C. galbinifrons, C. mccordi, C. mouhotii, C. pani, C. trifasciata, C. yunnanensis and C. zhoui.",
-Listing_annotations,12/06/2013,Present,Cuora mouhotii,10699,SPECIES,All,0,All,All,W,T,,"**II** [GENUS listing Cuora spp.] Except the species included in Appendix I; zero quota for wild specimens for commercial purposes for Cuora aurocapitata, C. flavomarginata, C. galbinifrons, C. mccordi, C. mouhotii, C. pani, C. trifasciata, C. yunnanensis and C. zhoui.",
+Listing_annotations,12/06/2013,31/12/2013,Cuora mouhotii,10699,SPECIES,All,0,All,All,W,T,,"**II** [GENUS listing Cuora spp.] Except the species included in Appendix I; zero quota for wild specimens for commercial purposes for Cuora aurocapitata, C. flavomarginata, C. galbinifrons, C. mccordi, C. mouhotii, C. pani, C. trifasciata, C. yunnanensis and C. zhoui.",
 Listing_annotations,12/06/2013,Present,Cuora pani,6783,SPECIES,All,0,All,All,W,T,,"**II** [GENUS listing Cuora spp.] Except the species included in Appendix I; zero quota for wild specimens for commercial purposes for Cuora aurocapitata, C. flavomarginata, C. galbinifrons, C. mccordi, C. mouhotii, C. pani, C. trifasciata, C. yunnanensis and C. zhoui.",
 Listing_annotations,12/06/2013,25/11/2019,Cuora picturata,65767,SPECIES,All,0,All,All,W,T,,,
 Listing_annotations,12/06/2013,Present,Cuora trifasciata,6023,SPECIES,All,0,All,All,W,T,,"**II** [GENUS listing Cuora spp.] Except the species included in Appendix I; zero quota for wild specimens for commercial purposes for Cuora aurocapitata, C. flavomarginata, C. galbinifrons, C. mccordi, C. mouhotii, C. pani, C. trifasciata, C. yunnanensis and C. zhoui.",
@@ -47,7 +46,7 @@ Listing_annotations,26/11/2019,Present,Lyriocephalus scutatus,98201,SPECIES,All,
 CoP_approved, 01/07/1975,Present,Acinonyx jubatus,8935,SPECIES,BW,5,All,LIV; TRO,All,All,,live and trophies,
 CoP_approved, 01/07/1975,Present,Acinonyx jubatus,8935,SPECIES,NA,150,All,LIV; TRO,All,All,,live and trophies,
 CoP_approved, 01/07/1975,Present,Acinonyx jubatus,8935,SPECIES,ZW,50,All,LIV; TRO,All,All,,live and trophies,
-Listing_annotations, 02/01/2017,Present,Crocodylus porosus,8560,SPECIES,MY,0,All,All,W,All,,"**I** Except the populations of Australia, Indonesia, Malaysia [wild harvest restricted to the State of Sarawak and a zero quota for wild specimens for the other States of Malaysia (Sabah and Peninsular Malaysia), with no change in the zero quota unless approved by the Parties] and Papua New Guinea, which are included in Appendix II.
+Listing_annotations, 02/01/2017,Present,Crocodylus porosus,8560,SPECIES,MY,0,All,All,W,All,,"**I** Except the populations of Australia, Indonesia, Malaysia [wild harvest restricted to the State of Sarawak and a zero quota for wild specimens for the other States of Malaysia (Sabah and Peninsular Malaysia), with no change in the zero quota unless approved by the Parties] and Papua New Guinea, which are included in Appendix II.
 **II** Only the populations of Australia, Indonesia, Malaysia [wild harvest restricted to the State of Sarawak and a zero quota for wild specimens for the other States of Malaysia (Sabah and Peninsular Malaysia), with no change in the zero quota unless approved by the Parties] and Papua New Guinea; all other populations are included in Appendix I.",
 Listing_annotations, 02/01/2017,Present,Abronia aurita,68179,SPECIES,All,0,All,All,W,All,,"**II** [GENUS listing Abronia spp.] Except the species included in Appendix I. Zero export quota for wild specimens for Abronia aurita, A. gaiophantasma, A. montecristoi, A. salvadorensis and A. vasconcelosii.",
 Listing_annotations, 02/01/2017,Present,Abronia gaiophantasma,68245,SPECIES,All,0,All,All,W,All,,"**II** [GENUS listing Abronia spp.] Except the species included in Appendix I. Zero export quota for wild specimens for Abronia aurita, A. gaiophantasma, A. montecristoi, A. salvadorensis and A. vasconcelosii.",
@@ -62,11 +61,11 @@ Listing_annotations, 13/02/2003,Present,Tursiops truncatus,7086,SPECIES,RU,0,All
 Listing_annotations, 13/02/2003,Present,Tursiops truncatus,7086,SPECIES,TR,0,All,LIV,W,T,,"**II** [ORDER listing Cetacea spp.] Included in Appendix II, except for the species included in Appendix I. A zero annual export quota has been established for live specimens from the Black Sea population of Tursiops truncatus removed from the wild and traded for primarily commercial purposes.",
 Listing_annotations, 13/02/2003,Present,Tursiops truncatus,7086,SPECIES,UA,0,All,LIV,W,T,,"**II** [ORDER listing Cetacea spp.] Included in Appendix II, except for the species included in Appendix I. A zero annual export quota has been established for live specimens from the Black Sea population of Tursiops truncatus removed from the wild and traded for primarily commercial purposes.",
 Listing_annotations, 23/06/2010, 01/01/2017,Crocodylus moreletii,7747,SPECIES,MX,0,All,All,W,T,,Zero quota for Mexico. Removed following listing in 02/01/2017,
-Listing_annotations, 23/06/2010,Present,Crocodylus moreletii,7747,SPECIES,BZ,0,All,All,W,T,,"**I** Except the population of Belize, which is included in Appendix II with a zero quota for wild specimens traded for commercial purposes, and the population of Mexico, which is included in Appendix II.
+Listing_annotations, 23/06/2010,Present,Crocodylus moreletii,7747,SPECIES,BZ,0,All,All,W,T,,"**I** Except the population of Belize, which is included in Appendix II with a zero quota for wild specimens traded for commercial purposes, and the population of Mexico, which is included in Appendix II.
 **II** Only the population of Belize, which is included in Appendix II with a zero quota for wild specimens traded for commercial purposes, and the population of Mexico.",
-Listing_annotations, 23/06/2010,Present,Crocodylus niloticus,10745,SPECIES,EG,0,All,All,W,T,,"**I** Included in Appendix I, except the populations of Botswana, Egypt (subject to a zero quota for wild specimens traded for commercial purposes), Ethiopia, Kenya, Madagascar, Malawi, Mozambique, Namibia, South Africa, Uganda, the United Republic of Tanzania (subject to an annual export quota of no more than 1,600 wild specimens including hunting trophies, in addition to ranched specimens), Zambia and Zimbabwe, which are included in Appendix II
+Listing_annotations, 23/06/2010,Present,Crocodylus niloticus,10745,SPECIES,EG,0,All,All,W,T,,"**I** Included in Appendix I, except the populations of Botswana, Egypt (subject to a zero quota for wild specimens traded for commercial purposes), Ethiopia, Kenya, Madagascar, Malawi, Mozambique, Namibia, South Africa, Uganda, the United Republic of Tanzania (subject to an annual export quota of no more than 1,600 wild specimens including hunting trophies, in addition to ranched specimens), Zambia and Zimbabwe, which are included in Appendix II
 **II** Populations of Botswana, Egypt (subject to a zero quota for wild specimens traded for commercial purposes), Ethiopia, Kenya, Madagascar, Malawi, Mozambique, Namibia, South Africa, Uganda, the United Republic of Tanzania (subject to an annual export quota of no more than 1,600 wild specimens including hunting trophies, in addition to ranched specimens), Zambia and Zimbabwe. ",
-Listing_annotations, 23/06/2010,Present,Crocodylus niloticus,10745,SPECIES,TZ,1600,All,All,W,All,,"**I** Included in Appendix I, except the populations of Botswana, Egypt (subject to a zero quota for wild specimens traded for commercial purposes), Ethiopia, Kenya, Madagascar, Malawi, Mozambique, Namibia, South Africa, Uganda, the United Republic of Tanzania (subject to an annual export quota of no more than 1,600 wild specimens including hunting trophies, in addition to ranched specimens), Zambia and Zimbabwe, which are included in Appendix II
+Listing_annotations, 23/06/2010,Present,Crocodylus niloticus,10745,SPECIES,TZ,1600,All,All,W,All,,"**I** Included in Appendix I, except the populations of Botswana, Egypt (subject to a zero quota for wild specimens traded for commercial purposes), Ethiopia, Kenya, Madagascar, Malawi, Mozambique, Namibia, South Africa, Uganda, the United Republic of Tanzania (subject to an annual export quota of no more than 1,600 wild specimens including hunting trophies, in addition to ranched specimens), Zambia and Zimbabwe, which are included in Appendix II
 **II** Populations of Botswana, Egypt (subject to a zero quota for wild specimens traded for commercial purposes), Ethiopia, Kenya, Madagascar, Malawi, Mozambique, Namibia, South Africa, Uganda, the United Republic of Tanzania (subject to an annual export quota of no more than 1,600 wild specimens including hunting trophies, in addition to ranched specimens), Zambia and Zimbabwe. ",
 Listing_annotations,23/02/2023,Present,Agalychnis lemur,101453,SPECIES,All,0,All,All,W,T,,wild-taken specimens traded for commercial purposes,
 Listing_annotations,23/02/2023,Present,Laotriton laoensis,10017,SPECIES,All,0,All,All,W,T,,zero annual export quota for wild-taken specimens traded for commercial purposes.,
@@ -76,3 +75,4 @@ Listing_annotations,23/02/2023,Present,Crocodylus porosus,8560,SPECIES,PH,0,All,
 CoP_approved,23/02/2023,Present,Panthera pardus,8619,SPECIES,ET,20,All,SKI; TRO,All,H; P,,Resolution Conf. 10.14 (Rev. CoP19),
 Listing_annotations,02/01/2017,31/12/2017,Panthera leo,6353,SPECIES,ZA,800,All,SKE; SKU,C,All,,"**II** [FAMILY listing Felidae spp.] For Panthera leo (African populations): a zero annual export quota is established for specimens of bones, bone pieces, bone products, claws, skeletons, skulls and teeth removed from the wild and traded for commercial purposes. Annual export quotas for trade in bones, bone pieces, bone products, claws, skeletons, skulls and teeth for commercial purposes, derived from captive breeding operations in South Africa, will be established and communicated annually to the CITES Secretariat.",quota published as 'Full skeletons (with or without the skull) derived from captive breeding operations'
 Listing_annotations,01/01/2018,31/12/2018,Panthera leo,6353,SPECIES,ZA,800,All,SKE; SKU,C,All,,"**II** [FAMILY listing Felidae spp.] For Panthera leo (African populations): a zero annual export quota is established for specimens of bones, bone pieces, bone products, claws, skeletons, skulls and teeth removed from the wild and traded for commercial purposes. Annual export quotas for trade in bones, bone pieces, bone products, claws, skeletons, skulls and teeth for commercial purposes, derived from captive breeding operations in South Africa, will be established and communicated annually to the CITES Secretariat.",quota published as 'Full skeletons (with or without the skull) derived from captive breeding operations'
+Listing_annotations,01/01/2014,Present,Eretmochelys imbricata,7257,SPECIES,All,0,All,BOD; CAP,All,All,,bodies and carapaces,

lib/data/reservations.csv

@@ -1,4 +1,4 @@
-start_date,end_date,taxon_name,taxon_concept_id,rank,iso_code2,notes,Internal Notes 
+start_date,end_date,taxon_name,taxon_concept_id,rank,iso_code2,notes,Internal Notes
 12/06/2013,20/05/2015,Platysternidae,531,FAMILY,CA, ,
 09/07/1975,11/08/1978,Eubalaena,923,GENUS,CA, ,
 28/06/1979,29/10/1982,Sousa,1286,GENUS,CA, ,
@@ -68,7 +68,7 @@ start_date,end_date,taxon_name,taxon_concept_id,rank,iso_code2,notes,Internal No
 01/08/1985,06/01/1989,Ara macao,4650,SPECIES,AT, ,
 01/08/1985,12/06/2014,Ara macao,4650,SPECIES,CH, ,
 01/08/1985,12/06/2014,Ara macao,4650,SPECIES,LI, ,
-01/08/1985,,Ara macao,4650,SPECIES,SR, ,
+01/08/1985,31/12/2014,Ara macao,4650,SPECIES,SR, ,
 29/07/1983,06/01/1989,Hyperoodon ampullatus,4979,SPECIES,AT, ,
 02/04/2000,,Hyperoodon ampullatus,4979,SPECIES,IS, ,
 29/07/1983,26/04/1995,Hyperoodon ampullatus,4979,SPECIES,RU,Original reservation as USSR. ,
@@ -110,7 +110,6 @@ start_date,end_date,taxon_name,taxon_concept_id,rank,iso_code2,notes,Internal No
 26/11/2019,02/12/2020,Crocodylus acutus,6436,SPECIES,CA,,
 02/04/2000,,Balaenoptera bonaerensis,6477,SPECIES,IS,Reservation entered under Balaenoptera acutorostrata. ,
 19/07/2000,,Balaenoptera bonaerensis,6477,SPECIES,JP, ,
-01/01/1986,,Balaenoptera bonaerensis,6477,SPECIES,NO,Reservation entered under Balaenoptera acutorostrata. ,
 31/10/1976,03/07/1978,Alligator sinensis,6693,SPECIES,GB,On behalf of HK. ,
 31/10/1976,03/07/1978,Terrapene coahuila,6708,SPECIES,GB,On behalf of HK. ,
 10/06/1996,,Aquila heliaca,6741,SPECIES,SA, ,
@@ -153,7 +152,6 @@ start_date,end_date,taxon_name,taxon_concept_id,rank,iso_code2,notes,Internal No
 01/01/1986,07/05/1991,Balaenoptera acutorostrata,8288,SPECIES,BR, ,
 02/04/2000,,Balaenoptera acutorostrata,8288,SPECIES,IS,Except stock of west Greenland. ,
 01/01/1986,,Balaenoptera acutorostrata,8288,SPECIES,JP,Except stock of west Greenland. ,
-01/01/1986,,Balaenoptera acutorostrata,8288,SPECIES,NO,Except stock of west Greenland. ,
 01/01/1986,24/10/2001,Balaenoptera acutorostrata,8288,SPECIES,PE,Except stock of west Greenland. ,
 15/07/2004,,Balaenoptera acutorostrata,8288,SPECIES,PW, ,
 01/01/1986,26/04/1995,Balaenoptera acutorostrata,8288,SPECIES,RU,Original reservation as USSR. ,
@@ -387,3 +385,4 @@ start_date,end_date,taxon_name,taxon_concept_id,rank,iso_code2,notes,Internal No
 23/02/2023,,Tiliqua adelaidensis,100209,SPECIES,CA,,
 23/02/2023,,Kinosternon cora,100473,SPECIES,CA,,
 23/02/2023,,Kinosternon vogti,100472,SPECIES,CA,,
+01/01/2015,,Oryx leucoryx,4154,SPECIES,AE,,

lib/data/trade_mapping.yml

@@ -500,12 +500,19 @@ rules:
       output:
         unit: 'NULL'
   standardise_terms_and_units:
-    - rule_name: 'Belly skins, back skins, hornback skins each count as half a skin'
+    - rule_name: 'Back skins count as half a skin'
       input:
         terms: ['SKI', 'SKP']
-        units: ['BSK', 'BAK', 'HRN']
+        units: ['BAK']
       output:
         term: 'SKI'
         unit: 'NULL'
         quantity_modifier: '/'
         modifier_value: 2
+    - rule_name: 'Belly skins, hornback skins each count as one whole skin'
+      input:
+        terms: ['SKI', 'SKP']
+        units: ['BSK', 'HRN']
+      output:
+        term: 'SKI'
+        unit: 'NULL'