Merge pull request #51 from EMODnet/develop

update BODC vocabs, add terms to dictionaries, deprecate Q01, fix cleandataframe
EMODnet · Jul 29, 2024 · 12163ec · 12163ec
2 parents 8dbb50a + 2c514b2
commit 12163ec
Show file tree

Hide file tree

Showing 16 changed files with 38 additions and 16 deletions.
diff --git a/R/checkdataset.R b/R/checkdataset.R
@@ -542,7 +542,7 @@ checkdataset = function(Event = NULL, Occurrence = NULL, eMoF = NULL, IPTreport
       mof_noSamplingdescriptor <- data.frame(level = "warning",
                                              field = "measurementType", 
                                              row = NA, 
-                                             message = "No sampling descriptors present: see http://vocab.nerc.ac.uk/collection/Q01/current/")
+                                             message = "No sampling descriptors/effort present: see https://github.com/EMODnet/EMODnetBiocheck/blob/master/files/workingEnvironment.R#L26")
     }
 
 

diff --git a/R/smallfunctions.R b/R/smallfunctions.R
@@ -76,17 +76,26 @@ cleandataframe <- function (x, vector = TRUE) {
     }
   }
 
-  # turn character NAs and spaces into an actual empty cell
-  x <- x %>% mutate(across(where(is.character), ~na_if(., "NA"))) %>%
-             mutate(across(where(is.character), ~na_if(., ""))) %>%
-             mutate(across(where(is.character), ~na_if(., " ")))
-
   # remove empty columns
   x <- x[,colSums(is.na(x))<nrow(x)]
 
+  # Transform to character
   if (vector == TRUE ){
     x <- data.frame(lapply(x, as.character), stringsAsFactors=FALSE)
   }
+
+  # Function to replace non-ASCII characters
+  replace_non_ascii <- function(x) {
+    iconv(x, from = "UTF-8", to = "ASCII//TRANSLIT")
+  }
+
+
+  # turn character NAs and spaces into an actual empty cell
+  x <- x %>% mutate(across(where(is.character), ~na_if(., "NA"))) %>%
+    mutate(across(where(is.character), ~na_if(., "NULL"))) %>%
+    mutate(across(where(is.character), ~na_if(., ""))) %>%
+    mutate(across(where(is.character), ~na_if(., " "))) %>%
+    mutate(across(where(is.character), replace_non_ascii))
 
   return (x)
 }

diff --git a/README.md b/README.md
@@ -105,7 +105,7 @@ The tool is based on and builds on top of the <a href="https://github.com/iobis/
 | Any missing measurementTypeIDs in the eMoF table (for non occurrence related mofs) | measurementTypeID is missing | IPTreport\$dtb\$mof_issues | For interoperability reasons, the measurementTypeID field should be populated if the term exists in BODC, in case it doesn't it is good practice to request the creation of one. | 
 | Check if measurementTypeIDs resolve to BODC terms (for non occurrence related mofs) | measurementTypeID does not resolve | IPTreport\$dtb\$mof_issues | The vocabulary terms used under the measurementTypeID field must resolve to an existing concept, ideally to a concept from BODC | 
 | Check if there is a BODC vocab term for "sampling instrument" in the eMoF table | No sampling instrument present | IPTreport\$dtb\$general_issues | It is highly recommended to add sampling related information to a dataset and to use standardised vocabulary terms for interoperability reasons. | 
-| Check if there is a BODC vocab term for "sampling descriptors" in the eMoF table | No sampling descriptors present: see http://vocab.nerc.ac.uk/collection/Q01/current/ | IPTreport\$dtb\$general_issues | It is highly recommended to add sampling related information to a dataset and to use standardised vocabulary terms for interoperability reasons. | 
+| Check if there is a BODC vocab term for "sampling descriptors" in the eMoF table | No sampling descriptors/effort present: see https://github.com/EMODnet/EMODnetBiocheck/blob/master/files/workingEnvironment.R#L26 | IPTreport\$dtb\$general_issues | It is highly recommended to add sampling related information to a dataset and to use standardised vocabulary terms for interoperability reasons. | 
 | Any missing measurementTypeIDs in the eMoF table | measurementTypeID is missing | IPTreport\$dtb\$mof_issues | For interoperability reasons, the measurementTypeID field should be populated if the term exists in BODC, in case it doesn't it is good practice to request the creation of one. | 
 | Check if occurrence related measurementTypeIDs resolve to BODC terms | measurementTypeID does not resolve | IPTreport\$dtb\$mof_issues | The vocabulary terms used under the measurementTypeID field must resolve to an existing concept, ideally to a concept from BODC | 
 | Check if measurementUnitIDs resolve to BODC terms (for non occurrence related mofs) | measurementUnitID does not resolve | IPTreport\$dtb\$mof_issues | The vocabulary terms used under the measurementUnitID field must resolve to an existing concept, ideally to a concept from BODC | 

diff --git a/data/BODCbiometrics.rda b/data/BODCbiometrics.rda
diff --git a/data/BODCeffort.rda b/data/BODCeffort.rda
diff --git a/data/BODCinstrument.rda b/data/BODCinstrument.rda
diff --git a/data/BODCnomofvalues.rda b/data/BODCnomofvalues.rda
diff --git a/data/BODCparameters.rda b/data/BODCparameters.rda
diff --git a/data/BODCquantity.rda b/data/BODCquantity.rda
diff --git a/data/BODCunits.rda b/data/BODCunits.rda
diff --git a/data/BODCvalues.rda b/data/BODCvalues.rda
diff --git a/data/p01todwc.rda b/data/p01todwc.rda
diff --git a/files/workingEnvironment.R b/files/workingEnvironment.R
@@ -21,11 +21,15 @@ BODCbiometrics <- c ('http://vocab.nerc.ac.uk/collection/P01/current/LSTAGE01/',
                       'http://vocab.nerc.ac.uk/collection/P01/current/CELLVOLM/', 'http://vocab.nerc.ac.uk/collection/P01/current/OBSMAXLX/',
                       'http://vocab.nerc.ac.uk/collection/P01/current/OBSMINLX/', 'http://vocab.nerc.ac.uk/collection/P01/current/CTROPH01/', 
                       'http://vocab.nerc.ac.uk/collection/P01/current/ENTSEX01/', 'http://vocab.nerc.ac.uk/collection/P01/current/SL01XX01/',
-                      'http://vocab.nerc.ac.uk/collection/S06/current/S0600089/', 'http://vocab.nerc.ac.uk/collection/S06/current/S0600270/') #create a list of the measurementtypeIDs to be taken into account when checking the occurrence table for duplicates 
+                      'http://vocab.nerc.ac.uk/collection/S06/current/S0600089/', 'http://vocab.nerc.ac.uk/collection/S06/current/S0600270/',
+                      'https://vocab.nerc.ac.uk/collection/P01/current/SZCLBIO1/') #create a list of the measurementtypeIDs to be taken into account when checking the occurrence table for duplicates 
 
-BODCeffort <- c('AREABEDS', 'Q01', 'VOLWBSMP', 'LENTRACK' , 'AZDRZZ01' ,'VOLFFMXX')
+BODCeffort <- c('Q01', 'AREABEDS', 'VOLWBSMP', 'LENTRACK', 'AZDRZZ01', 'VOLFFMXX', 'SZCLBIO1', 'PRSZSPLW', 'MSHSIZE2', 'DSSPXTMT',
+                'DSSPPRMT', 'DSAMPA01', 'MTHHGHT1', 'MTHAREA1', 'MTHWDTH1', 'DSSPMT01', 'MSHSIZE1', 'NMSPPF01', 'DSPSSP01',
+                'PRSZSPUP', 'MSHSIZE3', 'WASPXX01', 'COREWDTH', 'VOLXXXXX', 'VOLFFMDT', 'VOLFDGDT', 'VOLFMCXX', 'VOLSEDSM',
+                'VOLSPOPC', 'M91BA36A') # Mostly P01s linked to https://vocab.nerc.ac.uk/collection/P02/current/SAMP/
 
-BODCinstrument <- c('Q0100002')
+BODCinstrument <- c('NMSPINST')
 
 
 
@@ -59,9 +63,18 @@ p01todwc <- data.frame(P01s = c("SNANID01", "SCNAME01", "SSAMID01", "SAMPPROT",
                        stringsAsFactors = FALSE)
 
 
-# use_data(BODCunits, overwrite = TRUE)
-# use_data(BODCvalues, overwrite = TRUE)
-# use_data(BODCparameters, overwrite = TRUE)
+
+
+# Gets new params, values and units from https://gitlab.vliz.be/datac/eurobis-dmt/-/blob/master/BODC/BODC_tables_generator.py
+BODCunits <- read.csv("~/EMODnetBiocheck/files/BODCunits.csv") %>% cleandataframe()
+BODCparameters <- read.csv("~/EMODnetBiocheck/files/BODCparameters.csv") %>% cleandataframe()
+BODCvalues <- read.csv("~/EMODnetBiocheck/files/BODCvalues.csv") %>% cleandataframe()
+
+
+
+use_data(BODCunits, overwrite = TRUE)
+use_data(BODCvalues, overwrite = TRUE)
+use_data(BODCparameters, overwrite = TRUE)
 use_data(BODCnomofvalues, overwrite = TRUE)
 use_data(BODCbiometrics, overwrite = TRUE)
 use_data(BODCeffort, overwrite = TRUE)

diff --git a/man/BODCbiometrics.Rd b/man/BODCbiometrics.Rd
diff --git a/man/BODCeffort.Rd b/man/BODCeffort.Rd
diff --git a/man/BODCquantity.Rd b/man/BODCquantity.Rd