move to parquet format + document and fucntions keywording + organize…

… R folder by modules

.github/workflows/pkgdown.yaml

@@ -9,7 +9,9 @@ on:
     types: [published]
   workflow_dispatch:
 
-name: pkgdown
+name: pkgdown.yaml
+
+permissions: read-all
 
 jobs:
   pkgdown:
@@ -22,7 +24,7 @@ jobs:
     permissions:
       contents: write
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - uses: r-lib/actions/setup-pandoc@v2
 
@@ -41,7 +43,7 @@ jobs:
 
       - name: Deploy to GitHub pages 🚀
         if: github.event_name != 'pull_request'
-        uses: JamesIves/github-pages-deploy-action@v4.4.1
+        uses: JamesIves/github-pages-deploy-action@v4.5.0
         with:
           clean: false
           branch: gh-pages

.gitignore

@@ -43,3 +43,4 @@ docs
 *.xlsx
 *.csv
 *.tsv
+*.parquet

DESCRIPTION

@@ -1,7 +1,7 @@
 Package: peskas.zanzibar.data.pipeline
 Title: Functions to Implement the Zanzibar Small Scale Fisheries
     Data Pipeline
-Version: 0.2.0
+Version: 1.0.0
 Authors@R: 
     c(person(given = "Lorenzo",
              family = "Longobardi",
@@ -22,12 +22,12 @@ Imports:
     logger,
     magrittr,
     purrr,
-    readr,
     stringr,
     tidyr,
     rlang,
     lubridate,
-    taxize
+    taxize,
+    arrow
 Suggests: 
     covr,
     pkgdown,

Dockerfile

@@ -10,7 +10,7 @@ RUN install2.r --error --skipinstalled \
     logger \
     magrittr \
     purrr \
-    readr \
+    arrow \
     stringr \
     tidyr \
     lubridate \
@@ -30,9 +30,7 @@ RUN install2.r --error --skipinstalled \
     univOutl \
     taxize \
     reticulate \
-    stringi \
-    taxize
+    stringi
 
-#RUN Rscript -e "devtools::install_version('glmmTMB', version = '1.1.5')"
 # Rstudio interface preferences
 COPY rstudio-prefs.json /home/rstudio/.config/rstudio/rstudio-prefs.json

Dockerfile.prod

@@ -23,7 +23,7 @@ RUN install2.r --error --skipinstalled \
     logger \
     magrittr \
     purrr \
-    readr \
+    arrow \
     stringr \
     tidyr \
     lubridate \
@@ -41,10 +41,8 @@ RUN install2.r --error --skipinstalled \
     tidytext \
     KoboconnectR \
     univOutl \
-    taxize \
     reticulate \
-    stringi \
-    taxize
+    stringi
 
 # Install local package
 COPY . /home

NAMESPACE

@@ -5,6 +5,7 @@ export("%||%")
 export(":=")
 export(.data)
 export(add_version)
+export(alert_outlier)
 export(as_label)
 export(as_name)
 export(cloud_object_name)
@@ -14,17 +15,19 @@ export(enquo)
 export(enquos)
 export(expand_taxa)
 export(expr)
+export(flatten_field)
+export(flatten_row)
 export(get_preprocessed_surveys)
 export(get_validated_surveys)
 export(ingest_wcs_surveys)
 export(kepler_mapper)
 export(preprocess_wcs_surveys)
 export(pt_nest_attachments)
 export(pt_nest_catch)
-export(pt_nest_length)
 export(pt_nest_market)
 export(pt_nest_trip)
 export(read_config)
+export(rename_child)
 export(retrieve_wcs_surveys)
 export(sym)
 export(syms)
@@ -34,27 +37,9 @@ export(validate_length)
 export(validate_market)
 export(validate_surveys_time)
 export(validate_wcs_surveys)
-import(dplyr)
-import(purrr)
-import(stringr)
-import(taxize)
-import(tidyr)
-importFrom(dplyr,case_when)
-importFrom(dplyr,coalesce)
-importFrom(dplyr,group_by)
-importFrom(dplyr,left_join)
-importFrom(dplyr,mutate)
-importFrom(dplyr,select)
-importFrom(dplyr,ungroup)
-importFrom(logger,log_info)
-importFrom(logger,log_threshold)
-importFrom(lubridate,with_tz)
 importFrom(magrittr,"%<>%")
 importFrom(magrittr,"%>%")
 importFrom(magrittr,"%T>%")
-importFrom(purrr,map)
-importFrom(purrr,reduce)
-importFrom(readr,write_rds)
 importFrom(rlang,"%||%")
 importFrom(rlang,":=")
 importFrom(rlang,.data)
@@ -66,6 +51,4 @@ importFrom(rlang,expr)
 importFrom(rlang,sym)
 importFrom(rlang,syms)
 importFrom(stats,mad)
-importFrom(tidyr,nest)
-importFrom(tidyr,unnest)
 importFrom(utils,globalVariables)

NEWS.md

@@ -1,3 +1,13 @@
+# peskas.zanzibar.data.pipeline 1.0.0
+
+#### Improvements
+- All the functions are now documented and indexed according to keywords
+- Thin out the R folder gathering functions by modules
+
+#### Changes
+- Move to parquet format rather than CSV/RDS
+
+
 # peskas.zanzibar.data.pipeline 0.2.0
 
 #### New features