Merge branch 'TextFileDetect' into Lyrics

Computational-Cognitive-Musicology-Lab · Feb 15, 2024 · 672b356 · 672b356
2 parents e5c1d7e + a5721d5
commit 672b356
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 4 deletions.
diff --git a/R/Read.R b/R/Read.R
@@ -311,9 +311,15 @@ readTextFiles <- function(fpaths) {
     if (length(fpaths) == 0L) return(character(0))
 
     raw <- lapply(fpaths, stringi::stri_read_raw)
-    enc <- data.table::rbindlist(lapply(stringi::stri_enc_detect2(raw), head, n = 1))
-
-    text <- !is.na(enc$Encoding) & enc$Confidence >= 0.75
+    enc <- data.table::rbindlist(lapply(stringi::stri_enc_detect(raw), head, n = 1))
+
+    # text <- !is.na(enc$Encoding) & enc$Confidence >= 0.75
+    text <- stringi::stri_enc_isascii(raw) |
+      stringi::stri_enc_isutf16be(raw) |
+      stringi::stri_enc_isutf16le(raw) |
+      stringi::stri_enc_isutf32be(raw) |
+      stringi::stri_enc_isutf16le(raw) |
+      stringi::stri_enc_isutf8(raw)
 
     files <- unlist(Map(stringi::stri_encode, raw[text], enc$Encoding[text]))
 

diff --git a/R/humdrumR-package.R b/R/humdrumR-package.R
@@ -46,7 +46,7 @@
 #' @importFrom glue glue glue_collapse
 #' @importFrom abind abind
 #' @importFrom stringr str_count str_detect str_dup str_extract str_match str_pad str_replace str_split str_sub str_sort
-#' @importFrom stringi stri_enc_detect2 stri_read_raw stri_trans_totitle
+#' @importFrom stringi stri_enc_detect stri_read_raw stri_trans_totitle stri_enc_isascii stri_enc_isutf16be stri_enc_isutf16le stri_enc_isutf32be stri_enc_isutf32be stri_enc_isutf8
 #' @importFrom rlang %|% %||% 
 #' @importFrom bit64 as.integer64 is.integer64
 #' @importFrom numbers primeFactors