Expand file tokenisation

src/ARCTokenization/FileSystem.fs

@@ -9,6 +9,7 @@ open ARCTokenization.StructuralOntology
 open System.IO
 open System
 open ControlledVocabulary
+open Tokenization
 
 module internal FS =
 
@@ -51,4 +52,31 @@ module internal FS =
                 cvTerm = AFSO.``File Path``, 
                 v = file.Replace("\\","/")
             )
-        }
+        }
+
+
+    let private normalisePath (path:string) =
+        path.Replace("\\","/")
+
+    let tokenizeARCFileSystem (rootPath:string) =
+        let rootPathNormalised = rootPath|>normalisePath
+
+        let directories =
+            Directory.EnumerateDirectories(rootPath, "*", SearchOption.AllDirectories)
+            |> Seq.map(fun p -> 
+                Tokenization.SpecificTokens.PType.Directory,
+                p|>normalisePath
+            )
+
+        let files = 
+            Directory.EnumerateFiles(rootPath, "*", SearchOption.AllDirectories)
+            |> Seq.map(fun p -> 
+                Tokenization.SpecificTokens.PType.File,
+                p|>normalisePath
+            )
+        let collection: (Tokenization.SpecificTokens.PType * string) seq = Seq.concat (seq{directories;files})
+
+        collection
+        |>Seq.map(fun (pType,p) -> Tokenization.SpecificTokens.getSpecificTokens rootPathNormalised pType p)
+
+

src/ARCTokenization/Tokenization.fs

@@ -4,6 +4,7 @@ open ControlledVocabulary
 open FsSpreadsheet
 open MetadataSheet
 open ARCTokenization.Terms
+open ARCtrl
 open ARCtrl.ISA
 
 module Tokenization = 
@@ -158,3 +159,40 @@ module Tokenization =
                 at.Columns
                 |> Array.map CompositeColumn.tokenize
                 |> List.ofArray
+
+    module SpecificTokens =
+
+        /// Represents the type of file system entity (Directory or File)
+        type PType =
+            | File
+            | Directory
+
+        /// Matches a CvParam based on the relative path and file system type
+        let matchFileSystemTokenByRelativePath (pType:PType) (relativePath: string) = 
+                match pType with
+                | PType.Directory ->
+                    match (relativePath.Split '/') with
+                    | [|Path.StudiesFolderName|]        ->  StructuralOntology.AFSO.``Studies Directory``   |> fun t -> CvParam(t,relativePath)
+                    | [|Path.StudiesFolderName; _|]     ->  StructuralOntology.AFSO.``Study Directory``     |> fun t -> CvParam(t,relativePath)
+                    | [|Path.AssaysFolderName|]         ->  StructuralOntology.AFSO.``Assays Directory``    |> fun t -> CvParam(t,relativePath)
+                    | [|Path.AssaysFolderName; _|]      ->  StructuralOntology.AFSO.``Assay Directory``     |> fun t -> CvParam(t,relativePath)
+                    | [|Path.RunsFolderName|]           ->  StructuralOntology.AFSO.``Runs Directory``      |> fun t -> CvParam(t,relativePath)
+                    | [|Path.RunsFolderName; _|]        ->  StructuralOntology.AFSO.``Run Directory``       |> fun t -> CvParam(t,relativePath)
+                    | [|Path.WorkflowsFolderName|]      ->  StructuralOntology.AFSO.``Workflows Directory`` |> fun t -> CvParam(t,relativePath)
+                    | [|Path.WorkflowsFolderName; _|]   ->  StructuralOntology.AFSO.``Study Directory``     |> fun t -> CvParam(t,relativePath)
+                    | _                                 ->  StructuralOntology.AFSO.``Directory Path``      |> fun t -> CvParam(t,relativePath)
+                | PType.File ->
+                    match relativePath with
+                    | _ when relativePath.EndsWith "isa.investigation.xlsx" -> StructuralOntology.AFSO.``Investigation File``   |> fun t -> CvParam(t,relativePath)
+                    | _ when relativePath.EndsWith "isa.assay.xlsx"         -> StructuralOntology.AFSO.``Assay File``           |> fun t -> CvParam(t,relativePath)
+                    | _ when relativePath.EndsWith "isa.dataset.xlsx"       -> StructuralOntology.AFSO.``Dataset File``         |> fun t -> CvParam(t,relativePath)
+                    | _ when relativePath.EndsWith "isa.study.xlsx"         -> StructuralOntology.AFSO.``Study File``           |> fun t -> CvParam(t,relativePath)
+                    | _ when relativePath.EndsWith ".yml"                   -> StructuralOntology.AFSO.``YML File``             |> fun t -> CvParam(t,relativePath)
+                    | _ when relativePath.EndsWith ".cwl"                   -> StructuralOntology.AFSO.``CWL File``             |> fun t -> CvParam(t,relativePath)
+                    | _                                                     -> StructuralOntology.AFSO.``File Path``            |> fun t -> CvParam(t,relativePath)
+
+        /// Gets CvParams based on the root path, file system type, and full path
+        let getSpecificTokens (rootPath:string) (pType:PType) (path:string) =
+            let relativePath = path.Replace(rootPath,"")
+            matchFileSystemTokenByRelativePath pType relativePath
+

src/ARCTokenization/structural_ontologies/AFSO.fs

@@ -23,4 +23,28 @@ module AFSO =
     /// The full path, relative path or a Universal Naming ConventAFSOn (UNC) path
     let ``Path``           = CvTerm.create("AFSO:00000008","Path","AFSO")
     let ``File Path``      = CvTerm.create("AFSO:00000009","File Path","AFSO")
-    let ``Directory Path`` = CvTerm.create("AFSO:00000010","Directory Path","AFSO")
+    let ``Directory Path`` = CvTerm.create("AFSO:00000010","Directory Path","AFSO")
+
+    // Top level Directories
+    let ``Studies Directory``   = CvTerm.create("AFSO:00000011","Studies Directory","AFSO")
+    let ``Assays Directory``    = CvTerm.create("AFSO:00000012","Assays Directory","AFSO")
+    let ``Runs Directory``      = CvTerm.create("AFSO:00000013","Runs Directory","AFSO")
+    let ``Workflows Directory`` = CvTerm.create("AFSO:00000014","Workflows Directory","AFSO")
+
+    // Sub level folders
+    let ``Study Directory``   = CvTerm.create("AFSO:00000015","Study Directory","AFSO")
+    let ``Assay Directory``    = CvTerm.create("AFSO:00000016","Assay Directory","AFSO")
+    let ``Run Directory``      = CvTerm.create("AFSO:00000017","Run Directory","AFSO")
+    let ``Workflow Directory`` = CvTerm.create("AFSO:00000018","Workflow Directory","AFSO")
+
+
+    // Isa FileTypes 
+    let ``Investigation File``   = CvTerm.create("AFSO:00000019","Investigation File","AFSO")
+    let ``Study File``           = CvTerm.create("AFSO:00000020","Study File","AFSO")
+    let ``Assay File``           = CvTerm.create("AFSO:00000021","Assay File","AFSO")
+    let ``Dataset File``         = CvTerm.create("AFSO:00000022","Dataset File","AFSO")
+
+    // Additional FileTypes
+    let ``CWL File``         = CvTerm.create("AFSO:00000023","CWL File","AFSO")
+    let ``YML File``         = CvTerm.create("AFSO:00000024","YML File","AFSO")
+

src/ARCTokenization/structural_ontologies/arc_file_structure_ontology.obo

@@ -1,56 +1,126 @@
-!This file was auto generated on 2024-01-10. Do not edit it. All manual changes will be overwritten by the next generator run eventually.
-format-version: 1.2
-data-version: init/2023-10-26
-saved-by: Kevin Schneider
-default-namespace: afso
-ontology: AFSO
-
-[Term]
-id: AFSO:00000001
-name: File
-def: ""
-
-[Term]
-id: AFSO:00000002
-name: Directory
-def: ""
-
-[Term]
-id: AFSO:00000003
-name: File Type
-def: ""
-
-[Term]
-id: AFSO:00000004
-name: Extension
-def: ""
-
-[Term]
-id: AFSO:00000005
-name: File Name
-def: ""
-
-[Term]
-id: AFSO:00000006
-name: Directory Name
-def: ""
-
-[Term]
-id: AFSO:00000007
-name: Full Name
-def: ""
-
-[Term]
-id: AFSO:00000008
-name: Path
-def: ""
-
-[Term]
-id: AFSO:00000009
-name: File Path
-def: ""
-
-[Term]
-id: AFSO:00000010
-name: Directory Path
-def: ""
+!This file was auto generated on 26.02.2024. Do not edit it. All manual changes will be overwritten by the next generator run eventually.
+format-version: 1.2
+data-version: init/2023-10-26
+saved-by: Kevin Schneider
+default-namespace: afso
+ontology: AFSO
+
+[Term]
+id: AFSO:00000001
+name: File
+def: ""
+
+[Term]
+id: AFSO:00000002
+name: Directory
+def: ""
+
+[Term]
+id: AFSO:00000003
+name: File Type
+def: ""
+
+[Term]
+id: AFSO:00000004
+name: Extension
+def: ""
+
+[Term]
+id: AFSO:00000005
+name: File Name
+def: ""
+
+[Term]
+id: AFSO:00000006
+name: Directory Name
+def: ""
+
+[Term]
+id: AFSO:00000007
+name: Full Name
+def: ""
+
+[Term]
+id: AFSO:00000008
+name: Path
+def: ""
+
+[Term]
+id: AFSO:00000009
+name: File Path
+def: ""
+
+[Term]
+id: AFSO:00000010
+name: Directory Path
+def: ""
+
+[Term]
+id: AFSO:00000011
+name: Studies Directory
+def: ""
+
+[Term]
+id: AFSO:00000012
+name: Assays Directory
+def: ""
+
+[Term]
+id: AFSO:00000013
+name: Runs Directory
+def: ""
+
+[Term]
+id: AFSO:00000014
+name: Workflows Directory
+def: ""
+
+[Term]
+id: AFSO:00000015
+name: Study Directory
+def: ""
+
+[Term]
+id: AFSO:00000016
+name: Assay Directory
+def: ""
+
+[Term]
+id: AFSO:00000017
+name: Run Directory
+def: ""
+
+[Term]
+id: AFSO:00000018
+name: Workflow Directory
+def: ""
+
+[Term]
+id: AFSO:00000019
+name: Investigation File
+def: ""
+
+[Term]
+id: AFSO:00000020
+name: Study File
+def: ""
+
+[Term]
+id: AFSO:00000021
+name: Assay File
+def: ""
+
+[Term]
+id: AFSO:00000022
+name: Dataset File
+def: ""
+
+[Term]
+id: AFSO:00000023
+name: CWL File
+def: ""
+
+[Term]
+id: AFSO:00000024
+name: YML File
+def: ""

src/ARCTokenization/structural_ontologies/arc_file_structure_ontology.yml

@@ -52,4 +52,74 @@ def: ""
 [Term]
 id: AFSO:00000010
 name: Directory Path
+def: ""
+
+[Term]
+id: AFSO:00000011
+name: Studies Directory
+def: ""
+
+[Term]
+id: AFSO:00000012
+name: Assays Directory
+def: ""
+
+[Term]
+id: AFSO:00000013
+name: Runs Directory
+def: ""
+
+[Term]
+id: AFSO:00000014
+name: Workflows Directory
+def: ""
+
+[Term]
+id: AFSO:00000015
+name: Study Directory
+def: ""
+
+[Term]
+id: AFSO:00000016
+name: Assay Directory
+def: ""
+
+[Term]
+id: AFSO:00000017
+name: Run Directory
+def: ""
+
+[Term]
+id: AFSO:00000018
+name: Workflow Directory
+def: ""
+
+[Term]
+id: AFSO:00000019
+name: Investigation File
+def: ""
+
+[Term]
+id: AFSO:00000020
+name: Study File
+def: ""
+
+[Term]
+id: AFSO:00000021
+name: Assay File
+def: ""
+
+[Term]
+id: AFSO:00000022
+name: Dataset File
+def: ""
+
+[Term]
+id: AFSO:00000023
+name: CWL File
+def: ""
+
+[Term]
+id: AFSO:00000024
+name: YML File
 def: ""

src/ARCTokenization/structural_ontologies/arc_process_graph_structural_ontology.obo

@@ -1,4 +1,4 @@
-!This file was auto generated on 2024-01-10. Do not edit it. All manual changes will be overwritten by the next generator run eventually.
+!This file was auto generated on 26.02.2024. Do not edit it. All manual changes will be overwritten by the next generator run eventually.
 format-version: 1.2
 data-version: init/2024-01-09
 saved-by: Kevin Schneider