Merge pull request #317 from nfdi4plants/python_integration

Python integration

.config/dotnet-tools.json

@@ -3,7 +3,7 @@
   "isRoot": true,
   "tools": {
     "fable": {
-      "version": "4.6.0",
+      "version": "4.13.0",
       "commands": [
         "fable"
       ]

.github/workflows/build-test.yml

@@ -10,19 +10,24 @@ jobs:
   test:
 
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest]
 
     runs-on: ${{ matrix.os }}
 
     steps:
     - uses: actions/checkout@v3
+
+    # SETUP .NET
     - name: Setup .NET
       uses: actions/setup-dotnet@v3
       with:
         dotnet-version: 6.x.x
     - name: Restore fable
       run: dotnet tool restore
+
+    # SETUP NODE
     - name: Setup Node.js environment
       uses: actions/setup-node@v3
       with:
@@ -33,6 +38,28 @@ jobs:
     - name: install node modules ISA
       working-directory: ./src/ISA
       run: npm install
+
+    # SETUP PYTHON
+    - name: Setup Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+    - name: Setup Virtual Environment
+      run: python -m venv .venv
+    - name: Setup Poetry Windows
+      if: matrix.os == 'windows-latest'
+      run: |
+          .\.venv\Scripts\python.exe -m pip install -U pip setuptools
+          .\.venv\Scripts\python.exe -m pip install poetry
+          .\.venv\Scripts\python.exe -m poetry install --no-root
+    - name: Setup Poetry Unix
+      if: matrix.os == 'ubuntu-latest'
+      run: |
+          ./.venv/bin/python -m pip install -U pip setuptools
+          ./.venv/bin/python -m pip install poetry
+          ./.venv/bin/python -m poetry install --no-root
+
+    # BUILD
     - name: make script executable
       if: matrix.os == 'ubuntu-latest'
       run: chmod u+x build.sh

.gitignore

@@ -355,6 +355,7 @@ paket-files/
 # Custom ignored files
 dist/
 tmp/
+/.venv
 /tests/JavaScript/ARCtrl
 /tests/Python/ARCtrl
 /tests/UI/ARCtrl

ARCtrl.sln

@@ -12,6 +12,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		.config\dotnet-tools.json = .config\dotnet-tools.json
 		global.json = global.json
 		package.json = package.json
+		pyproject.toml = pyproject.toml
 		README.md = README.md
 		RELEASE_NOTES.md = RELEASE_NOTES.md
 	EndProjectSection

README.md

@@ -20,7 +20,7 @@ and __JavaScript__! ❤️
 ``` 
 
 ```bash
-<PackageReference Include="ARCtrl" Version="1.0.0-beta.1" />
+<PackageReference Include="ARCtrl" Version="1.1.0" />
 ```
 
 ### JavaScript
@@ -44,10 +44,27 @@ Currently we provide some documentation in form of markdown files in the `/docs`
     - verify with `npm --version` (Tested with v9.2.0)
 - [.NET SDK](https://dotnet.microsoft.com/en-us/download)
     - verify with `dotnet --version` (Tested with 7.0.306)
+- [Python](https://www.python.org/downloads/)
+    - verify with `py --version` (Tested with 3.12.2, known to work only for >=3.11)
 
 ### Local Setup
 
-1. `dotnet tool restore`
-3. `npm install`
+1. Setup dotnet tools
+
+   `dotnet tool restore`
+
+2. Install NPM dependencies
+
+   `npm install`
+
+3. Setup python environment
+
+   `py -m venv .venv`
+
+4. Install [Poetry](https://python-poetry.org/) and dependencies
+
+   1. `.\.venv\Scripts\python.exe -m pip install -U pip setuptools`
+   2. `.\.venv\Scripts\python.exe -m pip install poetry`
+   3. `.\.venv\Scripts\python.exe -m poetry install --no-root`
 
 Verify correct setup with `./build.cmd runtests` ✨

RELEASE_NOTES.md

@@ -1,3 +1,43 @@
+### 1.2.0+19d850e (Released 2024-3-8)
+* Additions:
+    * Added Python compatability
+    * [[#19d850e](https://github.com/nfdi4plants/ARCtrl/commit/19d850ed5e1474a5e82cd98537db3f398fbacc18)] several small cleanups according to PR #317 comments
+    * [[#8becc67](https://github.com/nfdi4plants/ARCtrl/commit/8becc672c432eb4399fc3a0e250a87e63044de2b)] update build project for releasing python package
+    * [[#df663cb](https://github.com/nfdi4plants/ARCtrl/commit/df663cbfe8cdfaa7ac48690f89539b17715ae0c0)] include python setup in CI
+    * [[#66b83a2](https://github.com/nfdi4plants/ARCtrl/commit/66b83a2292b082e96676a52c1dd176695173f949)] several small changes to test stack
+    * [[#3dfaad7](https://github.com/nfdi4plants/ARCtrl/commit/3dfaad77a14aa70715388ef6a1fa77cb337bd622)] adjustments to web and validation to allowe for python transpilation
+    * [[#688c628](https://github.com/nfdi4plants/ARCtrl/commit/688c6283e9b2bd332fd93388b10d8661e185bc1b)] work on json io support in python
+    * [[#93b6d94](https://github.com/nfdi4plants/ARCtrl/commit/93b6d9490acdf16045bd844f4cec22f2d54c7eca)] replace mocha and expecto with pyxpecto
+    * [[#aa30389](https://github.com/nfdi4plants/ARCtrl/commit/aa30389f070a248e547373779de64247d0b017cc)] move json schema validation to json tests
+    * [[#f0bb78d](https://github.com/nfdi4plants/ARCtrl/commit/f0bb78dcdccb1b2bfc31e6cc1c36c6905c950045)] switch towards using .venv for running transpiled python
+    * [[#2d35cfb](https://github.com/nfdi4plants/ARCtrl/commit/2d35cfbb7b75ca79e5b2d5f28cb6dc56514acab0)] update Fable and pyxpecto
+    * [[#32a4128](https://github.com/nfdi4plants/ARCtrl/commit/32a4128fcad38d35e695d4dd86db9b19478c1543)] add hashcode tests
+    * [[#cc8c116](https://github.com/nfdi4plants/ARCtrl/commit/cc8c11661e640ce96c73eb136ae4681f710a65b8)] increase json write test timeout
+    * [[#3d00621](https://github.com/nfdi4plants/ARCtrl/commit/3d00621ff1feb76f1a4628f6cdad7a3a7cdb49fd)] small adjustments in isa.json
+    * [[#10c5538](https://github.com/nfdi4plants/ARCtrl/commit/10c55387c420c12804b3f10e737e1d5b1f4b3b43)] cleanup merge of json-ld and thoth.json
+    * [[#58b3d5a](https://github.com/nfdi4plants/ARCtrl/commit/58b3d5a500c36dd09b34cac2cb2cc62efaf8d3d6)] merge json-ld changes into thoth.json update changes and refactor
+    * [[#87ab15d](https://github.com/nfdi4plants/ARCtrl/commit/87ab15df0930389a8a9e6bdd7824a89b1ae30790)] increase json parsing test timeout
+    * [[#90c60b9](https://github.com/nfdi4plants/ARCtrl/commit/90c60b90031b80efd6b23486e7e8061a2b2cc8cb)] rework json encoding
+    * [[#c482c86](https://github.com/nfdi4plants/ARCtrl/commit/c482c86806d5734755a46b9c5b80bbdc32f5e001)] finish thoth conversion
+    * [[#d0a99a3](https://github.com/nfdi4plants/ARCtrl/commit/d0a99a314da50eba37f23aa0e0e5c52f97dbd6ce)] start reworking json towards net Thoth.Json
+    * [[#4f77082](https://github.com/nfdi4plants/ARCtrl/commit/4f77082708909086ec0113e18b2ed8102cd975bd)] Merge pull request #271 from nfdi4plants/jsonld
+    * [[#3f84e17](https://github.com/nfdi4plants/ARCtrl/commit/3f84e17a2ac724a5fc74a33e5352e34b79b1022a)] increase speed of ARC to Json Type conversion
+    * [[#5ceb732](https://github.com/nfdi4plants/ARCtrl/commit/5ceb73209026e512b16461ca6ef6788100bdfe18)] bump to 1.1.0
+* Bugfixes:
+    * [[#890048e](https://github.com/nfdi4plants/ARCtrl/commit/890048e84e06a88b4b957fc46e30113c40c014d0)] set ci fail-fast to false and fix py encoding in windows
+    * [[#3e403cd](https://github.com/nfdi4plants/ARCtrl/commit/3e403cdd687be78f1cbadfafb30b148247800bdd)] small fix to CI
+    * [[#b0186f4](https://github.com/nfdi4plants/ARCtrl/commit/b0186f4c3e28fbf9a6678dde5795e6e0723934c0)] small fix to CI
+    * [[#89aa3c2](https://github.com/nfdi4plants/ARCtrl/commit/89aa3c2a93cb718ca77433e35599d2d22deb2886)] fixed python tests to work on all platforms
+    * [[#4262833](https://github.com/nfdi4plants/ARCtrl/commit/4262833f532c3ed8deedc1ca30b6ad30c04c17b3)] hotfix js webrequest
+    * [[#f75ce4d](https://github.com/nfdi4plants/ARCtrl/commit/f75ce4d3d9abc84600a769d8ead47a6bd2befe8b)] small python tests hotfix
+    * [[#9385d06](https://github.com/nfdi4plants/ARCtrl/commit/9385d06c4bb728edd6c3d95f528fd0a40a19062d)] small fixes in python and setup instructions
+    * [[#b5eaaed](https://github.com/nfdi4plants/ARCtrl/commit/b5eaaedd24cad0a24423aec39dbdf56bbaac14dd)] hotfix fable python hashing of person
+    * [[#556e2bf](https://github.com/nfdi4plants/ARCtrl/commit/556e2bf7f0bd85417ce43078ff3c2357819bef1d)] fix http requests in python
+    * [[#c534acf](https://github.com/nfdi4plants/ARCtrl/commit/c534acf43e16d1e76bb22bbc7e0d1e7134d213e5)] fix and test comment regex handling for python
+    * [[#2540056](https://github.com/nfdi4plants/ARCtrl/commit/2540056063a107fb18ad30b577214e946a6ae48c)] small fix for compressed json io stringtable conversion https://github.com/fable-compiler/Fable/issues/3771
+    * [[#92c72ee](https://github.com/nfdi4plants/ARCtrl/commit/92c72ee0135c712ded59c00c9a0111909ce99c3a)] array fixes in python compilation
+    * [[#b55d360](https://github.com/nfdi4plants/ARCtrl/commit/b55d3608eb73273a84cca756e0662811ff7b6244)] fix and finish up thoth migration
+
 ### 1.1.0+6309e03 (Released 2024-2-15)
 * Additions:
     * [[#544ffdc](https://github.com/nfdi4plants/ARCtrl/commit/544ffdcbcf2e19c12c995e408713b83157cda345)] add some additional information to failing spreadsheet parsers #306

build.cmd

@@ -1,5 +1,6 @@
 @echo off
 
+set PYTHONIOENCODING=utf-8
 dotnet tool restore
 cls 
 dotnet run --project ./build/build.fsproj %*

build/BasicTasks.fs

@@ -7,6 +7,122 @@ open Fake.IO.Globbing.Operators
 
 open ProjectInfo
 
+[<AutoOpen>]
+module Helper =
+
+    open Fake
+    open Fake.Core
+
+    let createProcess exe arg dir =
+        CreateProcess.fromRawCommandLine exe arg
+        |> CreateProcess.withWorkingDirectory dir
+        |> CreateProcess.ensureExitCode
+
+    module Proc =
+
+        module Parallel =
+
+            open System
+
+            let locker = obj()
+
+            let colors = [|  
+                ConsoleColor.DarkYellow
+                ConsoleColor.DarkCyan 
+                ConsoleColor.Magenta
+                ConsoleColor.Blue
+                ConsoleColor.Cyan
+                ConsoleColor.DarkMagenta
+                ConsoleColor.DarkBlue
+                ConsoleColor.Yellow
+            |]
+
+            let print color (colored: string) (line: string) =
+                lock locker
+                    (fun () ->
+                        let currentColor = Console.ForegroundColor
+                        Console.ForegroundColor <- color
+                        Console.Write colored
+                        Console.ForegroundColor <- currentColor
+                        Console.WriteLine line)
+
+            let onStdout index name (line: string) =
+                let color = colors.[index % colors.Length]
+                if isNull line then
+                    print color $"{name}: --- END ---" ""
+                else if String.isNotNullOrEmpty line then
+                    print color $"{name}: " line
+
+            let onStderr name (line: string) =
+                let color = ConsoleColor.Red
+                if isNull line |> not then
+                    print color $"{name}: " line
+
+            let redirect (index, (name, createProcess)) =
+                createProcess
+                |> CreateProcess.redirectOutputIfNotRedirected
+                |> CreateProcess.withOutputEvents (onStdout index name) (onStderr name)
+
+            let printStarting indexed =
+                for (index, (name, c: CreateProcess<_>)) in indexed do
+                    let color = colors.[index % colors.Length]
+                    let wd =
+                        c.WorkingDirectory
+                        |> Option.defaultValue ""
+                    let exe = c.Command.Executable
+                    let args = c.Command.Arguments.ToStartInfo
+                    print color $"{name}: {wd}> {exe} {args}" ""
+
+            let run cs =
+                cs
+                |> Seq.toArray
+                |> Array.indexed
+                |> fun x -> printStarting x; x
+                |> Array.map redirect
+                |> Array.Parallel.map Proc.run
+
+    let dotnet = createProcess "dotnet"
+
+    let npx =
+        let npmPath =
+            match ProcessUtils.tryFindFileOnPath "npx" with
+            | Some path -> path
+            | None ->
+                "npm was not found in path. Please install it and make sure it's available from your path. " +
+                "See https://safe-stack.github.io/docs/quickstart/#install-pre-requisites for more info"
+                |> failwith
+
+        createProcess npmPath
+
+    let npm =
+        let npmPath =
+            match ProcessUtils.tryFindFileOnPath "npm" with
+            | Some path -> path
+            | None ->
+                "npm was not found in path. Please install it and make sure it's available from your path. " +
+                "See https://safe-stack.github.io/docs/quickstart/#install-pre-requisites for more info"
+                |> failwith
+
+        createProcess npmPath
+
+    let python = 
+        if System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.Windows) then
+            Fake.Core.Trace.log "Detected Windows System."
+            createProcess (__SOURCE_DIRECTORY__.Replace(@"\build",@"\.venv\Scripts\python.exe"))
+        else
+            Fake.Core.Trace.log "Detected Unix System."
+            createProcess (__SOURCE_DIRECTORY__.Replace(@"/build",@"/.venv/bin/python"))
+
+    let run proc arg dir =
+        proc arg dir
+        |> Proc.run
+        |> ignore
+
+    let runParallel processes =
+        processes
+        |> Proc.Parallel.run
+        |> ignore
+
 let setPrereleaseTag = BuildTask.create "SetPrereleaseTag" [] {
     printfn "Please enter pre-release package suffix"
     let suffix = System.Console.ReadLine()
@@ -21,7 +137,7 @@ let clean = BuildTask.create "Clean" [] {
     ++ "tests/**/bin"
     ++ "tests/**/obj"
     ++ "dist"
-    ++ ProjectInfo.pkgDir
+    ++ ProjectInfo.netPkgDir
     |> Shell.cleanDirs 
 }
 

build/Build.fs

@@ -21,7 +21,7 @@ open ReleaseTasks
 let _release = 
     BuildTask.createEmpty 
         "Release" 
-        [clean; build; runTests; pack; createTag; publishNuget; publishNPM]
+        [clean; build; runTests; pack; createTag; publishNuget; publishNPM; publishPyPi]
 
 /// Full release of nuget package for the prerelease version.
 let _preRelease = 

build/Build.fsproj

@@ -12,6 +12,7 @@
     <Compile Include="ReleaseNotesTasks.fs" />
     <Compile Include="BasicTasks.fs" />
     <Compile Include="GenerateIndexJs.fs" />
+    <Compile Include="GenerateIndexPy.fs" />
     <Compile Include="TestTasks.fs" />
     <Compile Include="PackageTasks.fs" />
     <Compile Include="ReleaseTasks.fs" />

build/GenerateIndexPy.fs

@@ -0,0 +1,49 @@
+module GenerateIndexPy
+
+open System
+open System.IO
+open System.Text.RegularExpressions
+
+
+open System.Text
+
+let createImportStatement path (classes : string []) =
+    let classes = classes |> Array.reduce (fun acc x -> acc + ", " + x)
+    sprintf "from %s import %s" path classes
+
+let writePyIndexfile (path: string) (content: string) =
+    let filePath = Path.Combine(path, "arctrl.py")
+    File.WriteAllText(filePath, content)
+
+let generateIndexFileContent (classes : (string*string) []) =
+    classes
+    |> Array.groupBy fst
+    |> Array.map (fun (p,a) -> createImportStatement p (a |> Array.map snd))
+
+let classes =
+    [|
+        "__future__", "annotations"
+        "collections.abc", "Callable"
+        "typing", "Any"
+        ".ISA.ISA.JsonTypes.comment", "Comment"
+        ".ISA.ISA.JsonTypes.ontology_annotation","OntologyAnnotation"; 
+        ".ISA.ISA.JsonTypes.person", "Person";
+        ".ISA.ISA.JsonTypes.publication", "Publication";
+        ".ISA.ISA.ArcTypes.composite_header", "IOType"
+        ".ISA.ISA.ArcTypes.composite_header", "CompositeHeader"; 
+        ".ISA.ISA.ArcTypes.composite_cell", "CompositeCell"
+        ".ISA.ISA.ArcTypes.composite_column", "CompositeColumn"
+        ".ISA.ISA.ArcTypes.arc_table", "ArcTable"
+        ".ISA.ISA.ArcTypes.arc_types", "ArcAssay"; 
+        ".ISA.ISA.ArcTypes.arc_types", "ArcStudy"; 
+        ".ISA.ISA.ArcTypes.arc_types", "ArcInvestigation"; 
+        ".Templates.template", "Template"
+        ".Templates.templates", "Templates"
+        ".Templates.template", "Organisation"
+        ".arc","ARC"
+    |]
+
+let ARCtrl_generate (rootPath: string) = 
+    generateIndexFileContent classes
+    |> Array.reduce (fun a b -> a + "\n" + b)
+    |> writePyIndexfile rootPath