From 32a4128fcad38d35e695d4dd86db9b19478c1543 Mon Sep 17 00:00:00 2001 From: Heinrich Lukas Weil Date: Fri, 8 Dec 2023 13:56:35 +0100 Subject: [PATCH 01/25] add hashcode tests --- tests/ISA/ISA.Tests/ArcTable.Tests.fs | 14 +++++ tests/ISA/ISA.Tests/CompositeCell.Tests.fs | 18 ++++++ tests/ISA/ISA.Tests/CompositeHeader.Tests.fs | 62 ++++++++++++++++++++ tests/ISA/ISA.Tests/DataModel.Tests.fs | 23 ++++++++ 4 files changed, 117 insertions(+) diff --git a/tests/ISA/ISA.Tests/ArcTable.Tests.fs b/tests/ISA/ISA.Tests/ArcTable.Tests.fs index 741c9679..56bc2299 100644 --- a/tests/ISA/ISA.Tests/ArcTable.Tests.fs +++ b/tests/ISA/ISA.Tests/ArcTable.Tests.fs @@ -78,6 +78,20 @@ let private tests_member = ] let private tests_GetHashCode = testList "GetHashCode" [ + testCase "Headers" <| fun _ -> + let table1 = create_testTable() + let table2 = create_testTable() + let hash1 = table1.GetHeadersHashCode() + let hash2 = table2.GetHeadersHashCode() + Expect.sequenceEqual table1.Headers table2.Headers "Table" + Expect.equal hash1 hash2 "Headers HashCode" + testCase "Value" <| fun _ -> + let table1 = create_testTable() + let table2 = create_testTable() + let hash1 = table1.GetValuesHashCode() + let hash2 = table2.GetValuesHashCode() + //Expect.sequenceEqual table1.Headers table2.Headers "Table" + Expect.equal hash1 hash2 "Values HashCode" testCase "GetHashCode" <| fun _ -> let table1 = create_testTable() let table2 = create_testTable() diff --git a/tests/ISA/ISA.Tests/CompositeCell.Tests.fs b/tests/ISA/ISA.Tests/CompositeCell.Tests.fs index 8db5a497..cb34887e 100644 --- a/tests/ISA/ISA.Tests/CompositeCell.Tests.fs +++ b/tests/ISA/ISA.Tests/CompositeCell.Tests.fs @@ -175,6 +175,23 @@ let private tests_UIHelper = testList "UIHelper" [ ] ] +let private tests_GetHashCode = testList "GetHashCode" [ + testList "GetHashCode" [ + testCase "Term" <| fun _ -> + let cc = CompositeCell.createTermFromString("TestTerm", "TEST", "TEST:00001") + cc.GetHashCode() |> ignore + Expect.isTrue true " " + testCase "Unitized" <| fun _ -> + let cc = CompositeCell.createUnitizedFromString("12", "TestTerm", "TEST", "TEST:00001") + cc.GetHashCode() |> ignore + Expect.isTrue true " " + testCase "FreeText" <| fun _ -> + let cc = CompositeCell.createFreeText("TestTerm") + cc.GetHashCode() |> ignore + Expect.isTrue true " " + ] +] + let main = testList "CompositeCell" [ tests_cellConverter @@ -182,4 +199,5 @@ let main = tests_ToString tests_GetContent tests_UIHelper + tests_GetHashCode ] \ No newline at end of file diff --git a/tests/ISA/ISA.Tests/CompositeHeader.Tests.fs b/tests/ISA/ISA.Tests/CompositeHeader.Tests.fs index 51f57edc..ba1afe70 100644 --- a/tests/ISA/ISA.Tests/CompositeHeader.Tests.fs +++ b/tests/ISA/ISA.Tests/CompositeHeader.Tests.fs @@ -329,10 +329,72 @@ let tests_ToTerm = testList "ToTerm" [ yield! allHeaders |> List.map testToTerm ] +let tests_GetHashCode = testList "GetHashCode" [ + ftestCase "SimpleParamEqual" (fun () -> + let oa1 = OntologyAnnotation.fromString("MyTerm",tan = "LOL:123") + let p1 = CompositeHeader.Parameter(oa1) + let oa2 = OntologyAnnotation.fromString("MyTerm",tan = "LOL:123") + let p2 = CompositeHeader.Parameter(oa2) + let h1 = p1.GetHashCode() + let h2 = p2.GetHashCode() + Expect.equal h1 h2 "Same Param Headers should have equal hash" + ) + ftestCase "SimpleParamUnequal" (fun () -> + let oa1 = OntologyAnnotation.fromString("MyTerm",tan = "LOL:123") + let p1 = CompositeHeader.Parameter(oa1) + let oa2 = OntologyAnnotation.fromString("YourTerm",tan = "ROFL:321") + let p2 = CompositeHeader.Parameter(oa2) + let h1 = p1.GetHashCode() + let h2 = p2.GetHashCode() + Expect.notEqual h1 h2 "Different Param Headers should have unequal hash" + ) + ftestCase "EmptyParam" (fun () -> + let oa1 = OntologyAnnotation.empty + let p1 = CompositeHeader.Parameter(oa1) + let oa2 = OntologyAnnotation.empty + let p2 = CompositeHeader.Parameter(oa2) + let h1 = p1.GetHashCode() + let h2 = p2.GetHashCode() + Expect.equal h1 h2 "Empty Param Headers should be equal" + ) + ftestCase "EmptyParamAndFactor" (fun () -> + let oa1 = OntologyAnnotation.empty + let p1 = CompositeHeader.Parameter(oa1) + let oa2 = OntologyAnnotation.empty + let f2 = CompositeHeader.Factor(oa2) + let h1 = p1.GetHashCode() + let h2 = f2.GetHashCode() + Expect.notEqual h1 h2 "Empty Param Header should be unequal to empty factor" + ) + ftestCase "InputSameType" (fun () -> + let i1 = CompositeHeader.Input(IOType.Sample) + let i2 = CompositeHeader.Input(IOType.Sample) + let h1 = i1.GetHashCode() + let h2 = i2.GetHashCode() + Expect.equal h1 h2 "Input Sample Headers should be equal" + ) + ftestCase "InputDifferentType" (fun () -> + let i1 = CompositeHeader.Input(IOType.Sample) + let i2 = CompositeHeader.Input(IOType.RawDataFile) + let h1 = i1.GetHashCode() + let h2 = i2.GetHashCode() + Expect.notEqual h1 h2 "Input Sample Header should be unequal to Input Data" + ) + ftestCase "InputOutputSameType" (fun () -> + let i = CompositeHeader.Input(IOType.Sample) + let o = CompositeHeader.Output(IOType.Sample) + let h1 = i.GetHashCode() + let h2 = o.GetHashCode() + Expect.notEqual h1 h2 "Input Sample Header should be unequal to Output Sample" + ) +] + + let main = testList "CompositeHeader" [ tests_iotype tests_jsHelper tests_compositeHeader tests_ToTerm + tests_GetHashCode ] \ No newline at end of file diff --git a/tests/ISA/ISA.Tests/DataModel.Tests.fs b/tests/ISA/ISA.Tests/DataModel.Tests.fs index fc47e142..aee2d8af 100644 --- a/tests/ISA/ISA.Tests/DataModel.Tests.fs +++ b/tests/ISA/ISA.Tests/DataModel.Tests.fs @@ -260,6 +260,29 @@ let ontologyAnnotationTests = let uri = "http://purl.obolibrary.org/obo/EFO_0000721" let otherParseable = "http://www.ebi.ac.uk/efo/EFO_0000721" let other = "Unparseable" + testList "GetHashCode" [ + ftestCase "Empty" (fun () -> + let oa1 = OntologyAnnotation.fromString() + let oa2 = OntologyAnnotation.fromString() + let h1 = oa1.GetHashCode() + let h2 = oa2.GetHashCode() + Expect.equal h1 h2 "Hashes should be equal" + ) + ftestCase "Equal" (fun () -> + let oa1 = OntologyAnnotation.fromString("MyOntology",tsr = "EFO",tan = uri) + let oa2 = OntologyAnnotation.fromString("MyOntology",tsr = "EFO",tan = uri) + let h1 = oa1.GetHashCode() + let h2 = oa2.GetHashCode() + Expect.equal h1 h2 "Hashes should be equal" + ) + ftestCase "Different" (fun () -> + let oa1 = OntologyAnnotation.fromString("MyOntology",tsr = "EFO",tan = uri) + let oa2 = OntologyAnnotation.fromString("YourOntology",tsr = "NCBI",tan = "http://purl.obolibrary.org/obo/NCBI_0000123") + let h1 = oa1.GetHashCode() + let h2 = oa2.GetHashCode() + Expect.notEqual h1 h2 "Hashes should not be equal" + ) + ] testList "fromString" [ testCase "FromShort" (fun () -> From 92c72ee0135c712ded59c00c9a0111909ce99c3a Mon Sep 17 00:00:00 2001 From: Heinrich Lukas Weil Date: Wed, 13 Dec 2023 09:54:29 +0100 Subject: [PATCH 02/25] array fixes in python compilation --- .config/dotnet-tools.json | 2 +- pytests/arc_table_tests.ipynb | 238 +++++++++++++++++++++++++++ pytests/composite_header_tests.ipynb | 238 +++++++++++++++++++++++++++ src/ISA/ISA/ArcTypes/ArcTable.fs | 6 +- src/ISA/ISA/Fable.fs | 33 +++- testpy.cmd | 2 + 6 files changed, 514 insertions(+), 5 deletions(-) create mode 100644 pytests/arc_table_tests.ipynb create mode 100644 pytests/composite_header_tests.ipynb create mode 100644 testpy.cmd diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 6406f8ce..0d4a2909 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "fable": { - "version": "4.6.0", + "version": "4.7.0", "commands": [ "fable" ] diff --git a/pytests/arc_table_tests.ipynb b/pytests/arc_table_tests.ipynb new file mode 100644 index 00000000..19762e5e --- /dev/null +++ b/pytests/arc_table_tests.ipynb @@ -0,0 +1,238 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "dotnet_interactive": { + "language": "fsharp" + }, + "polyglot_notebook": { + "kernelName": "fsharp" + } + }, + "outputs": [], + "source": [ + "from __future__ import annotations\n", + "from array import array as array_3\n", + "from typing import (Any, Callable, TypeVar)\n", + "from fable_modules.fable_library.array_ import (initialize, add_range_in_place, map as map_1, indexed, reverse as reverse_1, append as append_2, map_indexed, take, fill)\n", + "from fable_modules.fable_library.list import (FSharpList, of_array, is_empty, iterate, append as append_1, cons, singleton as singleton_1)\n", + "from fable_modules.fable_library.map import of_list\n", + "from fable_modules.fable_library.map_util import (add_to_dict, get_item_from_dict)\n", + "from fable_modules.fable_library.mutable_map import Dictionary\n", + "from fable_modules.fable_library.option import value as value_8\n", + "from fable_modules.fable_library.range import range_big_int\n", + "from fable_modules.fable_library.seq import (to_list, delay, collect, singleton, append, to_array, map)\n", + "from fable_modules.fable_library.string_ import (to_text, printf)\n", + "from fable_modules.fable_library.system_text import (StringBuilder__ctor, StringBuilder__AppendLine_Z721C83C5)\n", + "from fable_modules.fable_library.types import (Array, to_string)\n", + "from fable_modules.fable_library.util import (IEnumerable_1, int32_to_string, assert_equal, equals as equals_4, safe_hash, to_enumerable, ignore, compare_arrays, equal_arrays, array_hash)\n", + "from fable_modules.fable_pyxpecto.pyxpecto import Helper_expectError\n", + "from fable_modules.fable_pyxpecto.pyxpecto import TestCase\n", + "from src.ISA.ISA.ArcTypes.arc_table_aux import try_find_duplicate_unique_in_array\n", + "from src.ISA.ISA.ArcTypes.arc_table import ArcTable\n", + "from src.ISA.ISA.ArcTypes.arc_table_aux import (SanityChecks_validateColumnIndex, SanityChecks_validateRowIndex, try_find_duplicate_unique)\n", + "from src.ISA.ISA.ArcTypes.composite_cell import CompositeCell\n", + "from src.ISA.ISA.ArcTypes.composite_column import CompositeColumn\n", + "from src.ISA.ISA.ArcTypes.composite_header import (IOType, CompositeHeader)\n", + "from src.ISA.ISA.ArcTypes.identifier_setters import set_arc_table_name\n", + "from src.ISA.ISA.JsonTypes.ontology_annotation import OntologyAnnotation\n", + "from library import (Test_testList, Test_testCase, Expect_sequenceEqual, Expect_isTrue, Expect_notEqual, Expect_throws, Expect_isNone, Expect_isSome, Expect_containsAll, Expect_isFalse)\n", + "\n", + "__A = TypeVar(\"__A\")\n", + "\n", + "TableName: str = \"Test\"\n", + "\n", + "oa_species: OntologyAnnotation = OntologyAnnotation.from_string(\"species\", \"GO\", \"GO:0123456\")\n", + "\n", + "oa_chlamy: OntologyAnnotation = OntologyAnnotation.from_string(\"Chlamy\", \"NCBI\", \"NCBI:0123456\")\n", + "\n", + "oa_instrument_model: OntologyAnnotation = OntologyAnnotation.from_string(\"instrument model\", \"MS\", \"MS:0123456\")\n", + "\n", + "oa_sciexinstrument_model: OntologyAnnotation = OntologyAnnotation.from_string(\"SCIEX instrument model\", \"MS\", \"MS:654321\")\n", + "\n", + "oa_temperature: OntologyAnnotation = OntologyAnnotation.from_string(\"temperature\", \"NCIT\", \"NCIT:0123210\")\n", + "\n", + "def table_values_printable(table: ArcTable) -> FSharpList[str]:\n", + " def _arrow1027(__unit: None=None, table: Any=table) -> IEnumerable_1[str]:\n", + " def _arrow1026(match_value: Any) -> IEnumerable_1[str]:\n", + " active_pattern_result: tuple[tuple[int, int], CompositeCell] = match_value\n", + " return singleton((((((\"(\" + str(active_pattern_result[0][0])) + \",\") + str(active_pattern_result[0][1])) + \") \") + str(active_pattern_result[1])) + \"\")\n", + "\n", + " return collect(_arrow1026, table.Values)\n", + "\n", + " return to_list(delay(_arrow1027))\n", + "\n", + "\n", + "def create_cells_free_text(pretext: Any, count: int) -> Array[CompositeCell]:\n", + " def _arrow1030(i: int, pretext: Any=pretext, count: Any=count) -> CompositeCell:\n", + " return CompositeCell.create_free_text((((\"\" + str(pretext)) + \"_\") + str(i)) + \"\")\n", + "\n", + " return initialize(count, _arrow1030, None)\n", + "\n", + "\n", + "def create_cells_term(count: int) -> Array[CompositeCell]:\n", + " def _arrow1032(_arg: int, count: Any=count) -> CompositeCell:\n", + " return CompositeCell.create_term(oa_sciexinstrument_model)\n", + "\n", + " return initialize(count, _arrow1032, None)\n", + "\n", + "\n", + "def create_cells_unitized(count: int) -> Array[CompositeCell]:\n", + " def _arrow1035(i: int, count: Any=count) -> CompositeCell:\n", + " return CompositeCell.create_unitized(int32_to_string(i), OntologyAnnotation.empty())\n", + "\n", + " return initialize(count, _arrow1035, None)\n", + "\n", + "\n", + "column_input: CompositeColumn = CompositeColumn.create(CompositeHeader(11, IOType(0)), create_cells_free_text(\"Source\", 5))\n", + "\n", + "column_output: CompositeColumn = CompositeColumn.create(CompositeHeader(12, IOType(1)), create_cells_free_text(\"Sample\", 5))\n", + "\n", + "column_component: CompositeColumn = CompositeColumn.create(CompositeHeader(0, oa_instrument_model), create_cells_term(5))\n", + "\n", + "column_param: CompositeColumn = CompositeColumn.create(CompositeHeader(3, OntologyAnnotation.empty()), create_cells_unitized(5))\n", + "\n", + "def create_test_table(__unit: None=None) -> ArcTable:\n", + " t: ArcTable = ArcTable.init(TableName)\n", + " columns: Array[CompositeColumn] = [column_input, column_output, column_param, column_component, column_param]\n", + " t.AddColumns(columns)\n", + " return t\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "table1: ArcTable = create_test_table()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "8821596988035483298" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# table1.GetHeadersHashCode()\n", + "from src.ISA.ISA.helper import HashCodes_boxHashArray\n", + "\n", + "# HashCodes_boxHashArray(list(table1.Headers))\n", + "\n", + "from fable_modules.fable_library.util import (get_enumerator, dispose, equals, structural_hash, identity_hash, number_hash, IEnumerable_1, ignore, is_iterable)\n", + "_A = TypeVar(\"_A\")\n", + "\n", + "copy_of_struct: _A = table1.Headers[0]\n", + "identity_hash(copy_of_struct)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "ename": "TypeError", + "evalue": "unhashable type: 'OntologyAnnotation'", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mTypeError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32mc:\\Users\\HLWei\\source\\repos\\ARC_tools\\ARCtrl\\pytests\\arc_table_tests.ipynb Cell 4\u001b[0m line \u001b[0;36m3\n\u001b[0;32m 33\u001b[0m identity_hash(oa)\n\u001b[0;32m 35\u001b[0m p \u001b[39m=\u001b[39m CompositeHeader\u001b[39m.\u001b[39mparameter(oa)\n\u001b[1;32m---> 36\u001b[0m identity_hash(p)\n", + "File \u001b[1;32mc:\\Users\\HLWei\\source\\repos\\ARC_tools\\ARCtrl\\pytests\\fable_modules\\fable_library\\util.py:2642\u001b[0m, in \u001b[0;36midentity_hash\u001b[1;34m(x)\u001b[0m\n\u001b[0;32m 2639\u001b[0m \u001b[39mreturn\u001b[39;00m x\u001b[39m.\u001b[39mGetHashCode()\n\u001b[0;32m 2641\u001b[0m \u001b[39mif\u001b[39;00m is_hashable_py(x):\n\u001b[1;32m-> 2642\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mhash\u001b[39m(x)\n\u001b[0;32m 2644\u001b[0m \u001b[39mreturn\u001b[39;00m physical_hash(x)\n", + "File \u001b[1;32mc:\\Users\\HLWei\\source\\repos\\ARC_tools\\ARCtrl\\pytests\\fable_modules\\fable_library\\types.py:85\u001b[0m, in \u001b[0;36mUnion.__hash__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 83\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__hash__\u001b[39m(\u001b[39mself\u001b[39m) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mint\u001b[39m:\n\u001b[0;32m 84\u001b[0m hashes \u001b[39m=\u001b[39m \u001b[39mmap\u001b[39m(\u001b[39mhash\u001b[39m, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mfields)\n\u001b[1;32m---> 85\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mhash\u001b[39m((\u001b[39mhash\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtag), \u001b[39m*\u001b[39mhashes))\n", + "\u001b[1;31mTypeError\u001b[0m: unhashable type: 'OntologyAnnotation'" + ] + } + ], + "source": [ + "from fable_modules.fable_library.array_ import (contains, fold, partition, exactly_one)\n", + "\n", + "a = table1.Headers\n", + "\n", + "# def folder(acc: int, o: _A, a: Any=a) -> int:\n", + "# def _arrow67(__unit: None=None, acc: Any=acc, o: Any=o) -> int:\n", + "# copy_of_struct: _A = o\n", + "# return identity_hash(copy_of_struct)\n", + "\n", + "# return ((-1640531527 + _arrow67()) + (acc << 6)) + (acc >> 2)\n", + "\n", + "# failt\n", + "# fold(folder, 0, a)\n", + "\n", + "\n", + "# identity_hash(table1.Headers[0])\n", + "# identity_hash(table1.Headers[1])\n", + "# failt\n", + "# identity_hash(table1.Headers[2])\n", + "# identity_hash(table1.Headers[3])\n", + "# identity_hash(table1.Headers[4])\n", + "\n", + "\n", + "# failt\n", + "# identity_hash(column_param.Header)\n", + "\n", + "\n", + "\n", + "table1.Headers[2]\n", + "\n", + "oa = OntologyAnnotation.empty()\n", + "\n", + "identity_hash(oa)\n", + "\n", + "# p = CompositeHeader.parameter(oa)\n", + "\n", + "# identity_hash(p)\n", + "\n", + "\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.0" + }, + "polyglot_notebook": { + "kernelInfo": { + "defaultKernelName": "fsharp", + "items": [ + { + "aliases": [], + "languageName": "fsharp", + "name": "fsharp" + } + ] + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/pytests/composite_header_tests.ipynb b/pytests/composite_header_tests.ipynb new file mode 100644 index 00000000..19762e5e --- /dev/null +++ b/pytests/composite_header_tests.ipynb @@ -0,0 +1,238 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "dotnet_interactive": { + "language": "fsharp" + }, + "polyglot_notebook": { + "kernelName": "fsharp" + } + }, + "outputs": [], + "source": [ + "from __future__ import annotations\n", + "from array import array as array_3\n", + "from typing import (Any, Callable, TypeVar)\n", + "from fable_modules.fable_library.array_ import (initialize, add_range_in_place, map as map_1, indexed, reverse as reverse_1, append as append_2, map_indexed, take, fill)\n", + "from fable_modules.fable_library.list import (FSharpList, of_array, is_empty, iterate, append as append_1, cons, singleton as singleton_1)\n", + "from fable_modules.fable_library.map import of_list\n", + "from fable_modules.fable_library.map_util import (add_to_dict, get_item_from_dict)\n", + "from fable_modules.fable_library.mutable_map import Dictionary\n", + "from fable_modules.fable_library.option import value as value_8\n", + "from fable_modules.fable_library.range import range_big_int\n", + "from fable_modules.fable_library.seq import (to_list, delay, collect, singleton, append, to_array, map)\n", + "from fable_modules.fable_library.string_ import (to_text, printf)\n", + "from fable_modules.fable_library.system_text import (StringBuilder__ctor, StringBuilder__AppendLine_Z721C83C5)\n", + "from fable_modules.fable_library.types import (Array, to_string)\n", + "from fable_modules.fable_library.util import (IEnumerable_1, int32_to_string, assert_equal, equals as equals_4, safe_hash, to_enumerable, ignore, compare_arrays, equal_arrays, array_hash)\n", + "from fable_modules.fable_pyxpecto.pyxpecto import Helper_expectError\n", + "from fable_modules.fable_pyxpecto.pyxpecto import TestCase\n", + "from src.ISA.ISA.ArcTypes.arc_table_aux import try_find_duplicate_unique_in_array\n", + "from src.ISA.ISA.ArcTypes.arc_table import ArcTable\n", + "from src.ISA.ISA.ArcTypes.arc_table_aux import (SanityChecks_validateColumnIndex, SanityChecks_validateRowIndex, try_find_duplicate_unique)\n", + "from src.ISA.ISA.ArcTypes.composite_cell import CompositeCell\n", + "from src.ISA.ISA.ArcTypes.composite_column import CompositeColumn\n", + "from src.ISA.ISA.ArcTypes.composite_header import (IOType, CompositeHeader)\n", + "from src.ISA.ISA.ArcTypes.identifier_setters import set_arc_table_name\n", + "from src.ISA.ISA.JsonTypes.ontology_annotation import OntologyAnnotation\n", + "from library import (Test_testList, Test_testCase, Expect_sequenceEqual, Expect_isTrue, Expect_notEqual, Expect_throws, Expect_isNone, Expect_isSome, Expect_containsAll, Expect_isFalse)\n", + "\n", + "__A = TypeVar(\"__A\")\n", + "\n", + "TableName: str = \"Test\"\n", + "\n", + "oa_species: OntologyAnnotation = OntologyAnnotation.from_string(\"species\", \"GO\", \"GO:0123456\")\n", + "\n", + "oa_chlamy: OntologyAnnotation = OntologyAnnotation.from_string(\"Chlamy\", \"NCBI\", \"NCBI:0123456\")\n", + "\n", + "oa_instrument_model: OntologyAnnotation = OntologyAnnotation.from_string(\"instrument model\", \"MS\", \"MS:0123456\")\n", + "\n", + "oa_sciexinstrument_model: OntologyAnnotation = OntologyAnnotation.from_string(\"SCIEX instrument model\", \"MS\", \"MS:654321\")\n", + "\n", + "oa_temperature: OntologyAnnotation = OntologyAnnotation.from_string(\"temperature\", \"NCIT\", \"NCIT:0123210\")\n", + "\n", + "def table_values_printable(table: ArcTable) -> FSharpList[str]:\n", + " def _arrow1027(__unit: None=None, table: Any=table) -> IEnumerable_1[str]:\n", + " def _arrow1026(match_value: Any) -> IEnumerable_1[str]:\n", + " active_pattern_result: tuple[tuple[int, int], CompositeCell] = match_value\n", + " return singleton((((((\"(\" + str(active_pattern_result[0][0])) + \",\") + str(active_pattern_result[0][1])) + \") \") + str(active_pattern_result[1])) + \"\")\n", + "\n", + " return collect(_arrow1026, table.Values)\n", + "\n", + " return to_list(delay(_arrow1027))\n", + "\n", + "\n", + "def create_cells_free_text(pretext: Any, count: int) -> Array[CompositeCell]:\n", + " def _arrow1030(i: int, pretext: Any=pretext, count: Any=count) -> CompositeCell:\n", + " return CompositeCell.create_free_text((((\"\" + str(pretext)) + \"_\") + str(i)) + \"\")\n", + "\n", + " return initialize(count, _arrow1030, None)\n", + "\n", + "\n", + "def create_cells_term(count: int) -> Array[CompositeCell]:\n", + " def _arrow1032(_arg: int, count: Any=count) -> CompositeCell:\n", + " return CompositeCell.create_term(oa_sciexinstrument_model)\n", + "\n", + " return initialize(count, _arrow1032, None)\n", + "\n", + "\n", + "def create_cells_unitized(count: int) -> Array[CompositeCell]:\n", + " def _arrow1035(i: int, count: Any=count) -> CompositeCell:\n", + " return CompositeCell.create_unitized(int32_to_string(i), OntologyAnnotation.empty())\n", + "\n", + " return initialize(count, _arrow1035, None)\n", + "\n", + "\n", + "column_input: CompositeColumn = CompositeColumn.create(CompositeHeader(11, IOType(0)), create_cells_free_text(\"Source\", 5))\n", + "\n", + "column_output: CompositeColumn = CompositeColumn.create(CompositeHeader(12, IOType(1)), create_cells_free_text(\"Sample\", 5))\n", + "\n", + "column_component: CompositeColumn = CompositeColumn.create(CompositeHeader(0, oa_instrument_model), create_cells_term(5))\n", + "\n", + "column_param: CompositeColumn = CompositeColumn.create(CompositeHeader(3, OntologyAnnotation.empty()), create_cells_unitized(5))\n", + "\n", + "def create_test_table(__unit: None=None) -> ArcTable:\n", + " t: ArcTable = ArcTable.init(TableName)\n", + " columns: Array[CompositeColumn] = [column_input, column_output, column_param, column_component, column_param]\n", + " t.AddColumns(columns)\n", + " return t\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "table1: ArcTable = create_test_table()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "8821596988035483298" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# table1.GetHeadersHashCode()\n", + "from src.ISA.ISA.helper import HashCodes_boxHashArray\n", + "\n", + "# HashCodes_boxHashArray(list(table1.Headers))\n", + "\n", + "from fable_modules.fable_library.util import (get_enumerator, dispose, equals, structural_hash, identity_hash, number_hash, IEnumerable_1, ignore, is_iterable)\n", + "_A = TypeVar(\"_A\")\n", + "\n", + "copy_of_struct: _A = table1.Headers[0]\n", + "identity_hash(copy_of_struct)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "ename": "TypeError", + "evalue": "unhashable type: 'OntologyAnnotation'", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mTypeError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32mc:\\Users\\HLWei\\source\\repos\\ARC_tools\\ARCtrl\\pytests\\arc_table_tests.ipynb Cell 4\u001b[0m line \u001b[0;36m3\n\u001b[0;32m 33\u001b[0m identity_hash(oa)\n\u001b[0;32m 35\u001b[0m p \u001b[39m=\u001b[39m CompositeHeader\u001b[39m.\u001b[39mparameter(oa)\n\u001b[1;32m---> 36\u001b[0m identity_hash(p)\n", + "File \u001b[1;32mc:\\Users\\HLWei\\source\\repos\\ARC_tools\\ARCtrl\\pytests\\fable_modules\\fable_library\\util.py:2642\u001b[0m, in \u001b[0;36midentity_hash\u001b[1;34m(x)\u001b[0m\n\u001b[0;32m 2639\u001b[0m \u001b[39mreturn\u001b[39;00m x\u001b[39m.\u001b[39mGetHashCode()\n\u001b[0;32m 2641\u001b[0m \u001b[39mif\u001b[39;00m is_hashable_py(x):\n\u001b[1;32m-> 2642\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mhash\u001b[39m(x)\n\u001b[0;32m 2644\u001b[0m \u001b[39mreturn\u001b[39;00m physical_hash(x)\n", + "File \u001b[1;32mc:\\Users\\HLWei\\source\\repos\\ARC_tools\\ARCtrl\\pytests\\fable_modules\\fable_library\\types.py:85\u001b[0m, in \u001b[0;36mUnion.__hash__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 83\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__hash__\u001b[39m(\u001b[39mself\u001b[39m) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mint\u001b[39m:\n\u001b[0;32m 84\u001b[0m hashes \u001b[39m=\u001b[39m \u001b[39mmap\u001b[39m(\u001b[39mhash\u001b[39m, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mfields)\n\u001b[1;32m---> 85\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mhash\u001b[39m((\u001b[39mhash\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtag), \u001b[39m*\u001b[39mhashes))\n", + "\u001b[1;31mTypeError\u001b[0m: unhashable type: 'OntologyAnnotation'" + ] + } + ], + "source": [ + "from fable_modules.fable_library.array_ import (contains, fold, partition, exactly_one)\n", + "\n", + "a = table1.Headers\n", + "\n", + "# def folder(acc: int, o: _A, a: Any=a) -> int:\n", + "# def _arrow67(__unit: None=None, acc: Any=acc, o: Any=o) -> int:\n", + "# copy_of_struct: _A = o\n", + "# return identity_hash(copy_of_struct)\n", + "\n", + "# return ((-1640531527 + _arrow67()) + (acc << 6)) + (acc >> 2)\n", + "\n", + "# failt\n", + "# fold(folder, 0, a)\n", + "\n", + "\n", + "# identity_hash(table1.Headers[0])\n", + "# identity_hash(table1.Headers[1])\n", + "# failt\n", + "# identity_hash(table1.Headers[2])\n", + "# identity_hash(table1.Headers[3])\n", + "# identity_hash(table1.Headers[4])\n", + "\n", + "\n", + "# failt\n", + "# identity_hash(column_param.Header)\n", + "\n", + "\n", + "\n", + "table1.Headers[2]\n", + "\n", + "oa = OntologyAnnotation.empty()\n", + "\n", + "identity_hash(oa)\n", + "\n", + "# p = CompositeHeader.parameter(oa)\n", + "\n", + "# identity_hash(p)\n", + "\n", + "\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.0" + }, + "polyglot_notebook": { + "kernelInfo": { + "defaultKernelName": "fsharp", + "items": [ + { + "aliases": [], + "languageName": "fsharp", + "name": "fsharp" + } + ] + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/src/ISA/ISA/ArcTypes/ArcTable.fs b/src/ISA/ISA/ArcTypes/ArcTable.fs index 0c43d20c..afe35ddb 100644 --- a/src/ISA/ISA/ArcTypes/ArcTable.fs +++ b/src/ISA/ISA/ArcTypes/ArcTable.fs @@ -271,7 +271,7 @@ type ArcTable(name: string, headers: ResizeArray, values: Syste // Sanity check here too, to avoid removing things from mutable to fail in the middle Array.iter (fun index -> SanityChecks.validateColumnIndex index this.ColumnCount false) indexArr /// go from highest to lowest so no wrong column gets removed after index shift - let indexArr = indexArr |> Array.sortDescending + let indexArr = indexArr |> ARCtrl.ISA.Fable.Array.sortDescending Array.iter (fun index -> this.RemoveColumn index) indexArr static member removeColumns(indexArr:int []) = @@ -476,7 +476,7 @@ type ArcTable(name: string, headers: ResizeArray, values: Syste // Sanity check here too, to avoid removing things from mutable to fail in the middle Array.iter (fun index -> ArcTableAux.SanityChecks.validateRowIndex index this.RowCount false) indexArr /// go from highest to lowest so no wrong column gets removed after index shift - let indexArr = indexArr |> Array.sortDescending + let indexArr = indexArr |> ARCtrl.ISA.Fable.Array.sortDescending Array.iter (fun index -> this.RemoveRow index) indexArr static member removeRows (indexArr:int []) = @@ -757,7 +757,7 @@ type ArcTable(name: string, headers: ResizeArray, values: Syste |> String.concat "\n" member this.StructurallyEquals (other: ArcTable) = - let sort = Array.ofSeq >> Array.sortBy (function |KeyValue (key,_) -> key) + let sort = Array.ofSeq >> ARCtrl.ISA.Fable.Array.sortBy (function |KeyValue (key,_) -> key) let n = this.Name = other.Name let headers = Aux.compareSeq this.Headers other.Headers let values = Aux.compareSeq (sort this.Values) (sort other.Values) diff --git a/src/ISA/ISA/Fable.fs b/src/ISA/ISA/Fable.fs index fb8286e0..e4d2c2ec 100644 --- a/src/ISA/ISA/Fable.fs +++ b/src/ISA/ISA/Fable.fs @@ -118,4 +118,35 @@ let distinct_generic l1 = let hashSeq (s:seq<'a>) = s |> Seq.map (fun x -> x.GetHashCode()) - |> Seq.reduce (fun a b -> a + b) \ No newline at end of file + |> Seq.reduce (fun a b -> a + b) + +module Array = + + + let sort (a : 'T []) = + #if FABLE_COMPILER_PYTHON + a |> List.ofArray |> List.sort |> Array.ofList + #else + a |> Array.sort + #endif + + let sortBy (f : 'T -> 'U) (a : 'T []) = + #if FABLE_COMPILER_PYTHON + a |> List.ofArray |> List.sortBy f |> Array.ofList + #else + a |> Array.sortBy f + #endif + + let sortDescending (a : 'T []) = + #if FABLE_COMPILER_PYTHON + a |> List.ofArray |> List.sortDescending |> Array.ofList + #else + a |> Array.sortDescending + #endif + + let sortDescendingBy (f : 'T -> 'U) (a : 'T []) = + #if FABLE_COMPILER_PYTHON + a |> List.ofArray |> List.sortByDescending f |> Array.ofList + #else + a |> Array.sortByDescending f + #endif diff --git a/testpy.cmd b/testpy.cmd new file mode 100644 index 00000000..045ac823 --- /dev/null +++ b/testpy.cmd @@ -0,0 +1,2 @@ +dotnet fable .\tests\ISA\ISA.Tests --lang py -o ./pytests --nocache +python .\pytests\main.py \ No newline at end of file From 2d35cfbb7b75ca79e5b2d5f28cb6dc56514acab0 Mon Sep 17 00:00:00 2001 From: Heinrich Lukas Weil Date: Fri, 26 Jan 2024 09:38:28 +0100 Subject: [PATCH 03/25] update Fable and pyxpecto --- .config/dotnet-tools.json | 2 +- tests/ISA/ISA.Tests/CompositeHeader.Tests.fs | 14 +++++++------- tests/ISA/ISA.Tests/DataModel.Tests.fs | 6 +++--- tests/ISA/ISA.Tests/ISA.Tests.fsproj | 2 +- tests/ISA/ISA.Tests/Main.fs | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 0d4a2909..33059ffa 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "fable": { - "version": "4.7.0", + "version": "4.10.0", "commands": [ "fable" ] diff --git a/tests/ISA/ISA.Tests/CompositeHeader.Tests.fs b/tests/ISA/ISA.Tests/CompositeHeader.Tests.fs index ba1afe70..7c17a5c1 100644 --- a/tests/ISA/ISA.Tests/CompositeHeader.Tests.fs +++ b/tests/ISA/ISA.Tests/CompositeHeader.Tests.fs @@ -330,7 +330,7 @@ let tests_ToTerm = testList "ToTerm" [ ] let tests_GetHashCode = testList "GetHashCode" [ - ftestCase "SimpleParamEqual" (fun () -> + testCase "SimpleParamEqual" (fun () -> let oa1 = OntologyAnnotation.fromString("MyTerm",tan = "LOL:123") let p1 = CompositeHeader.Parameter(oa1) let oa2 = OntologyAnnotation.fromString("MyTerm",tan = "LOL:123") @@ -339,7 +339,7 @@ let tests_GetHashCode = testList "GetHashCode" [ let h2 = p2.GetHashCode() Expect.equal h1 h2 "Same Param Headers should have equal hash" ) - ftestCase "SimpleParamUnequal" (fun () -> + testCase "SimpleParamUnequal" (fun () -> let oa1 = OntologyAnnotation.fromString("MyTerm",tan = "LOL:123") let p1 = CompositeHeader.Parameter(oa1) let oa2 = OntologyAnnotation.fromString("YourTerm",tan = "ROFL:321") @@ -348,7 +348,7 @@ let tests_GetHashCode = testList "GetHashCode" [ let h2 = p2.GetHashCode() Expect.notEqual h1 h2 "Different Param Headers should have unequal hash" ) - ftestCase "EmptyParam" (fun () -> + testCase "EmptyParam" (fun () -> let oa1 = OntologyAnnotation.empty let p1 = CompositeHeader.Parameter(oa1) let oa2 = OntologyAnnotation.empty @@ -357,7 +357,7 @@ let tests_GetHashCode = testList "GetHashCode" [ let h2 = p2.GetHashCode() Expect.equal h1 h2 "Empty Param Headers should be equal" ) - ftestCase "EmptyParamAndFactor" (fun () -> + testCase "EmptyParamAndFactor" (fun () -> let oa1 = OntologyAnnotation.empty let p1 = CompositeHeader.Parameter(oa1) let oa2 = OntologyAnnotation.empty @@ -366,21 +366,21 @@ let tests_GetHashCode = testList "GetHashCode" [ let h2 = f2.GetHashCode() Expect.notEqual h1 h2 "Empty Param Header should be unequal to empty factor" ) - ftestCase "InputSameType" (fun () -> + testCase "InputSameType" (fun () -> let i1 = CompositeHeader.Input(IOType.Sample) let i2 = CompositeHeader.Input(IOType.Sample) let h1 = i1.GetHashCode() let h2 = i2.GetHashCode() Expect.equal h1 h2 "Input Sample Headers should be equal" ) - ftestCase "InputDifferentType" (fun () -> + testCase "InputDifferentType" (fun () -> let i1 = CompositeHeader.Input(IOType.Sample) let i2 = CompositeHeader.Input(IOType.RawDataFile) let h1 = i1.GetHashCode() let h2 = i2.GetHashCode() Expect.notEqual h1 h2 "Input Sample Header should be unequal to Input Data" ) - ftestCase "InputOutputSameType" (fun () -> + testCase "InputOutputSameType" (fun () -> let i = CompositeHeader.Input(IOType.Sample) let o = CompositeHeader.Output(IOType.Sample) let h1 = i.GetHashCode() diff --git a/tests/ISA/ISA.Tests/DataModel.Tests.fs b/tests/ISA/ISA.Tests/DataModel.Tests.fs index aee2d8af..093332d4 100644 --- a/tests/ISA/ISA.Tests/DataModel.Tests.fs +++ b/tests/ISA/ISA.Tests/DataModel.Tests.fs @@ -261,21 +261,21 @@ let ontologyAnnotationTests = let otherParseable = "http://www.ebi.ac.uk/efo/EFO_0000721" let other = "Unparseable" testList "GetHashCode" [ - ftestCase "Empty" (fun () -> + testCase "Empty" (fun () -> let oa1 = OntologyAnnotation.fromString() let oa2 = OntologyAnnotation.fromString() let h1 = oa1.GetHashCode() let h2 = oa2.GetHashCode() Expect.equal h1 h2 "Hashes should be equal" ) - ftestCase "Equal" (fun () -> + testCase "Equal" (fun () -> let oa1 = OntologyAnnotation.fromString("MyOntology",tsr = "EFO",tan = uri) let oa2 = OntologyAnnotation.fromString("MyOntology",tsr = "EFO",tan = uri) let h1 = oa1.GetHashCode() let h2 = oa2.GetHashCode() Expect.equal h1 h2 "Hashes should be equal" ) - ftestCase "Different" (fun () -> + testCase "Different" (fun () -> let oa1 = OntologyAnnotation.fromString("MyOntology",tsr = "EFO",tan = uri) let oa2 = OntologyAnnotation.fromString("YourOntology",tsr = "NCBI",tan = "http://purl.obolibrary.org/obo/NCBI_0000123") let h1 = oa1.GetHashCode() diff --git a/tests/ISA/ISA.Tests/ISA.Tests.fsproj b/tests/ISA/ISA.Tests/ISA.Tests.fsproj index ffab6396..c2f068c2 100644 --- a/tests/ISA/ISA.Tests/ISA.Tests.fsproj +++ b/tests/ISA/ISA.Tests/ISA.Tests.fsproj @@ -31,7 +31,7 @@ - + diff --git a/tests/ISA/ISA.Tests/Main.fs b/tests/ISA/ISA.Tests/Main.fs index a3b45073..1fea5799 100644 --- a/tests/ISA/ISA.Tests/Main.fs +++ b/tests/ISA/ISA.Tests/Main.fs @@ -31,7 +31,7 @@ let all = testSequenced <| testList "ISA" [ let [] main argv = #if FABLE_COMPILER_PYTHON - Pyxpecto.runTests all + Pyxpecto.runTests (ConfigArg.fromStrings argv) all #endif #if FABLE_COMPILER_JAVASCRIPT Mocha.runTests all From f0bb78dcdccb1b2bfc31e6cc1c36c6905c950045 Mon Sep 17 00:00:00 2001 From: HLWeil Date: Fri, 23 Feb 2024 17:18:05 +0100 Subject: [PATCH 04/25] switch towards using .venv for running transpiled python --- .config/dotnet-tools.json | 2 +- .gitignore | 1 + build/BasicTasks.fs | 110 +++++++++++++++++++++++ build/TestTasks.fs | 120 +------------------------- tests/ISA/ISA.Tests/ArcTable.Tests.fs | 36 ++++---- 5 files changed, 133 insertions(+), 136 deletions(-) diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 33059ffa..535dbf2f 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "fable": { - "version": "4.10.0", + "version": "4.13.0", "commands": [ "fable" ] diff --git a/.gitignore b/.gitignore index 97f8e205..1d785f29 100644 --- a/.gitignore +++ b/.gitignore @@ -355,6 +355,7 @@ paket-files/ # Custom ignored files dist/ tmp/ +/.venv /tests/JavaScript/ARCtrl /tests/Python/ARCtrl /tests/UI/ARCtrl diff --git a/build/BasicTasks.fs b/build/BasicTasks.fs index b74fc141..0255088f 100644 --- a/build/BasicTasks.fs +++ b/build/BasicTasks.fs @@ -7,6 +7,116 @@ open Fake.IO.Globbing.Operators open ProjectInfo +[] +module Helper = + + open Fake + open Fake.Core + + let createProcess exe arg dir = + CreateProcess.fromRawCommandLine exe arg + |> CreateProcess.withWorkingDirectory dir + |> CreateProcess.ensureExitCode + + module Proc = + + module Parallel = + + open System + + let locker = obj() + + let colors = [| + ConsoleColor.DarkYellow + ConsoleColor.DarkCyan + ConsoleColor.Magenta + ConsoleColor.Blue + ConsoleColor.Cyan + ConsoleColor.DarkMagenta + ConsoleColor.DarkBlue + ConsoleColor.Yellow + |] + + let print color (colored: string) (line: string) = + lock locker + (fun () -> + let currentColor = Console.ForegroundColor + Console.ForegroundColor <- color + Console.Write colored + Console.ForegroundColor <- currentColor + Console.WriteLine line) + + let onStdout index name (line: string) = + let color = colors.[index % colors.Length] + if isNull line then + print color $"{name}: --- END ---" "" + else if String.isNotNullOrEmpty line then + print color $"{name}: " line + + let onStderr name (line: string) = + let color = ConsoleColor.Red + if isNull line |> not then + print color $"{name}: " line + + let redirect (index, (name, createProcess)) = + createProcess + |> CreateProcess.redirectOutputIfNotRedirected + |> CreateProcess.withOutputEvents (onStdout index name) (onStderr name) + + let printStarting indexed = + for (index, (name, c: CreateProcess<_>)) in indexed do + let color = colors.[index % colors.Length] + let wd = + c.WorkingDirectory + |> Option.defaultValue "" + let exe = c.Command.Executable + let args = c.Command.Arguments.ToStartInfo + print color $"{name}: {wd}> {exe} {args}" "" + + let run cs = + cs + |> Seq.toArray + |> Array.indexed + |> fun x -> printStarting x; x + |> Array.map redirect + |> Array.Parallel.map Proc.run + + let dotnet = createProcess "dotnet" + + let npx = + let npmPath = + match ProcessUtils.tryFindFileOnPath "npx" with + | Some path -> path + | None -> + "npm was not found in path. Please install it and make sure it's available from your path. " + + "See https://safe-stack.github.io/docs/quickstart/#install-pre-requisites for more info" + |> failwith + + createProcess npmPath + + let npm = + let npmPath = + match ProcessUtils.tryFindFileOnPath "npm" with + | Some path -> path + | None -> + "npm was not found in path. Please install it and make sure it's available from your path. " + + "See https://safe-stack.github.io/docs/quickstart/#install-pre-requisites for more info" + |> failwith + + createProcess npmPath + + let python = createProcess @".\.venv\Scripts\python.exe" + + let run proc arg dir = + proc arg dir + |> Proc.run + |> ignore + + let runParallel processes = + processes + |> Proc.Parallel.run + |> ignore + let setPrereleaseTag = BuildTask.create "SetPrereleaseTag" [] { printfn "Please enter pre-release package suffix" let suffix = System.Console.ReadLine() diff --git a/build/TestTasks.fs b/build/TestTasks.fs index 77b82417..a9f8eb25 100644 --- a/build/TestTasks.fs +++ b/build/TestTasks.fs @@ -7,124 +7,6 @@ open ProjectInfo open BasicTasks open Fake.Core -[] -module private Helper = - - open Fake - open Fake.Core - - let createProcess exe arg dir = - CreateProcess.fromRawCommandLine exe arg - |> CreateProcess.withWorkingDirectory dir - |> CreateProcess.ensureExitCode - - module Proc = - - module Parallel = - - open System - - let locker = obj() - - let colors = [| - ConsoleColor.DarkYellow - ConsoleColor.DarkCyan - ConsoleColor.Magenta - ConsoleColor.Blue - ConsoleColor.Cyan - ConsoleColor.DarkMagenta - ConsoleColor.DarkBlue - ConsoleColor.Yellow - |] - - let print color (colored: string) (line: string) = - lock locker - (fun () -> - let currentColor = Console.ForegroundColor - Console.ForegroundColor <- color - Console.Write colored - Console.ForegroundColor <- currentColor - Console.WriteLine line) - - let onStdout index name (line: string) = - let color = colors.[index % colors.Length] - if isNull line then - print color $"{name}: --- END ---" "" - else if String.isNotNullOrEmpty line then - print color $"{name}: " line - - let onStderr name (line: string) = - let color = ConsoleColor.Red - if isNull line |> not then - print color $"{name}: " line - - let redirect (index, (name, createProcess)) = - createProcess - |> CreateProcess.redirectOutputIfNotRedirected - |> CreateProcess.withOutputEvents (onStdout index name) (onStderr name) - - let printStarting indexed = - for (index, (name, c: CreateProcess<_>)) in indexed do - let color = colors.[index % colors.Length] - let wd = - c.WorkingDirectory - |> Option.defaultValue "" - let exe = c.Command.Executable - let args = c.Command.Arguments.ToStartInfo - print color $"{name}: {wd}> {exe} {args}" "" - - let run cs = - cs - |> Seq.toArray - |> Array.indexed - |> fun x -> printStarting x; x - |> Array.map redirect - |> Array.Parallel.map Proc.run - - let dotnet = createProcess "dotnet" - - let npm = - let npmPath = - match ProcessUtils.tryFindFileOnPath "npm" with - | Some path -> path - | None -> - "npm was not found in path. Please install it and make sure it's available from your path. " + - "See https://safe-stack.github.io/docs/quickstart/#install-pre-requisites for more info" - |> failwith - - createProcess npmPath - - let npx = - let npmPath = - match ProcessUtils.tryFindFileOnPath "npx" with - | Some path -> path - | None -> - "npm was not found in path. Please install it and make sure it's available from your path. " + - "See https://safe-stack.github.io/docs/quickstart/#install-pre-requisites for more info" - |> failwith - - createProcess npmPath - - let python = - let path = - match ProcessUtils.tryFindFileOnPath "python" with - | Some path -> path - | None -> - "py was not found in path. Please install it and make sure it's available from your path." - |> failwith - - createProcess path - - let run proc arg dir = - proc arg dir - |> Proc.run - |> ignore - - let runParallel processes = - processes - |> Proc.Parallel.run - |> ignore - module RunTests = let runTestsUI = BuildTask.create "runTestsUI" [clean; build] { @@ -160,6 +42,8 @@ module RunTests = let testProjectsPy = [ "tests/ISA/ISA.Tests" + "tests/ISA/ISA.Json.Tests" + "tests/ISA/ISA.Spreadsheet.Tests" ] let runTestsPy = BuildTask.create "runTestsPy" [clean; build] { diff --git a/tests/ISA/ISA.Tests/ArcTable.Tests.fs b/tests/ISA/ISA.Tests/ArcTable.Tests.fs index 56bc2299..64b437c9 100644 --- a/tests/ISA/ISA.Tests/ArcTable.Tests.fs +++ b/tests/ISA/ISA.Tests/ArcTable.Tests.fs @@ -78,20 +78,6 @@ let private tests_member = ] let private tests_GetHashCode = testList "GetHashCode" [ - testCase "Headers" <| fun _ -> - let table1 = create_testTable() - let table2 = create_testTable() - let hash1 = table1.GetHeadersHashCode() - let hash2 = table2.GetHeadersHashCode() - Expect.sequenceEqual table1.Headers table2.Headers "Table" - Expect.equal hash1 hash2 "Headers HashCode" - testCase "Value" <| fun _ -> - let table1 = create_testTable() - let table2 = create_testTable() - let hash1 = table1.GetValuesHashCode() - let hash2 = table2.GetValuesHashCode() - //Expect.sequenceEqual table1.Headers table2.Headers "Table" - Expect.equal hash1 hash2 "Values HashCode" testCase "GetHashCode" <| fun _ -> let table1 = create_testTable() let table2 = create_testTable() @@ -127,8 +113,13 @@ let private tests_GetHashCode = testList "GetHashCode" [ let values = Array.init 10000 (fun i -> CompositeCell.createFreeText (string i)) testTable.AddColumn(CompositeHeader.FreeText "Header", values) let f1 () = testTable.GetHashCode() + #if FABLE_COMPILER_PYTHON + let expectedMs = 500 + #else + let expectedMs = 50 + #endif // On i7-13800H, 2ms in Dotnet and 18ms in javascript - Expect.wantFaster f1 50 "GetHashCode is too slow" |> ignore + Expect.wantFaster f1 expectedMs "GetHashCode is too slow" |> ignore ] let private tests_validate = @@ -2038,7 +2029,12 @@ let private tests_AddRows = Array.init 10000 (fun i -> [|CompositeCell.FreeText $"Source_{i}"; CompositeCell.FreeText $"FT1_{i}"; CompositeCell.FreeText $"FT2_{i}"; CompositeCell.FreeText $"Sample_{i}"; |]) let testF = fun () -> table.AddRows(rows) - Expect.wantFaster testF 100 $"AddRows is too slow." |> ignore + #if FABLE_COMPILER_PYTHON + let expectedMs = 1000 + #else + let expectedMs = 100 + #endif + Expect.wantFaster testF expectedMs $"AddRows is too slow." |> ignore ) ] @@ -2339,7 +2335,13 @@ let private tests_fillMissing = testList "fillMissing" [ ArcTableAux.Unchecked.setCellAt(5,i,(CompositeCell.FreeText $"FT5_{i}")) values ArcTableAux.Unchecked.setCellAt(6,i,(CompositeCell.FreeText $"Sample_{i}")) values let testF = fun () -> ArcTableAux.Unchecked.fillMissingCells headers values - Expect.wantFaster testF 220 "fillMissing is too slow." |> ignore // 130ms in javascript, dotnet faster than 100ms + #if FABLE_COMPILER_PYTHON + let expectedMs = 8000 + #else + let expectedMs = 220 + #endif + //4800ms in python on i7-13800H + Expect.wantFaster testF expectedMs "fillMissing is too slow." |> ignore // 130ms in javascript, dotnet faster than 100ms ] From aa30389f070a248e547373779de64247d0b017cc Mon Sep 17 00:00:00 2001 From: HLWeil Date: Fri, 23 Feb 2024 18:11:50 +0100 Subject: [PATCH 05/25] move json schema validation to json tests --- src/ISA/ISA.Json/ARCtrl.ISA.Json.fsproj | 6 ------ tests/ISA/ISA.Json.Tests/ISA.Json.Tests.fsproj | 12 ++++++++++++ .../ISA/ISA.Json.Tests}/JsonSchemaValidation.fs | 0 .../ISA/ISA.Json.Tests}/Validation/Fable.fs | 6 +++++- .../Validation/JsonSchemaValidation.fs | 4 +++- .../ISA/ISA.Json.Tests}/Validation/JsonValidation.js | 0 .../ISA.Json.Tests}/Validation/ValidationResult.fs | 0 .../ISA/ISA.Json.Tests}/ValidationResult.fs | 0 8 files changed, 20 insertions(+), 8 deletions(-) rename {src/ISA/ISA.Json => tests/ISA/ISA.Json.Tests}/JsonSchemaValidation.fs (100%) rename {src/ISA/ISA.Json => tests/ISA/ISA.Json.Tests}/Validation/Fable.fs (97%) rename {src/ISA/ISA.Json => tests/ISA/ISA.Json.Tests}/Validation/JsonSchemaValidation.fs (99%) rename {src/ISA/ISA.Json => tests/ISA/ISA.Json.Tests}/Validation/JsonValidation.js (100%) rename {src/ISA/ISA.Json => tests/ISA/ISA.Json.Tests}/Validation/ValidationResult.fs (100%) rename {src/ISA/ISA.Json => tests/ISA/ISA.Json.Tests}/ValidationResult.fs (100%) diff --git a/src/ISA/ISA.Json/ARCtrl.ISA.Json.fsproj b/src/ISA/ISA.Json/ARCtrl.ISA.Json.fsproj index 1e564eb9..ec41c674 100644 --- a/src/ISA/ISA.Json/ARCtrl.ISA.Json.fsproj +++ b/src/ISA/ISA.Json/ARCtrl.ISA.Json.fsproj @@ -5,12 +5,6 @@ true - - Always - - - - diff --git a/tests/ISA/ISA.Json.Tests/ISA.Json.Tests.fsproj b/tests/ISA/ISA.Json.Tests/ISA.Json.Tests.fsproj index 5b0ae263..738b470e 100644 --- a/tests/ISA/ISA.Json.Tests/ISA.Json.Tests.fsproj +++ b/tests/ISA/ISA.Json.Tests/ISA.Json.Tests.fsproj @@ -6,6 +6,18 @@ false + + Always + + + + + + fable\ + + + fable\ + diff --git a/src/ISA/ISA.Json/JsonSchemaValidation.fs b/tests/ISA/ISA.Json.Tests/JsonSchemaValidation.fs similarity index 100% rename from src/ISA/ISA.Json/JsonSchemaValidation.fs rename to tests/ISA/ISA.Json.Tests/JsonSchemaValidation.fs diff --git a/src/ISA/ISA.Json/Validation/Fable.fs b/tests/ISA/ISA.Json.Tests/Validation/Fable.fs similarity index 97% rename from src/ISA/ISA.Json/Validation/Fable.fs rename to tests/ISA/ISA.Json.Tests/Validation/Fable.fs index 6add8b2e..f49ae88b 100644 --- a/src/ISA/ISA.Json/Validation/Fable.fs +++ b/tests/ISA/ISA.Json.Tests/Validation/Fable.fs @@ -1,5 +1,7 @@ namespace ARCtrl.ISA.Json +#if !FABLE_COMPILER_PYTHON + module Fable = open Fable.Core @@ -71,4 +73,6 @@ module Fable = .Value .ToValidationResult() return output - } \ No newline at end of file + } + +#endif \ No newline at end of file diff --git a/src/ISA/ISA.Json/Validation/JsonSchemaValidation.fs b/tests/ISA/ISA.Json.Tests/Validation/JsonSchemaValidation.fs similarity index 99% rename from src/ISA/ISA.Json/Validation/JsonSchemaValidation.fs rename to tests/ISA/ISA.Json.Tests/Validation/JsonSchemaValidation.fs index cb714abb..9f3064de 100644 --- a/src/ISA/ISA.Json/Validation/JsonSchemaValidation.fs +++ b/tests/ISA/ISA.Json.Tests/Validation/JsonSchemaValidation.fs @@ -1,5 +1,7 @@ namespace ARCtrl.ISA.Json + + open ValidationTypes module JsonSchemaUrls = @@ -83,7 +85,7 @@ module Validation = let validate (schemaURL : string) (objectString : string) = async { try - #if FABLE_COMPILER + #if FABLE_COMPILER_JAVASCRIPT let! isValid, errorList = Fable.validate (schemaURL) (objectString) #else let settings = NJsonSchema.Validation.JsonSchemaValidatorSettings() diff --git a/src/ISA/ISA.Json/Validation/JsonValidation.js b/tests/ISA/ISA.Json.Tests/Validation/JsonValidation.js similarity index 100% rename from src/ISA/ISA.Json/Validation/JsonValidation.js rename to tests/ISA/ISA.Json.Tests/Validation/JsonValidation.js diff --git a/src/ISA/ISA.Json/Validation/ValidationResult.fs b/tests/ISA/ISA.Json.Tests/Validation/ValidationResult.fs similarity index 100% rename from src/ISA/ISA.Json/Validation/ValidationResult.fs rename to tests/ISA/ISA.Json.Tests/Validation/ValidationResult.fs diff --git a/src/ISA/ISA.Json/ValidationResult.fs b/tests/ISA/ISA.Json.Tests/ValidationResult.fs similarity index 100% rename from src/ISA/ISA.Json/ValidationResult.fs rename to tests/ISA/ISA.Json.Tests/ValidationResult.fs From 93b6d9490acdf16045bd844f4cec22f2d54c7eca Mon Sep 17 00:00:00 2001 From: HLWeil Date: Fri, 23 Feb 2024 18:12:50 +0100 Subject: [PATCH 06/25] replace mocha and expecto with pyxpecto --- tests/ARCtrl/Main.fs | 15 +++---------- tests/FileSystem/Main.fs | 15 +++---------- .../ISA/ISA.Json.Tests/ISA.Json.Tests.fsproj | 1 + tests/ISA/ISA.Json.Tests/Main.fs | 17 ++++----------- tests/ISA/ISA.Spreadsheet.Tests/Main.fs | 15 +++---------- tests/ISA/ISA.Tests/ISA.Tests.fsproj | 4 +--- tests/ISA/ISA.Tests/Library.fs | 8 ------- tests/ISA/ISA.Tests/Main.fs | 21 ++----------------- tests/TestingUtils/Library.fs | 9 +------- tests/TestingUtils/TestingUtils.fsproj | 4 +--- 10 files changed, 19 insertions(+), 90 deletions(-) diff --git a/tests/ARCtrl/Main.fs b/tests/ARCtrl/Main.fs index 67d2d2be..6f7cfce1 100644 --- a/tests/ARCtrl/Main.fs +++ b/tests/ARCtrl/Main.fs @@ -1,12 +1,7 @@ module Main.Tests -#if FABLE_COMPILER -open Fable.Mocha -#else -open Expecto +open Fable.Pyxpecto -[] -#endif let all = testSequenced <| testList "ARCtrl" [ ARCtrl.Contracts.Tests.main ARCtrl.WebRequest.Tests.main @@ -15,9 +10,5 @@ let all = testSequenced <| testList "ARCtrl" [ ARCtrl.Tests.main ] -let [] main argv = - #if FABLE_COMPILER - Mocha.runTests all - #else - Tests.runTestsWithCLIArgs [] argv all - #endif \ No newline at end of file +[] +let main argv = Pyxpecto.runTests [||] all \ No newline at end of file diff --git a/tests/FileSystem/Main.fs b/tests/FileSystem/Main.fs index fd17cf93..62ae33f3 100644 --- a/tests/FileSystem/Main.fs +++ b/tests/FileSystem/Main.fs @@ -1,19 +1,10 @@ module ARCtrl.FileSystem.Tests -#if FABLE_COMPILER -open Fable.Mocha -#else -open Expecto +open Fable.Pyxpecto -[] -#endif let all = testSequenced <| testList "FileSystem" [ ARCtrl.FileSystemTree.Tests.main ] -let [] main argv = - #if FABLE_COMPILER - Mocha.runTests all - #else - Tests.runTestsWithCLIArgs [] argv all - #endif +[] +let main argv = Pyxpecto.runTests [||] all diff --git a/tests/ISA/ISA.Json.Tests/ISA.Json.Tests.fsproj b/tests/ISA/ISA.Json.Tests/ISA.Json.Tests.fsproj index 738b470e..154df992 100644 --- a/tests/ISA/ISA.Json.Tests/ISA.Json.Tests.fsproj +++ b/tests/ISA/ISA.Json.Tests/ISA.Json.Tests.fsproj @@ -23,6 +23,7 @@ + diff --git a/tests/ISA/ISA.Json.Tests/Main.fs b/tests/ISA/ISA.Json.Tests/Main.fs index 7723b53c..4391afa9 100644 --- a/tests/ISA/ISA.Json.Tests/Main.fs +++ b/tests/ISA/ISA.Json.Tests/Main.fs @@ -1,22 +1,13 @@ module ARCtrl.ISA.Json.Tests -#if FABLE_COMPILER -open Fable.Mocha -#else -open Expecto +open Fable.Pyxpecto + -[] -#endif let all = testSequenced <| testList "ISA.JSON" [ Json.Tests.main JsonSchema.Tests.main Tests.ArcTypes.Main ] - -let [] main argv = - #if FABLE_COMPILER - Mocha.runTests all - #else - Tests.runTestsWithCLIArgs [] argv all - #endif \ No newline at end of file +[] +let main argv = Pyxpecto.runTests [||] all \ No newline at end of file diff --git a/tests/ISA/ISA.Spreadsheet.Tests/Main.fs b/tests/ISA/ISA.Spreadsheet.Tests/Main.fs index cd748b13..14e0be7e 100644 --- a/tests/ISA/ISA.Spreadsheet.Tests/Main.fs +++ b/tests/ISA/ISA.Spreadsheet.Tests/Main.fs @@ -1,12 +1,7 @@ module ARCtrl.ISA.XLSX.Tests -#if FABLE_COMPILER -open Fable.Mocha -#else -open Expecto +open Fable.Pyxpecto -[] -#endif let all = testSequenced <| testList "ISA.Spreadsheet" [ FableTests.main ArcInvestigationTests.main @@ -19,9 +14,5 @@ let all = testSequenced <| testList "ISA.Spreadsheet" [ Tests.Performance.Main ] -let [] main argv = - #if FABLE_COMPILER - Mocha.runTests all - #else - Tests.runTestsWithCLIArgs [] argv all - #endif \ No newline at end of file +[] +let main argv = Pyxpecto.runTests [||] all diff --git a/tests/ISA/ISA.Tests/ISA.Tests.fsproj b/tests/ISA/ISA.Tests/ISA.Tests.fsproj index c2f068c2..68a58fc3 100644 --- a/tests/ISA/ISA.Tests/ISA.Tests.fsproj +++ b/tests/ISA/ISA.Tests/ISA.Tests.fsproj @@ -30,9 +30,7 @@ - - - + diff --git a/tests/ISA/ISA.Tests/Library.fs b/tests/ISA/ISA.Tests/Library.fs index 9276cc75..62cb9914 100644 --- a/tests/ISA/ISA.Tests/Library.fs +++ b/tests/ISA/ISA.Tests/Library.fs @@ -3,15 +3,7 @@ open ARCtrl.ISA open ARCtrl.FileSystem -#if FABLE_COMPILER_PYTHON open Fable.Pyxpecto -#endif -#if FABLE_COMPILER_JAVASCRIPT -open Fable.Mocha -#endif -#if !FABLE_COMPILER -open Expecto -#endif module Async = diff --git a/tests/ISA/ISA.Tests/Main.fs b/tests/ISA/ISA.Tests/Main.fs index 1fea5799..1433c8d1 100644 --- a/tests/ISA/ISA.Tests/Main.fs +++ b/tests/ISA/ISA.Tests/Main.fs @@ -1,16 +1,7 @@ module ARCtrl.ISADotnet.Tests -#if FABLE_COMPILER_PYTHON open Fable.Pyxpecto -#endif -#if FABLE_COMPILER_JAVASCRIPT -open Fable.Mocha -#endif -#if !FABLE_COMPILER -open Expecto -[] -#endif let all = testSequenced <| testList "ISA" [ Update.Tests.main DataModel.Tests.main @@ -29,13 +20,5 @@ let all = testSequenced <| testList "ISA" [ Fable.Tests.main ] -let [] main argv = - #if FABLE_COMPILER_PYTHON - Pyxpecto.runTests (ConfigArg.fromStrings argv) all - #endif - #if FABLE_COMPILER_JAVASCRIPT - Mocha.runTests all - #endif - #if !FABLE_COMPILER - Tests.runTestsWithCLIArgs [] [||] all - #endif \ No newline at end of file +[] +let main argv = Pyxpecto.runTests [||] all \ No newline at end of file diff --git a/tests/TestingUtils/Library.fs b/tests/TestingUtils/Library.fs index 7f3c8de1..3240f2d8 100644 --- a/tests/TestingUtils/Library.fs +++ b/tests/TestingUtils/Library.fs @@ -4,15 +4,8 @@ open FsSpreadsheet open ARCtrl.ISA open ARCtrl.FileSystem -#if FABLE_COMPILER_PYTHON open Fable.Pyxpecto -#endif -#if FABLE_COMPILER_JAVASCRIPT -open Fable.Mocha -#endif -#if !FABLE_COMPILER -open Expecto -#endif + module Async = diff --git a/tests/TestingUtils/TestingUtils.fsproj b/tests/TestingUtils/TestingUtils.fsproj index c20b064b..8c6d504a 100644 --- a/tests/TestingUtils/TestingUtils.fsproj +++ b/tests/TestingUtils/TestingUtils.fsproj @@ -24,9 +24,7 @@ - - - + From 688c6283e9b2bd332fd93388b10d8661e185bc1b Mon Sep 17 00:00:00 2001 From: HLWeil Date: Fri, 23 Feb 2024 18:13:46 +0100 Subject: [PATCH 07/25] work on json io support in python --- src/ISA/ISA.Json/ARCtrl.ISA.Json.fsproj | 1 + src/ISA/ISA.Json/Decode.fs | 17 +++++++--- src/ISA/ISA.Json/GEncode.fs | 45 +++++++++++-------------- 3 files changed, 32 insertions(+), 31 deletions(-) diff --git a/src/ISA/ISA.Json/ARCtrl.ISA.Json.fsproj b/src/ISA/ISA.Json/ARCtrl.ISA.Json.fsproj index ec41c674..6d349e16 100644 --- a/src/ISA/ISA.Json/ARCtrl.ISA.Json.fsproj +++ b/src/ISA/ISA.Json/ARCtrl.ISA.Json.fsproj @@ -60,6 +60,7 @@ + diff --git a/src/ISA/ISA.Json/Decode.fs b/src/ISA/ISA.Json/Decode.fs index 0c085fee..1294a4aa 100644 --- a/src/ISA/ISA.Json/Decode.fs +++ b/src/ISA/ISA.Json/Decode.fs @@ -8,14 +8,17 @@ open Fable.Core module GDecode = let helpers = - #if FABLE_COMPILER + #if FABLE_COMPILER_PYTHON + Thoth.Json.Python.Decode.helpers + #endif + #if FABLE_COMPILER_JAVASCRIPT Thoth.Json.JavaScript.Decode.helpers - #else + #endif + #if !FABLE_COMPILER Thoth.Json.Newtonsoft.Decode.helpers #endif - let isURI (s : string) = true //s.StartsWith("http://") || s.StartsWith("https://") @@ -30,9 +33,13 @@ module GDecode = } let inline fromJsonString (decoder : Decoder<'a>) (s : string) : 'a = - #if FABLE_COMPILER + #if FABLE_COMPILER_PYTHON + match Thoth.Json.Python.Decode.fromString decoder s with + #endif + #if FABLE_COMPILER_JAVASCRIPT match Thoth.Json.JavaScript.Decode.fromString decoder s with - #else + #endif + #if !FABLE_COMPILER match Thoth.Json.Newtonsoft.Decode.fromString decoder s with #endif | Ok a -> a diff --git a/src/ISA/ISA.Json/GEncode.fs b/src/ISA/ISA.Json/GEncode.fs index aea49a47..0decb6bf 100644 --- a/src/ISA/ISA.Json/GEncode.fs +++ b/src/ISA/ISA.Json/GEncode.fs @@ -3,18 +3,26 @@ open Thoth.Json.Core open Fable.Core + +#if FABLE_COMPILER_PYTHON +open Fable.Core.PyInterop +#endif +#if FABLE_COMPILER_JAVASCRIPT open Fable.Core.JsInterop +#endif [] module GEncode = - + + #if FABLE_COMPILER_JAVASCRIPT [] let getFieldFable (name : string) (object : 'T) = jsNative + #endif /// Try to get a property value from a record by its name let inline tryGetPropertyValue (name : string) (object : 'T) = - #if FABLE_COMPILER - getFieldFable name object + #if FABLE_COMPILER_JAVASCRIPT + getFieldFable name object #else let property = FSharp.Reflection.FSharpType.GetRecordFields(object.GetType()) @@ -32,9 +40,13 @@ module GEncode = #endif let inline toJsonString spaces (value : Json) = - #if FABLE_COMPILER + #if FABLE_COMPILER_PYTHON + Thoth.Json.Python.Encode.toString spaces value + #endif + #if FABLE_COMPILER_JAVASCRIPT Thoth.Json.JavaScript.Encode.toString spaces value - #else + #endif + #if !FABLE_COMPILER Thoth.Json.Newtonsoft.Encode.toString spaces value #endif @@ -54,7 +66,7 @@ module GEncode = let tryIncludeObj name (encoder : obj -> Json) (value : obj option) = name, match value with - #if FABLE_COMPILER + #if FABLE_COMPILER_JAVASCRIPT | Some (:? System.Collections.IEnumerable as v) -> !!Seq.map encoder v |> Encode.seq #else @@ -89,23 +101,4 @@ module GEncode = name, match value with | Some(os) -> os |> List.map encoder |> Encode.list - | _ -> Encode.nil - - // This seems to fail because due to dotnet not able to match the boxed lists against nongeneric System.Collections.IEnumerable - // - // -> ///// Try to encode the given object using the given encoder, or return Encode.nil if the object is null - // -> ///// - // -> ///// If the object is a sequence, encode each element using the given encoder and return the resulting sequence - // -> //let tryInclude name (encoder : obj -> JsonValue) (value : obj option) = - // -> // name, - // -> // match value with - // -> // | Some(:? System.Collections.IEnumerable as v) -> - // -> // let os = - // -> // #if FABLE_COMPILER - // -> // !!v - // -> // #else - // -> // v |> Seq.cast - // -> // #endif - // -> // Seq.map encoder os |> Encode.seq - // -> // | Some(o) -> encoder o - // -> // | _ -> Encode.nil \ No newline at end of file + | _ -> Encode.nil \ No newline at end of file From 3dfaad77a14aa70715388ef6a1fa77cb337bd622 Mon Sep 17 00:00:00 2001 From: HLWeil Date: Mon, 26 Feb 2024 15:04:24 +0100 Subject: [PATCH 08/25] adjustments to web and validation to allowe for python transpilation --- src/ARCtrl/ARCtrl.fsproj | 2 +- src/ARCtrl/Templates/Template.Json.fs | 2 +- src/ARCtrl/Templates/Template.Web.fs | 5 +- src/ARCtrl/WebRequest/WebRequest.fs | 27 +++-- src/ISA/ISA.Json/ARCtrl.ISA.Json.fsproj | 4 +- src/ISA/ISA.Json/ArcTypes/ArcTable.fs | 2 +- .../ISA/ISA.Json.Tests/ISA.Json.Tests.fsproj | 6 - tests/ISA/ISA.Json.Tests/Json.Tests.fs | 27 ++--- tests/ISA/ISA.Json.Tests/JsonSchema.Tests.fs | 7 +- .../ISA.Json.Tests/JsonSchemaValidation.fs | 109 ------------------ .../Validation/JsonSchemaValidation.fs | 8 +- tests/ISA/ISA.Json.Tests/ValidationResult.fs | 20 ---- 12 files changed, 51 insertions(+), 168 deletions(-) delete mode 100644 tests/ISA/ISA.Json.Tests/JsonSchemaValidation.fs delete mode 100644 tests/ISA/ISA.Json.Tests/ValidationResult.fs diff --git a/src/ARCtrl/ARCtrl.fsproj b/src/ARCtrl/ARCtrl.fsproj index 04e8156e..4ac4b723 100644 --- a/src/ARCtrl/ARCtrl.fsproj +++ b/src/ARCtrl/ARCtrl.fsproj @@ -34,7 +34,7 @@ - + diff --git a/src/ARCtrl/Templates/Template.Json.fs b/src/ARCtrl/Templates/Template.Json.fs index 76dd18a5..8ed20d28 100644 --- a/src/ARCtrl/Templates/Template.Json.fs +++ b/src/ARCtrl/Templates/Template.Json.fs @@ -61,7 +61,7 @@ module Template = get.Required.Field "authors" (Decode.array personDecoder), get.Required.Field "endpoint_repositories" (Decode.array oaDecoder), get.Required.Field "tags" (Decode.array oaDecoder), - get.Required.Field "last_updated" Decode.datetimeUtc + get.Required.Field "last_updated" Decode.datetimeLocal // Currently not supported in Thoth.Json.Core for python ) ) diff --git a/src/ARCtrl/Templates/Template.Web.fs b/src/ARCtrl/Templates/Template.Web.fs index d12460e8..3a53a3ff 100644 --- a/src/ARCtrl/Templates/Template.Web.fs +++ b/src/ARCtrl/Templates/Template.Web.fs @@ -14,6 +14,8 @@ let getTemplates(url: string option) = return mapResult } +#if FABLE_COMPILER_JAVASCRIPT + /// /// This class is used to make async functions more accessible from JavaScript. /// @@ -24,4 +26,5 @@ type JsWeb = let! templates = getTemplates(url) return templates } - |> Async.StartAsPromise \ No newline at end of file + |> Async.StartAsPromise +#endif \ No newline at end of file diff --git a/src/ARCtrl/WebRequest/WebRequest.fs b/src/ARCtrl/WebRequest/WebRequest.fs index 3e152b2b..4b7bac9d 100644 --- a/src/ARCtrl/WebRequest/WebRequest.fs +++ b/src/ARCtrl/WebRequest/WebRequest.fs @@ -2,6 +2,8 @@ open Fable.Core open Fable.SimpleHttp + +#if FABLE_COMPILER_JAVASCRIPT open Fable.Core.JsInterop open Fable.SimpleHttp @@ -31,12 +33,9 @@ process.versions.node != null;""" txt ) |> Async.AwaitPromise +#endif let downloadFile url = - let mutable isFable = false - #if FABLE_COMPILER - isFable <- true - #endif let browserAndDotnet() = async { let! (statusCode, responseText) = Http.get url @@ -46,12 +45,16 @@ let downloadFile url = | 200 -> responseText | _ -> failwithf "Status %d => %s" statusCode responseText } - if not isFable then - browserAndDotnet() + #if FABLE_COMPILER_JAVASCRIPT + if NodeJs.isNode() then + // From here: https://github.com/fable-compiler/fable3-samples/blob/25ea2404b28c897988b144f0141bc116da292679/nodejs/src/App.fs#L7 + importSideEffects "isomorphic-fetch" + NodeJs.downloadFile url else - if NodeJs.isNode() then - // From here: https://github.com/fable-compiler/fable3-samples/blob/25ea2404b28c897988b144f0141bc116da292679/nodejs/src/App.fs#L7 - importSideEffects "isomorphic-fetch" - NodeJs.downloadFile url - else - browserAndDotnet() + browserAndDotnet() + #else + + browserAndDotnet() + #endif + + diff --git a/src/ISA/ISA.Json/ARCtrl.ISA.Json.fsproj b/src/ISA/ISA.Json/ARCtrl.ISA.Json.fsproj index 6d349e16..4c7c991f 100644 --- a/src/ISA/ISA.Json/ARCtrl.ISA.Json.fsproj +++ b/src/ISA/ISA.Json/ARCtrl.ISA.Json.fsproj @@ -58,9 +58,9 @@ - - + + diff --git a/src/ISA/ISA.Json/ArcTypes/ArcTable.fs b/src/ISA/ISA.Json/ArcTypes/ArcTable.fs index 9de244c6..44bd86c9 100644 --- a/src/ISA/ISA.Json/ArcTypes/ArcTable.fs +++ b/src/ISA/ISA.Json/ArcTypes/ArcTable.fs @@ -160,7 +160,7 @@ module ArcTableExtensions = member this.ToCompressedJsonString(?spaces) : string = let spaces = defaultArg spaces 0 - let stringTable = Dictionary() + let stringTable = Dictionary() let oaTable = Dictionary() let cellTable = Dictionary() let arcTable = ArcTable.compressedEncoder stringTable oaTable cellTable this diff --git a/tests/ISA/ISA.Json.Tests/ISA.Json.Tests.fsproj b/tests/ISA/ISA.Json.Tests/ISA.Json.Tests.fsproj index 154df992..c263dd54 100644 --- a/tests/ISA/ISA.Json.Tests/ISA.Json.Tests.fsproj +++ b/tests/ISA/ISA.Json.Tests/ISA.Json.Tests.fsproj @@ -12,12 +12,6 @@ - - fable\ - - - fable\ - diff --git a/tests/ISA/ISA.Json.Tests/Json.Tests.fs b/tests/ISA/ISA.Json.Tests/Json.Tests.fs index 062a6f4f..713fa809 100644 --- a/tests/ISA/ISA.Json.Tests/Json.Tests.fs +++ b/tests/ISA/ISA.Json.Tests/Json.Tests.fs @@ -524,7 +524,7 @@ let testProtocolFile = Expect.isOk writingSuccess (Result.getMessage writingSuccess) ) - + #if !FABLE_COMPILER_PYTHON testAsync "WriterSchemaCorrectness" { let p = Protocol.fromJsonString Protocol.protocol @@ -535,6 +535,7 @@ let testProtocolFile = Expect.isTrue validation.Success $"Protocol did not match schema: {validation.GetErrors()}" } + #endif testCase "OutputMatchesInput" (fun () -> @@ -688,7 +689,7 @@ let testProcessFile = Expect.isOk writingSuccess (Result.getMessage writingSuccess) ) - + #if !FABLE_COMPILER_PYTHON testAsync "WriterSchemaCorrectness" { let p = Process.fromJsonString Process.process' @@ -699,7 +700,7 @@ let testProcessFile = Expect.isTrue validation.Success $"Process did not match schema: {validation.GetErrors()}" } - + #endif testCase "OutputMatchesInput" (fun () -> let o = @@ -835,7 +836,7 @@ let testPersonFile = Expect.isOk writingSuccess (Result.getMessage writingSuccess) ) - + #if !FABLE_COMPILER_PYTHON testAsync "WriterSchemaCorrectness" { let a = Person.fromJsonString Person.person @@ -846,7 +847,7 @@ let testPersonFile = Expect.isTrue validation.Success $"Person did not match schema: {validation.GetErrors()}" } - + #endif testCase "OutputMatchesInput" (fun () -> let o = @@ -877,7 +878,7 @@ let testPersonFile = Expect.equal p.ORCID.Value "0000-0002-1825-0097" "ORCID not as expected" ) - + #if !FABLE_COMPILER_PYTHON testAsync "WithORCID WriterSchemaCorrectness" { let a = Person.fromJsonString Person.personWithORCID @@ -888,7 +889,7 @@ let testPersonFile = Expect.isTrue validation.Success $"Person did not match schema: {validation.GetErrors()}" } - + #endif testCase "WithORCID OutputMatchesInput" (fun () -> let o = @@ -1024,7 +1025,7 @@ let testPublicationFile = Expect.isOk writingSuccess (Result.getMessage writingSuccess) ) - + #if !FABLE_COMPILER_PYTHON testAsync "WriterSchemaCorrectness" { let a = Publication.fromJsonString Publication.publication @@ -1035,7 +1036,7 @@ let testPublicationFile = Expect.isTrue validation.Success $"Publication did not match schema: {validation.GetErrors()}" } - + #endif testCase "OutputMatchesInput" (fun () -> let o = @@ -1150,7 +1151,7 @@ let testAssayFile = Expect.isOk writingSuccess (Result.getMessage writingSuccess) ) - + #if !FABLE_COMPILER_PYTHON testAsync "WriterSchemaCorrectness" { let a = Assay.fromJsonString Assay.assay @@ -1161,7 +1162,7 @@ let testAssayFile = Expect.isTrue validation.Success $"Assay did not match schema: {validation.GetErrors()}" } - + #endif testCase "OutputMatchesInput" (fun () -> let o = @@ -1212,7 +1213,7 @@ let testInvestigationFile = Expect.isOk writingSuccess (Result.getMessage writingSuccess) ) - + #if !FABLE_COMPILER_PYTHON testAsync "WriterSchemaCorrectness" { let i = Investigation.fromJsonString Investigation.investigation @@ -1223,7 +1224,7 @@ let testInvestigationFile = Expect.isTrue validation.Success $"Investigation did not match schema: {validation.GetErrors()}" } - + #endif testCase "OutputMatchesInput" (fun () -> let o = diff --git a/tests/ISA/ISA.Json.Tests/JsonSchema.Tests.fs b/tests/ISA/ISA.Json.Tests/JsonSchema.Tests.fs index 526caf4c..968a6d14 100644 --- a/tests/ISA/ISA.Json.Tests/JsonSchema.Tests.fs +++ b/tests/ISA/ISA.Json.Tests/JsonSchema.Tests.fs @@ -4,6 +4,7 @@ open TestingUtils open ARCtrl.ISA.Json +#if !FABLE_COMPILER_PYTHON /// ⚠️ This testlist is only used to test correct execution of Fable bindings on `JsonValidation.js` functions. /// ⚠️ Never execute in dotnet environment let tests_FableBindings = testList "FableBindings" [ @@ -126,11 +127,15 @@ let tests_ProcessParameterValue = } ] +#endif + let main = testList "JsonSchema-Validation" [ - #if FABLE_COMPILER + #if FABLE_COMPILER_JAVASCRIPT tests_FableBindings #endif + #if !FABLE_COMPILER_PYTHON tests_Process tests_ProcessParameterValue + #endif ] \ No newline at end of file diff --git a/tests/ISA/ISA.Json.Tests/JsonSchemaValidation.fs b/tests/ISA/ISA.Json.Tests/JsonSchemaValidation.fs deleted file mode 100644 index a6a7a6e5..00000000 --- a/tests/ISA/ISA.Json.Tests/JsonSchemaValidation.fs +++ /dev/null @@ -1,109 +0,0 @@ -namespace ARCtrl.ISA.Validation - -module JSchema = - -#if FABLE_COMPILER - let validate (schemaURL : string) (objectString : string) = - ValidationResult.Ok -#else - - let tryDownloadSchema (schemaURL : string) = - let rec download (tryNum) = - try NJsonSchema.JsonSchema.FromUrlAsync(schemaURL) - with - | err when tryNum <= 3 -> - System.Threading.Thread.Sleep(30) - download (tryNum + 1) - | err -> failwith $"Could not download schema from url {schemaURL}: \n{err.Message}" - download 1 - - let validate (schemaURL : string) (objectString : string) = - - try - let settings = NJsonSchema.Validation.JsonSchemaValidatorSettings() - let schema = tryDownloadSchema schemaURL - let r = schema.Result.Validate(objectString,settings) - - ValidationResult.OfJSchemaOutput(r |> Seq.length |> (=) 0,r |> Seq.map (fun err -> err.ToString()) |> Seq.toArray) - with - | err -> Failed [|err.Message|] -#endif - let validateAssay (assayString : string) = - let assayUrl = "https://raw.githubusercontent.com/HLWeil/isa-specs/anyof/source/_static/isajson/assay_schema.json" - validate assayUrl assayString - - let validateComment (commentString : string) = - let commentUrl = "https://raw.githubusercontent.com/HLWeil/isa-specs/anyof/source/_static/isajson/comment_schema.json" - validate commentUrl commentString - - let validateData (dataString : string) = - let dataUrl = "https://raw.githubusercontent.com/HLWeil/isa-specs/anyof/source/_static/isajson/data_schema.json" - validate dataUrl dataString - - let validateFactor (factorString : string) = - let factorUrl = "https://raw.githubusercontent.com/HLWeil/isa-specs/anyof/source/_static/isajson/factor_schema.json" - validate factorUrl factorString - - let validateFactorValue (factorValueString : string) = - let factorValueUrl = "https://raw.githubusercontent.com/HLWeil/isa-specs/anyof/source/_static/isajson/factor_value_schema.json" - validate factorValueUrl factorValueString - - let validateInvestigation (investigationString : string) = - let investigationUrl = "https://raw.githubusercontent.com/HLWeil/isa-specs/anyof/source/_static/isajson/investigation_schema.json" - validate investigationUrl investigationString - - let validateMaterialAttribute (materialAttributeString : string) = - let materialAttributeUrl = "https://raw.githubusercontent.com/HLWeil/isa-specs/anyof/source/_static/isajson/material_attribute_schema.json" - validate materialAttributeUrl materialAttributeString - - let validateMaterialAttributeValue (materialAttributeValueString : string) = - let materialAttributeValueUrl = "https://raw.githubusercontent.com/HLWeil/isa-specs/anyof/source/_static/isajson/material_attribute_value_schema.json" - validate materialAttributeValueUrl materialAttributeValueString - - let validateMaterial (materialString : string) = - let materialUrl = "https://raw.githubusercontent.com/HLWeil/isa-specs/anyof/source/_static/isajson/material_schema.json" - validate materialUrl materialString - - let validateOntologyAnnotation (ontologyAnnotationString : string) = - let ontologyAnnotationUrl = "https://raw.githubusercontent.com/HLWeil/isa-specs/anyof/source/_static/isajson/ontology_annotation_schema.json" - validate ontologyAnnotationUrl ontologyAnnotationString - - let validateOntologySourceReference (ontologySourceReferenceString : string) = - let ontologySourceReferenceUrl = "https://raw.githubusercontent.com/HLWeil/isa-specs/anyof/source/_static/isajson/ontology_source_reference_schema.json" - validate ontologySourceReferenceUrl ontologySourceReferenceString - - let validatePerson (personString : string) = - let personUrl = "https://raw.githubusercontent.com/HLWeil/isa-specs/anyof/source/_static/isajson/person_schema.json" - validate personUrl personString - - let validateProcessParameterValue (processParameterValueString : string) = - let processParameterValueUrl = "https://raw.githubusercontent.com/HLWeil/isa-specs/anyof/source/_static/isajson/process_parameter_value_schema.json" - validate processParameterValueUrl processParameterValueString - - let validateProcess (processString : string) = - let processUrl = "https://raw.githubusercontent.com/HLWeil/isa-specs/anyof/source/_static/isajson/process_schema.json" - validate processUrl processString - - let validateProtocolParameter (protocolParameterString : string) = - let protocolParameterUrl = "https://raw.githubusercontent.com/HLWeil/isa-specs/anyof/source/_static/isajson/protocol_parameter_schema.json" - validate protocolParameterUrl protocolParameterString - - let validateProtocol (protocolString : string) = - let protocolUrl = "https://raw.githubusercontent.com/HLWeil/isa-specs/anyof/source/_static/isajson/protocol_schema.json" - validate protocolUrl protocolString - - let validatePublication (publicationString : string) = - let publicationUrl = "https://raw.githubusercontent.com/HLWeil/isa-specs/anyof/source/_static/isajson/publication_schema.json" - validate publicationUrl publicationString - - let validateSample (sampleString : string) = - let sampleUrl = "https://raw.githubusercontent.com/HLWeil/isa-specs/anyof/source/_static/isajson/sample_schema.json" - validate sampleUrl sampleString - - let validateSource (sourceString : string) = - let sourceUrl = "https://raw.githubusercontent.com/HLWeil/isa-specs/anyof/source/_static/isajson/source_schema.json" - validate sourceUrl sourceString - - let validateStudy (studyString : string) = - let studyUrl = "https://raw.githubusercontent.com/HLWeil/isa-specs/anyof/source/_static/isajson/study_schema.json" - validate studyUrl studyString diff --git a/tests/ISA/ISA.Json.Tests/Validation/JsonSchemaValidation.fs b/tests/ISA/ISA.Json.Tests/Validation/JsonSchemaValidation.fs index 9f3064de..29686975 100644 --- a/tests/ISA/ISA.Json.Tests/Validation/JsonSchemaValidation.fs +++ b/tests/ISA/ISA.Json.Tests/Validation/JsonSchemaValidation.fs @@ -80,14 +80,18 @@ module JSchema = download 1 #endif + +#if !FABLE_COMPILER_PYTHON module Validation = let validate (schemaURL : string) (objectString : string) = async { try +#endif #if FABLE_COMPILER_JAVASCRIPT let! isValid, errorList = Fable.validate (schemaURL) (objectString) - #else + #endif + #if !FABLE_COMPILER let settings = NJsonSchema.Validation.JsonSchemaValidatorSettings() let schema = JSchema.tryDownloadSchema schemaURL let r = schema.Result.Validate(objectString,settings) @@ -97,6 +101,7 @@ module Validation = |> Seq.map (fun err -> err.ToString()) |> Seq.toArray #endif +#if !FABLE_COMPILER_PYTHON // if you change isValid and errorList remember to check for fable compatibility. // for exmaple must use same name as in `let! isValid, errorList =...` return ValidationResult.OfJSchemaOutput(isValid, errorList) @@ -165,3 +170,4 @@ module Validation = let validateStudy (studyString : string) = validate JsonSchemaUrls.Study studyString +#endif \ No newline at end of file diff --git a/tests/ISA/ISA.Json.Tests/ValidationResult.fs b/tests/ISA/ISA.Json.Tests/ValidationResult.fs deleted file mode 100644 index 2e806579..00000000 --- a/tests/ISA/ISA.Json.Tests/ValidationResult.fs +++ /dev/null @@ -1,20 +0,0 @@ -namespace ARCtrl.ISA.Validation - -type ValidationResult = - | Ok - | Failed of string [] - - member this.Success = - match this with - | Ok -> true - | _ -> false - - member this.GetErrors() = - match this with - | Ok -> [||] - | Failed errors -> errors - - static member internal OfJSchemaOutput (output : bool * string []) = - match output with - | true, _ -> Ok - | false, errors -> Failed errors \ No newline at end of file From 2540056063a107fb18ad30b577214e946a6ae48c Mon Sep 17 00:00:00 2001 From: HLWeil Date: Mon, 26 Feb 2024 15:04:53 +0100 Subject: [PATCH 09/25] small fix for compressed json io stringtable conversion https://github.com/fable-compiler/Fable/issues/3771 --- src/ISA/ISA.Json/StringTable.fs | 8 ++++---- tests/ISA/ISA.Json.Tests/ArcTypes.Tests.fs | 14 ++++++++++++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/ISA/ISA.Json/StringTable.fs b/src/ISA/ISA.Json/StringTable.fs index 5d5db241..c970d459 100644 --- a/src/ISA/ISA.Json/StringTable.fs +++ b/src/ISA/ISA.Json/StringTable.fs @@ -12,11 +12,11 @@ type StringTableArray = array module StringTable = - let arrayFromMap (otm : StringTableMap) : StringTableArray= + let arrayFromMap (otm : StringTableMap) : StringTableArray = + let a = Array.zeroCreate otm.Count otm - |> Seq.sortBy (fun kv -> kv.Value) - |> Seq.map (fun kv -> kv.Key) - |> Seq.toArray + |> Seq.iter (fun kv -> a.[kv.Value] <- kv.Key) + a let encoder (ot: StringTableArray) = ot diff --git a/tests/ISA/ISA.Json.Tests/ArcTypes.Tests.fs b/tests/ISA/ISA.Json.Tests/ArcTypes.Tests.fs index af9cb748..bd8c6c03 100644 --- a/tests/ISA/ISA.Json.Tests/ArcTypes.Tests.fs +++ b/tests/ISA/ISA.Json.Tests/ArcTypes.Tests.fs @@ -282,9 +282,19 @@ let tests_ArcAssay = testList "ArcAssay" [ for j = 0 to cells.Length - 1 do t.Values.[(j,i)] <- cells.[j] let f() = ArcAssay.toJsonString a + #if FABLE_COMPILER_JAVASCRIPT + let expectedMs = 5000 + #endif + #if FABLE_COMPILER_PYTHON + let expectedMs = 40000 + #endif + #if !FABLE_COMPILER + let expectedMs = 2000 + #endif // 1200ms in Dotnet on i7-13800H - // 3412ms in Javascript on i7-13800H // 5857 on another occasion - Expect.wantFaster f 7000 "toJsonString should be faster" |> ignore + // 3412ms in Javascript on i7-13800H + // 24562ms in Javascript on i7-13800H + Expect.wantFaster f expectedMs "toJsonString should be faster" |> ignore ] ] From c534acf43e16d1e76bb22bbc7e0d1e7134d213e5 Mon Sep 17 00:00:00 2001 From: HLWeil Date: Tue, 27 Feb 2024 10:49:22 +0100 Subject: [PATCH 10/25] fix and test comment regex handling for python --- src/ISA/ISA.Spreadsheet/Metadata/Comment.fs | 40 ++++++---- .../ISA.Spreadsheet/Metadata/SparseTable.fs | 2 +- .../ISA.Spreadsheet.Tests.fsproj | 2 +- tests/ISA/ISA.Spreadsheet.Tests/Main.fs | 1 + tests/ISA/ISA.Spreadsheet.Tests/RegexTests.fs | 74 +++++++++++++++++++ 5 files changed, 101 insertions(+), 18 deletions(-) create mode 100644 tests/ISA/ISA.Spreadsheet.Tests/RegexTests.fs diff --git a/src/ISA/ISA.Spreadsheet/Metadata/Comment.fs b/src/ISA/ISA.Spreadsheet/Metadata/Comment.fs index 453edecf..d731e25a 100644 --- a/src/ISA/ISA.Spreadsheet/Metadata/Comment.fs +++ b/src/ISA/ISA.Spreadsheet/Metadata/Comment.fs @@ -2,22 +2,27 @@ namespace ARCtrl.ISA.Spreadsheet open ARCtrl.ISA open System.Text.RegularExpressions +open ARCtrl.ISA.Regex.ActivePatterns module Comment = - let commentRegex = Regex(@"(?<=Comment\[<).*(?=>\])") - let commentRegexNoAngleBrackets = Regex(@"(?<=Comment\[).*(?=\])") + + let commentValueKey = "commentValue" - let (|Comment|_|) (key : Option) = - key - |> Option.bind (fun k -> - let r = commentRegex.Match(k) - if r.Success then Some r.Value - else - let r = commentRegexNoAngleBrackets.Match(k) - if r.Success then Some r.Value - else None - ) + let commentPattern = $@"Comment\s*\[<(?<{commentValueKey}>.+)>\]" + + let commentPatternNoAngleBrackets = $@"Comment\s*\[(?<{commentValueKey}>.+)\]" + + let (|Comment|_|) (key : string) = + + match key with + | Regex commentPattern r -> + Some r.Groups.[commentValueKey].Value + | Regex commentPatternNoAngleBrackets r -> + let v = r.Groups.[commentValueKey].Value + if v = "<>" then None else Some v + | _ -> None + let wrapCommentKey k = sprintf "Comment[%s]" k @@ -34,15 +39,18 @@ module Comment = module Remark = - let remarkRegex = Regex(@"(?<=#).*") + let remarkValueKey = "remarkValue" + + let remarkPattern = $@"#(?<{remarkValueKey}>.+)" let (|Remark|_|) (key : Option) = key |> Option.bind (fun k -> - let r = remarkRegex.Match(k) - if r.Success then Some r.Value - else None + match k with + | Regex remarkPattern r -> + Some r.Groups.[remarkValueKey].Value + | _ -> None ) diff --git a/src/ISA/ISA.Spreadsheet/Metadata/SparseTable.fs b/src/ISA/ISA.Spreadsheet/Metadata/SparseTable.fs index c720b20a..1c68dd93 100644 --- a/src/ISA/ISA.Spreadsheet/Metadata/SparseTable.fs +++ b/src/ISA/ISA.Spreadsheet/Metadata/SparseTable.fs @@ -131,7 +131,7 @@ type SparseTable = let row = en.Current |> Seq.map (fun (i,v) -> int i - 1,v) let key,vals = Seq.tryItem 0 row |> Option.map snd, Seq.trySkip 1 row match key,vals with - | Comment.Comment k, Option.Some v -> + | Option.Some (Comment.Comment k), Option.Some v -> loop (SparseTable.AddComment k v matrix) remarks (lineNumber + 1) | Remark.Remark k, _ -> diff --git a/tests/ISA/ISA.Spreadsheet.Tests/ISA.Spreadsheet.Tests.fsproj b/tests/ISA/ISA.Spreadsheet.Tests/ISA.Spreadsheet.Tests.fsproj index 343a18c3..8728b2bf 100644 --- a/tests/ISA/ISA.Spreadsheet.Tests/ISA.Spreadsheet.Tests.fsproj +++ b/tests/ISA/ISA.Spreadsheet.Tests/ISA.Spreadsheet.Tests.fsproj @@ -7,7 +7,7 @@ - + diff --git a/tests/ISA/ISA.Spreadsheet.Tests/Main.fs b/tests/ISA/ISA.Spreadsheet.Tests/Main.fs index 14e0be7e..77b328f5 100644 --- a/tests/ISA/ISA.Spreadsheet.Tests/Main.fs +++ b/tests/ISA/ISA.Spreadsheet.Tests/Main.fs @@ -4,6 +4,7 @@ open Fable.Pyxpecto let all = testSequenced <| testList "ISA.Spreadsheet" [ FableTests.main + RegexTests.main ArcInvestigationTests.main CompositeColumnTests.main ArcTableTests.main diff --git a/tests/ISA/ISA.Spreadsheet.Tests/RegexTests.fs b/tests/ISA/ISA.Spreadsheet.Tests/RegexTests.fs new file mode 100644 index 00000000..6133272f --- /dev/null +++ b/tests/ISA/ISA.Spreadsheet.Tests/RegexTests.fs @@ -0,0 +1,74 @@ +module RegexTests + +open ARCtrl.ISA + +open TestingUtils + +open ARCtrl.ISA.Spreadsheet.Comment + +let comment_tests = + testList "Comment" [ + testCase "Simple[<" (fun () -> + let value = "Name" + let key = $"Comment[<{value}>]" + let result = + match key with + | Comment r -> Some r + | _ -> None + let r = Expect.wantSome result "Matching correct comment against regex did not return result" + Expect.equal value r "Matching correct comment against regex did not return correct value" + ) + testCase "Simple [<" (fun () -> + let value = "Name" + let key = $"Comment [<{value}>]" + let result = + match key with + | Comment r -> Some r + | _ -> None + let r = Expect.wantSome result "Matching correct comment against regex did not return result" + Expect.equal value r "Matching correct comment against regex did not return correct value" + ) + testCase "Simple[" (fun () -> + let value = "Name" + let key = $"Comment[<{value}>]" + let result = + match key with + | Comment r -> Some r + | _ -> None + let r = Expect.wantSome result "Matching correct comment against regex did not return result" + Expect.equal value r "Matching correct comment against regex did not return correct value" + ) + testCase "Simple [" (fun () -> + let value = "Name" + let key = $"Comment [{value}]" + let result = + match key with + | Comment r -> Some r + | _ -> None + let r = Expect.wantSome result "Matching correct comment against regex did not return result" + Expect.equal value r "Matching correct comment against regex did not return correct value" + ) + testCase "Wrong (" (fun () -> + let value = "Name" + let key = $"Comment ({value})" + let result = + match key with + | Comment r -> Some r + | _ -> None + Expect.isNone result "Matching incorrect comment against regex did not return None" + ) + testCase "Wrong NoValue [<" (fun () -> + let key = "Comment [<>]" + let result = + match key with + | Comment r -> Some r + | _ -> None + Expect.isNone result "Matching incorrect comment against regex did not return None" + ) + ] + + +let main = + testList "Regex" [ + comment_tests + ] \ No newline at end of file From 556e2bf7f0bd85417ce43078ff3c2357819bef1d Mon Sep 17 00:00:00 2001 From: HLWeil Date: Tue, 27 Feb 2024 10:52:13 +0100 Subject: [PATCH 11/25] fix http requests in python --- ARCtrl.sln | 1 + poetry.lock | 79 ++++++ pyproject.toml | 14 + src/ARCtrl/ARCtrl.fsproj | 2 + src/ARCtrl/WebRequest/WebRequest.Node.fs | 33 +++ src/ARCtrl/WebRequest/WebRequest.Py.fs | 118 +++++++++ src/ARCtrl/WebRequest/WebRequest.fs | 50 +--- .../StringConversionTests.fs | 244 ------------------ 8 files changed, 258 insertions(+), 283 deletions(-) create mode 100644 poetry.lock create mode 100644 pyproject.toml create mode 100644 src/ARCtrl/WebRequest/WebRequest.Node.fs create mode 100644 src/ARCtrl/WebRequest/WebRequest.Py.fs delete mode 100644 tests/ISA/ISA.Spreadsheet.Tests/StringConversionTests.fs diff --git a/ARCtrl.sln b/ARCtrl.sln index 804fe8ec..04467019 100644 --- a/ARCtrl.sln +++ b/ARCtrl.sln @@ -12,6 +12,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution .config\dotnet-tools.json = .config\dotnet-tools.json global.json = global.json package.json = package.json + pyproject.toml = pyproject.toml README.md = README.md RELEASE_NOTES.md = RELEASE_NOTES.md EndProjectSection diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 00000000..b4ec8d23 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,79 @@ +# This file is automatically @generated by Poetry 1.8.1 and should not be changed by hand. + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "2.1.1" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.6.0" +files = [ + {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"}, + {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, +] + +[package.extras] +unicode-backport = ["unicodedata2"] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "requests" +version = "2.28.1" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7, <4" +files = [ + {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, + {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<3" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "urllib3" +version = "1.26.18" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"}, + {file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"}, +] + +[package.extras] +brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.10" +content-hash = "94c05cf3d10061c7909f56cbd76bde18dd178b4a85ace087cc31dfbc4827bfa8" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..ed391162 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,14 @@ +[tool.poetry] +name = "ARCtrl" +version = "1.0.0" +description = "Library for management of Annotated Research Contexts (ARCs) using an in-memory representation and runtimer agnostic contract systems." +authors = ["Heinrich Lukas Weil "] +readme = "README.md" + +[tool.poetry.dependencies] +python = "^3.10" +requests = "2.28.1" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" \ No newline at end of file diff --git a/src/ARCtrl/ARCtrl.fsproj b/src/ARCtrl/ARCtrl.fsproj index 4ac4b723..79de8a7e 100644 --- a/src/ARCtrl/ARCtrl.fsproj +++ b/src/ARCtrl/ARCtrl.fsproj @@ -21,6 +21,8 @@ + + diff --git a/src/ARCtrl/WebRequest/WebRequest.Node.fs b/src/ARCtrl/WebRequest/WebRequest.Node.fs new file mode 100644 index 00000000..a5a5eb38 --- /dev/null +++ b/src/ARCtrl/WebRequest/WebRequest.Node.fs @@ -0,0 +1,33 @@ +module ARCtrl.WebRequestHelpers.NodeJs + +open Fable.Core +open Fable.SimpleHttp +let x = 1 +#if FABLE_COMPILER_JAVASCRIPT +open Fable.Core.JsInterop + +open Fable.SimpleHttp + +open Fable.Core +open Fable.Core.JsInterop +open Fetch + +//// From here: https://github.com/fable-compiler/fable3-samples/blob/25ea2404b28c897988b144f0141bc116da292679/nodejs/src/App.fs#L7 + +let importSideEffects() = importSideEffects "isomorphic-fetch" + +let isNode() = + emitJsExpr + () + """typeof process !== "undefined" && +process.versions != null && +process.versions.node != null;""" + +let downloadFile url = + fetch url [] + |> Promise.bind (fun res -> res.text()) // get the result + |> Promise.map (fun txt -> // bind the result to make further operation + txt + ) + |> Async.AwaitPromise +#endif \ No newline at end of file diff --git a/src/ARCtrl/WebRequest/WebRequest.Py.fs b/src/ARCtrl/WebRequest/WebRequest.Py.fs new file mode 100644 index 00000000..05891b28 --- /dev/null +++ b/src/ARCtrl/WebRequest/WebRequest.Py.fs @@ -0,0 +1,118 @@ +module ARCtrl.WebRequestHelpers.Py + +open Fable.Core + +// Taken from https://github.com/Zaid-Ajaj/Fable.Requests/blob/master/src/Library.fs +// Can be replaced either with Fable.SimpleHttp when it supports Python +// or Fable.Requests when it supports netstandard2.0 + + +#if FABLE_COMPILER_PYTHON +type private InteropResponseType = + abstract status_code : int + abstract text : string + abstract encoding : string + abstract headers : obj + +[] +module Interop = + [] + let get<'T> (key: string) (dict: obj) : 'T = nativeOnly + [] + let keys (dict: obj) : string array = nativeOnly + +type Response = { + statusCode: int + text: string + headers: Map + encoding: string +} + +[] +type private RequestsApi = + [] + abstract get: url:string * ?headers:obj -> InteropResponseType + [] + abstract post: url: string * ?data: string * ?headers:obj -> InteropResponseType + [] + abstract put: url: string * ?data: string * ?headers:obj -> InteropResponseType + [] + abstract delete: url: string * ?data:string * ?headers:obj -> InteropResponseType + [] + abstract head: url:string * ?data: string * ?headers:obj -> InteropResponseType + [] + abstract options: url:string * ?data: string * ?headers:obj -> InteropResponseType + +type Requests() = + static member private createHeadersDict(headers:Map option) = + headers + |> Option.map (fun values -> PyInterop.createObj [ for pair in values -> pair.Key, box pair.Value ]) + + static member private mapResponseType(response: InteropResponseType) : Response = { + statusCode = response.status_code + text = response.text + encoding = response.text + headers = Map.ofList [ + for headerName in Interop.keys response.headers do + let headerValue = Interop.get headerName response.headers + headerName, headerValue + ] + } + + [] + static member private requestsApi: RequestsApi = nativeOnly + + /// + /// Sends a GET request to the specified URL and returns a response. + /// + static member get(url: string, ?headers:Map) : Response = + let headersDict = Requests.createHeadersDict headers + let response = Requests.requestsApi.get(url, ?headers=headersDict) + Requests.mapResponseType response + + /// + /// Sends a POST request to the specified URL and returns a response. + /// + static member post(url: string, ?data: string, ?headers:Map) : Response = + let headersDict = Requests.createHeadersDict headers + let response: InteropResponseType = Requests.requestsApi.post(url, ?data=data, ?headers=headersDict) + Requests.mapResponseType response + + /// + /// Sends a PUT request to the specified URL and returns a response. + /// + static member put(url: string, ?data: string, ?headers:Map) : Response = + let headersDict = Requests.createHeadersDict headers + let response: InteropResponseType = Requests.requestsApi.put(url, ?data=data, ?headers=headersDict) + Requests.mapResponseType response + + /// + /// Sends a DELETE request to the specified URL and returns a response. + /// + static member delete(url: string, ?data: string, ?headers:Map) : Response = + let headersDict = Requests.createHeadersDict headers + let response: InteropResponseType = Requests.requestsApi.delete(url, ?data=data, ?headers=headersDict) + Requests.mapResponseType response + + /// + /// Sends a HEAD request to the specified URL and returns a response. + /// + static member head(url: string, ?data: string, ?headers:Map) : Response = + let headersDict = Requests.createHeadersDict headers + let response: InteropResponseType = Requests.requestsApi.head(url, ?data=data, ?headers=headersDict) + Requests.mapResponseType response + + /// + /// Sends an OPTIONS request to the specified URL and returns a response. + /// + static member options(url: string, ?data: string, ?headers:Map) : Response = + let headersDict = Requests.createHeadersDict headers + let response: InteropResponseType = Requests.requestsApi.head(url, ?data=data, ?headers=headersDict) + Requests.mapResponseType response + +let downloadFile url = + async { + let response = Requests.get url + return response.text + } +#endif \ No newline at end of file diff --git a/src/ARCtrl/WebRequest/WebRequest.fs b/src/ARCtrl/WebRequest/WebRequest.fs index 4b7bac9d..0c1a1554 100644 --- a/src/ARCtrl/WebRequest/WebRequest.fs +++ b/src/ARCtrl/WebRequest/WebRequest.fs @@ -3,38 +3,6 @@ open Fable.Core open Fable.SimpleHttp -#if FABLE_COMPILER_JAVASCRIPT -open Fable.Core.JsInterop - -open Fable.SimpleHttp - -[] -module NodeJs = - open Fable.Core - open Fable.Core.JsInterop - open Fetch - - //// From here: https://github.com/fable-compiler/fable3-samples/blob/25ea2404b28c897988b144f0141bc116da292679/nodejs/src/App.fs#L7 - //#if FABLE_COMPILER - //importSideEffects "isomorphic-fetch" - //#endif - - let isNode() = - emitJsExpr - () - """typeof process !== "undefined" && -process.versions != null && -process.versions.node != null;""" - - let downloadFile url = - fetch url [] - |> Promise.bind (fun res -> res.text()) // get the result - |> Promise.map (fun txt -> // bind the result to make further operation - txt - ) - |> Async.AwaitPromise -#endif - let downloadFile url = let browserAndDotnet() = async { @@ -46,15 +14,19 @@ let downloadFile url = | _ -> failwithf "Status %d => %s" statusCode responseText } #if FABLE_COMPILER_JAVASCRIPT - if NodeJs.isNode() then + + if ARCtrl.WebRequestHelpers.NodeJs.isNode() then // From here: https://github.com/fable-compiler/fable3-samples/blob/25ea2404b28c897988b144f0141bc116da292679/nodejs/src/App.fs#L7 - importSideEffects "isomorphic-fetch" - NodeJs.downloadFile url + ARCtrl.WebRequestHelpers.NodeJs.importSideEffects() + ARCtrl.WebRequestHelpers.NodeJs.downloadFile url else browserAndDotnet() - #else - - browserAndDotnet() #endif - + #if FABLE_COMPILER_PYTHON + ARCtrl.WebRequestHelpers.Py.downloadFile url + #endif + + #if !FABLE_COMPILER + browserAndDotnet() + #endif \ No newline at end of file diff --git a/tests/ISA/ISA.Spreadsheet.Tests/StringConversionTests.fs b/tests/ISA/ISA.Spreadsheet.Tests/StringConversionTests.fs deleted file mode 100644 index 6d740bf2..00000000 --- a/tests/ISA/ISA.Spreadsheet.Tests/StringConversionTests.fs +++ /dev/null @@ -1,244 +0,0 @@ -module StringConversionTests - -open ARCtrl.ISA - -open TestingUtils - -let singleItems = - - testList "SingleItems" [ - - //testCase "FullStrings" (fun () -> - // let name = "Name" - // let term = "Term" - // let accession = "Accession" - // let source = "Source" - - // let testOntology = OntologyAnnotation.make None (Some (Text term)) (Some source) (Some accession) None - // let ontology = OntologyAnnotation.fromString(term,source,accession) - - // Expect.equal ontology testOntology "Ontology Annotation was not created correctly from strings" - // Expect.equal (OntologyAnnotation.toString ontology) (term,source,accession) "Ontology Annotation was not parsed correctly to strings" - - // let testComponent = Component.make (Some name) (Some (Value.Name name)) None (Some testOntology) - // let componentx = Component.fromString name term source accession - - // Expect.equal componentx testComponent "Component was not created correctly from strings" - // Expect.equal (Component.toString componentx) (name,term,source,accession) "Component was not parsed correctly to strings" - - // let testPParam = ProtocolParameter.make None (Some testOntology) - // let pParam = ProtocolParameter.fromString term source accession - - // Expect.equal pParam testPParam "Protocol Parameter was not created correctly from strings" - // Expect.equal (ProtocolParameter.toString pParam) (term,source,accession) "Protocol Parameter was not parsed correctly to strings" - //) - //testCase "EmptyString" (fun () -> - // let name = "" - // let term = "" - // let accession = "" - // let source = "" - - // let testOntology = OntologyAnnotation.make None None None None None - // let ontology = OntologyAnnotation.fromString term accession source - - // Expect.equal ontology testOntology "Empty Ontology Annotation was not created correctly from strings" - // Expect.equal (OntologyAnnotation.toString ontology) (term,accession,source) "Empty Ontology Annotation was not parsed correctly to strings" - - // let testComponent = Component.make None None None None - // let componentx = Component.fromString name term accession source - - // Expect.equal componentx testComponent "Empty Component was not created correctly from strings" - // Expect.equal (Component.toString componentx) (name,term,accession,source) "Empty Component was not parsed correctly to strings" - - // let testPParam = ProtocolParameter.make None None - // let pParam = ProtocolParameter.fromString term accession source - - // Expect.equal pParam testPParam "Empty Protocol Parameter was not created correctly from strings" - // Expect.equal (ProtocolParameter.toString pParam) (term,accession,source) "Empty Protocol Parameter was not parsed correctly to strings" - //) - - ] - - -let aggregated = - testList "StringAggregationTests" [ - - //testCase "FullStrings" (fun () -> - // let names = "Name1;Name2" - // let terms = "Term1;Term2" - // let accessions = "Accession1;Accession2" - // let sources = "Source1;Source2" - - // let testOntologies = - // [ - // OntologyAnnotation.fromString "Term1" "Accession1" "Source1" - // OntologyAnnotation.fromString "Term2" "Accession2" "Source2" - // ] - // let ontologies = OntologyAnnotation.fromAggregatedStrings ';' terms accessions sources - - // Expect.sequenceEqual ontologies testOntologies "Ontology Annotations were not created correctly from aggregated strings" - // Expect.equal (OntologyAnnotation.toAggregatedStrings ';' ontologies) (terms,accessions,sources) "Ontology Annotations were not parsed correctly to aggregated strings" - - // let testComponents = - // [ - // Component.fromString "Name1" "Term1" "Accession1" "Source1" - // Component.fromString "Name2" "Term2" "Accession2" "Source2" - // ] - // let components = Component.fromAggregatedStrings ';' names terms accessions sources - - // Expect.sequenceEqual components testComponents "Components were not created correctly from aggregated strings" - // Expect.equal (Component.toAggregatedStrings ';' components) (names,terms,accessions,sources) "Components were not parsed correctly to aggregated strings" - - // let testPParams = - // [ - // ProtocolParameter.fromString "Term1" "Accession1" "Source1" - // ProtocolParameter.fromString "Term2" "Accession2" "Source2" - // ] - // let pParams = ProtocolParameter.fromAggregatedStrings ';' terms accessions sources - - // Expect.sequenceEqual pParams testPParams "Protocol Parameters were not created correctly from aggregated strings" - // Expect.equal (ProtocolParameter.toAggregatedStrings ';' pParams) (terms,accessions,sources) "Protocol Parameters were not parsed correctly to aggregated strings" - //) - - //testCase "EmptyStrings" (fun () -> - // let names = "" - // let terms = "" - // let accessions = "" - // let sources = "" - - // let testOntologies = [ ] - // let ontologies = OntologyAnnotation.fromAggregatedStrings ';' terms accessions sources - - // Expect.sequenceEqual ontologies testOntologies "Ontology Annotations were not created correctly from empty aggregated strings. Empty strings should results in an empty list" - // Expect.equal (OntologyAnnotation.toAggregatedStrings ';' ontologies) (terms,accessions,sources) "Ontology Annotations were not parsed correctly to empty aggregated strings" - - // let testComponents = [] - // let components = Component.fromAggregatedStrings ';' names terms accessions sources - - // Expect.sequenceEqual components testComponents "Components were not created correctly from empty aggregated strings. Empty strings should results in an empty list" - // Expect.equal (Component.toAggregatedStrings ';' components) (names,terms,accessions,sources) "Components were not parsed correctly to empty aggregated strings" - - // let testPParams = [] - // let pParams = ProtocolParameter.fromAggregatedStrings ';' terms accessions sources - - // Expect.sequenceEqual pParams testPParams "Protocol Parameters were not created correctly from aggregated strings. Empty strings should results in an empty list" - // Expect.equal (ProtocolParameter.toAggregatedStrings ';' pParams) (terms,accessions,sources) "Protocol Parameters were not parsed correctly to aggregated strings" - //) - //testCase "PartlyEmptyStrings" (fun () -> - // let names = "" - // let terms = "Term1;Term2" - // let accessions = "Accession1;Accession2" - // let sources = "" - - // let testOntologies = - // [ - // OntologyAnnotation.fromString "Term1" "Accession1" "" - // OntologyAnnotation.fromString "Term2" "Accession2" "" - // ] - // let ontologies = OntologyAnnotation.fromAggregatedStrings ';' terms accessions sources - - // Expect.sequenceEqual ontologies testOntologies "Ontology Annotations were not created correctly from partly empty aggregated strings" - // Expect.equal (OntologyAnnotation.toAggregatedStrings ';' ontologies) (terms,accessions,";") "Ontology Annotations were not parsed correctly to partly empty aggregated strings" - - // let testComponents = - // [ - // Component.fromString "" "Term1" "Accession1" "" - // Component.fromString "" "Term2" "Accession2" "" - // ] - // let components = Component.fromAggregatedStrings ';' names terms accessions sources - - // Expect.sequenceEqual components testComponents "Components were not created correctly from partly empty aggregated strings" - // Expect.equal (Component.toAggregatedStrings ';' components) (";",terms,accessions,";") "Components were not parsed correctly to partly empty aggregated strings" - - // let testPParams = - // [ - // ProtocolParameter.fromString "Term1" "Accession1" "" - // ProtocolParameter.fromString "Term2" "Accession2" "" - // ] - // let pParams = ProtocolParameter.fromAggregatedStrings ';' terms accessions sources - - // Expect.sequenceEqual pParams testPParams "Protocol Parameters were not created correctly from partly empty aggregated strings" - // Expect.equal (ProtocolParameter.toAggregatedStrings ';' pParams) (terms,accessions,";") "Protocol Parameters were not parsed correctly to partly empty aggregated strings" - //) - - //testCase "DifferingLengths" (fun () -> - // let names = "Name1" - // let terms = "Term1;Term2" - // let accessions = "Accession1;Accession2" - // let sources = "Accession2" - - // let ontologies = - // try OntologyAnnotation.fromAggregatedStrings ';' terms accessions sources |> Some - // with - // | _ -> None - - // Expect.isNone ontologies "Parsing aggregated string to ontologies should have failed because of differing lengths" - - // let components = - // try Component.fromAggregatedStrings ';' names terms accessions sources |> Some - // with - // | _ -> None - - // Expect.isNone components "Parsing aggregated string to compnents should have failed because of differing lengths" - - // let pParams = - // try ProtocolParameter.fromAggregatedStrings ';' terms accessions sources |> Some - // with - // | _ -> None - - // Expect.isNone pParams "Parsing aggregated string to protocol parameters should have failed because of differing lengths" - - //) - ] - -let value = - testList "Value" [ - //testCase "ParseOntology"(fun () -> - - // let value = Value.fromOptions (Some "Name") (Some "Source") (Some "Accession") - - // Expect.isSome value "Should have returned Value but returned None" - - // let expectedAnnotationValue = AnnotationValue.Text "Name" - // let expectedAnnotation = OntologyAnnotation.make None (Some expectedAnnotationValue) (Some "Source") (Some "Accession") None - // let expectedValue = Value.Ontology expectedAnnotation - - // Expect.equal value.Value expectedValue "Value was parsed incorrectly" - //) - //testCase "ParseText"(fun () -> - - // let value = Value.fromOptions (Some "Name") None None - - // Expect.isSome value "Should have returned Value but returned None" - - // let expectedValue = Value.Name "Name" - - // Expect.equal value.Value expectedValue "Value was parsed incorrectly" - //) - //testCase "ParseInt"(fun () -> - - // let value = Value.fromOptions (Some "5") None None - - // Expect.isSome value "Should have returned Value but returned None" - - // let expectedValue = Value.Int 5 - - // Expect.equal value.Value expectedValue "Value was parsed incorrectly" - //) - //testCase "ParseFloat"(fun () -> - - // let value = Value.fromOptions (Some "2.3") None None - - // Expect.isSome value "Should have returned Value but returned None" - - // let expectedValue = Value.Float 2.3 - - // Expect.equal value.Value expectedValue "Value was parsed incorrectly" - //) - - ] - - -let main = - testList "StringConversion" [ - ] \ No newline at end of file From b5eaaedd24cad0a24423aec39dbdf56bbaac14dd Mon Sep 17 00:00:00 2001 From: HLWeil Date: Tue, 27 Feb 2024 22:52:35 +0100 Subject: [PATCH 12/25] hotfix fable python hashing of person --- build/TestTasks.fs | 9 ++++---- src/ARCtrl/WebRequest/WebRequest.Py.fs | 1 - src/Contract/Contract.fs | 6 +++++ src/ISA/ISA/Helper.fs | 20 ++++++++++++++++- src/ISA/ISA/JsonTypes/Person.fs | 22 ++++++++++++++++++- src/ISA/ISA/JsonTypes/Publication.fs | 2 +- .../ISA.Spreadsheet.Tests/Identifier.Tests.fs | 8 +++---- .../Performance.Tests.fs | 15 ++++++++++--- 8 files changed, 68 insertions(+), 15 deletions(-) diff --git a/build/TestTasks.fs b/build/TestTasks.fs index a9f8eb25..5b1b3b28 100644 --- a/build/TestTasks.fs +++ b/build/TestTasks.fs @@ -41,9 +41,10 @@ module RunTests = /// let testProjectsPy = [ - "tests/ISA/ISA.Tests" - "tests/ISA/ISA.Json.Tests" - "tests/ISA/ISA.Spreadsheet.Tests" + //"tests/ISA/ISA.Tests" + //"tests/ISA/ISA.Json.Tests" + //"tests/ISA/ISA.Spreadsheet.Tests" + "tests/ARCtrl" ] let runTestsPy = BuildTask.create "runTestsPy" [clean; build] { @@ -60,6 +61,6 @@ module RunTests = |> Seq.iter dotnetRun } -let runTests = BuildTask.create "RunTests" [clean; build; RunTests.runTestsJs; RunTests.runTestsJsNative; (*RunTests.runTestsUI;*) RunTests.runTestsDotnet] { +let runTests = BuildTask.create "RunTests" [clean; build; RunTests.runTestsJs; RunTests.runTestsJsNative; RunTests.runTestsPy; RunTests.runTestsDotnet] { () } \ No newline at end of file diff --git a/src/ARCtrl/WebRequest/WebRequest.Py.fs b/src/ARCtrl/WebRequest/WebRequest.Py.fs index 05891b28..093e8b45 100644 --- a/src/ARCtrl/WebRequest/WebRequest.Py.fs +++ b/src/ARCtrl/WebRequest/WebRequest.Py.fs @@ -6,7 +6,6 @@ open Fable.Core // Can be replaced either with Fable.SimpleHttp when it supports Python // or Fable.Requests when it supports netstandard2.0 - #if FABLE_COMPILER_PYTHON type private InteropResponseType = abstract status_code : int diff --git a/src/Contract/Contract.fs b/src/Contract/Contract.fs index 7dbff9db..45628e83 100644 --- a/src/Contract/Contract.fs +++ b/src/Contract/Contract.fs @@ -3,7 +3,9 @@ open Fable.Core open Fable.Core.JsInterop +#if FABLE_COMPILER_JAVASCRIPT [] +#endif [] type DTOType = | [] ISA_Assay // isa.assay.xlsx @@ -25,7 +27,9 @@ type CLITool = static member create(name,arguments) = {Name = name; Arguments = arguments} +#if FABLE_COMPILER_JAVASCRIPT [] +#endif [] type DTO = | Spreadsheet of obj @@ -70,7 +74,9 @@ type Contract = /// The actual DTO, as discriminate union. DTO: DTO option } + #if FABLE_COMPILER_JAVASCRIPT [] + #endif static member create(op, path, ?dtoType, ?dto) = {Operation= op; Path = path; DTOType = dtoType; DTO = dto} /// Create a CREATE contract with all necessary information. /// The path relative from ARC root, at which the new file should be created. diff --git a/src/ISA/ISA/Helper.fs b/src/ISA/ISA/Helper.fs index c2c7785e..5389fed9 100644 --- a/src/ISA/ISA/Helper.fs +++ b/src/ISA/ISA/Helper.fs @@ -75,6 +75,24 @@ module ResizeArray = a.Count = 0 module HashCodes = + + open Fable.Core + + [] + let pyHasCustomHash (obj) : bool = nativeOnly + + [] + let pyCustomHash (obj) : int = nativeOnly + + let hash obj = + #if FABLE_COMPILER_PYTHON + if pyHasCustomHash obj then + pyCustomHash obj + else + obj.GetHashCode() + #else + obj.GetHashCode() + #endif let boxHashOption (a: 'a option) : obj = if a.IsSome then a.Value.GetHashCode() else (0).GetHashCode() @@ -83,7 +101,7 @@ module HashCodes = let boxHashArray (a: 'a []) : obj = a // from https://stackoverflow.com/a/53507559 - |> Array.fold (fun acc o -> 0x9e3779b9 + o.GetHashCode() + (acc <<< 6) + (acc >>> 2) ) 0 + |> Array.fold (fun acc o -> 0x9e3779b9 + (hash o) + (acc <<< 6) + (acc >>> 2) ) 0 |> box let boxHashSeq (a: seq<'a>) : obj = diff --git a/src/ISA/ISA/JsonTypes/Person.fs b/src/ISA/ISA/JsonTypes/Person.fs index 0fa4e77e..c5c7beb4 100644 --- a/src/ISA/ISA/JsonTypes/Person.fs +++ b/src/ISA/ISA/JsonTypes/Person.fs @@ -4,7 +4,7 @@ open ARCtrl.ISA.Aux open Update open Fable.Core -[] +[][][] type Person = { ID : URI option @@ -180,3 +180,23 @@ type Person = this.Affiliation nextRoles nextComments + + override this.GetHashCode() : int = + [| + Aux.HashCodes.boxHashOption this.ORCID + Aux.HashCodes.boxHashOption this.LastName + Aux.HashCodes.boxHashOption this.FirstName + Aux.HashCodes.boxHashOption this.MidInitials + Aux.HashCodes.boxHashOption this.EMail + Aux.HashCodes.boxHashOption this.Phone + Aux.HashCodes.boxHashOption this.Fax + Aux.HashCodes.boxHashOption this.Address + Aux.HashCodes.boxHashOption this.Affiliation + if this.Roles.IsSome then Aux.HashCodes.boxHashArray this.Roles.Value else (0).GetHashCode() + if this.Comments.IsSome then Aux.HashCodes.boxHashArray this.Comments.Value else (0).GetHashCode() + |] + |> Aux.HashCodes.boxHashArray + |> fun x -> x :?> int + + override this.Equals(obj) : bool = + Aux.HashCodes.hash this = Aux.HashCodes.hash obj \ No newline at end of file diff --git a/src/ISA/ISA/JsonTypes/Publication.fs b/src/ISA/ISA/JsonTypes/Publication.fs index 9408f2df..aa00b7e5 100644 --- a/src/ISA/ISA/JsonTypes/Publication.fs +++ b/src/ISA/ISA/JsonTypes/Publication.fs @@ -5,7 +5,7 @@ open Update open Fable.Core [] -type Publication = +type Publication = // ScholarlyArticle { PubMedID : URI option DOI : string option diff --git a/tests/ISA/ISA.Spreadsheet.Tests/Identifier.Tests.fs b/tests/ISA/ISA.Spreadsheet.Tests/Identifier.Tests.fs index 36f920f9..4d8e4264 100644 --- a/tests/ISA/ISA.Spreadsheet.Tests/Identifier.Tests.fs +++ b/tests/ISA/ISA.Spreadsheet.Tests/Identifier.Tests.fs @@ -5,7 +5,7 @@ open TestingUtils open ARCtrl.ISA.Identifier // Function to test identifierFromFileName -let private testIdentifierFromFileName () = +let private initTestIdentifierFromFileName () = let validTestFileNames = [ @"assays/MyAssay/isa.assay.xlsx", "MyAssay" @"MyAssay/isa.assay.xlsx", "MyAssay" @@ -39,7 +39,7 @@ let private testIdentifierFromFileName () = validTestCases@invalidTestCases // Function to test fileNameFromIdentifier -let private testFileNameFromIdentifier () = +let private initTestFileNameFromIdentifier () = let validTestIdentifier = [ "MyAssay", "assays/MyAssay/isa.assay.xlsx" "My_Assay", "assays/My_Assay/isa.assay.xlsx" @@ -68,14 +68,14 @@ let private testFileNameFromIdentifier () = validTestCases@invalidTestCases let test_identifierFromFileName = testList "identifierFromFileName" [ - yield! testIdentifierFromFileName () + yield! initTestIdentifierFromFileName () testCase "createMissingIdentifier" <| fun _ -> let pseudoFileName = createMissingIdentifier() let identifierFromFileName = Assay.identifierFromFileName pseudoFileName Expect.equal identifierFromFileName pseudoFileName "" ] -let test_fileNameFromIdentifier = testList "fileNameFromIdentifier" (testFileNameFromIdentifier ()) +let test_fileNameFromIdentifier = testList "fileNameFromIdentifier" (initTestFileNameFromIdentifier ()) let tests_Assay = testList "Assay" [ test_identifierFromFileName diff --git a/tests/ISA/ISA.Spreadsheet.Tests/Performance.Tests.fs b/tests/ISA/ISA.Spreadsheet.Tests/Performance.Tests.fs index d2e49a0f..e344213f 100644 --- a/tests/ISA/ISA.Spreadsheet.Tests/Performance.Tests.fs +++ b/tests/ISA/ISA.Spreadsheet.Tests/Performance.Tests.fs @@ -13,9 +13,13 @@ let private tests_Study = testList "Study" [ let s,_ = ArcStudy.fromFsWorkbook fswb let timer_end = System.DateTime.Now let runtime = (timer_end - timer_start).Milliseconds - let expected = 300 // this is too high and should be reduced + #if FABLE_COMPILER_PYTHON + let expectedMs = 1500 + #else + let expectedMs = 300 // this is too high and should be reduced + #endif Expect.equal s.TableCount 1 "Table count" - Expect.isTrue (runtime <= expected) $"Expected conversion to be finished in under {expected}, but it took {runtime}" + Expect.isTrue (runtime <= expectedMs) $"Expected conversion to be finished in under {expectedMs}, but it took {runtime}" convertToArcFile fswb ] @@ -26,7 +30,12 @@ let private tests_Investigation = testList "Investigation" [ let s = ArcStudy.init($"Study{i}") inv.AddRegisteredStudy(s) let testF = fun () -> ArcInvestigation.toFsWorkbook inv - let wb = Expect.wantFaster testF 1000 "Parsing investigation to Workbook is too slow" + #if FABLE_COMPILER_PYTHON + let expectedMs = 50000 + #else + let expectedMs = 1000 + #endif + let wb = Expect.wantFaster testF expectedMs "Parsing investigation to Workbook is too slow" Expect.equal (wb.GetWorksheets().Count) 1 "Worksheet count" ] From 66b83a2292b082e96676a52c1dd176695173f949 Mon Sep 17 00:00:00 2001 From: HLWeil Date: Wed, 28 Feb 2024 13:14:54 +0100 Subject: [PATCH 13/25] several small changes to test stack --- build/TestTasks.fs | 2 +- src/ARCtrl/Templates/Template.Json.fs | 4 ++++ src/ARCtrl/WebRequest/WebRequest.Node.fs | 2 +- src/ISA/ISA.Json/ARCtrl.ISA.Json.fsproj | 5 ----- src/ISA/ISA/Helper.fs | 2 ++ tests/ISA/ISA.Json.Tests/ArcTypes.Tests.fs | 4 ++-- tests/ISA/ISA.Json.Tests/ISA.Json.Tests.fsproj | 6 +++++- tests/ISA/ISA.Tests/ArcTable.Tests.fs | 2 +- tests/JavaScript/ISA.CompositeCell.js | 2 +- tests/JavaScript/ISA.CompositeHeader.js | 2 +- 10 files changed, 18 insertions(+), 13 deletions(-) diff --git a/build/TestTasks.fs b/build/TestTasks.fs index 5b1b3b28..8ebc7921 100644 --- a/build/TestTasks.fs +++ b/build/TestTasks.fs @@ -48,7 +48,7 @@ module RunTests = ] let runTestsPy = BuildTask.create "runTestsPy" [clean; build] { - for path in testProjectsPy do + for path in ProjectInfo.testProjects do //transpile py files from fsharp code run dotnet $"fable {path} -o {path}/py --lang python" "" // run pyxpecto in target path to execute tests in python diff --git a/src/ARCtrl/Templates/Template.Json.fs b/src/ARCtrl/Templates/Template.Json.fs index 8ed20d28..78c4260e 100644 --- a/src/ARCtrl/Templates/Template.Json.fs +++ b/src/ARCtrl/Templates/Template.Json.fs @@ -61,7 +61,11 @@ module Template = get.Required.Field "authors" (Decode.array personDecoder), get.Required.Field "endpoint_repositories" (Decode.array oaDecoder), get.Required.Field "tags" (Decode.array oaDecoder), + #if FABLE_COMPILER_PYTHON get.Required.Field "last_updated" Decode.datetimeLocal // Currently not supported in Thoth.Json.Core for python + #else + get.Required.Field "last_updated" Decode.datetimeUtc + #endif ) ) diff --git a/src/ARCtrl/WebRequest/WebRequest.Node.fs b/src/ARCtrl/WebRequest/WebRequest.Node.fs index a5a5eb38..ad59270b 100644 --- a/src/ARCtrl/WebRequest/WebRequest.Node.fs +++ b/src/ARCtrl/WebRequest/WebRequest.Node.fs @@ -2,7 +2,7 @@ open Fable.Core open Fable.SimpleHttp -let x = 1 + #if FABLE_COMPILER_JAVASCRIPT open Fable.Core.JsInterop diff --git a/src/ISA/ISA.Json/ARCtrl.ISA.Json.fsproj b/src/ISA/ISA.Json/ARCtrl.ISA.Json.fsproj index 4c7c991f..d10a9482 100644 --- a/src/ISA/ISA.Json/ARCtrl.ISA.Json.fsproj +++ b/src/ISA/ISA.Json/ARCtrl.ISA.Json.fsproj @@ -65,11 +65,6 @@ - - - - - diff --git a/src/ISA/ISA/Helper.fs b/src/ISA/ISA/Helper.fs index 5389fed9..979edaee 100644 --- a/src/ISA/ISA/Helper.fs +++ b/src/ISA/ISA/Helper.fs @@ -78,11 +78,13 @@ module HashCodes = open Fable.Core + #if FABLE_COMPILER_PYTHON [] let pyHasCustomHash (obj) : bool = nativeOnly [] let pyCustomHash (obj) : int = nativeOnly + #endif let hash obj = #if FABLE_COMPILER_PYTHON diff --git a/tests/ISA/ISA.Json.Tests/ArcTypes.Tests.fs b/tests/ISA/ISA.Json.Tests/ArcTypes.Tests.fs index bd8c6c03..8c9d1dfc 100644 --- a/tests/ISA/ISA.Json.Tests/ArcTypes.Tests.fs +++ b/tests/ISA/ISA.Json.Tests/ArcTypes.Tests.fs @@ -286,10 +286,10 @@ let tests_ArcAssay = testList "ArcAssay" [ let expectedMs = 5000 #endif #if FABLE_COMPILER_PYTHON - let expectedMs = 40000 + let expectedMs = 100000 #endif #if !FABLE_COMPILER - let expectedMs = 2000 + let expectedMs = 2500 #endif // 1200ms in Dotnet on i7-13800H // 3412ms in Javascript on i7-13800H diff --git a/tests/ISA/ISA.Json.Tests/ISA.Json.Tests.fsproj b/tests/ISA/ISA.Json.Tests/ISA.Json.Tests.fsproj index c263dd54..e7b79f4f 100644 --- a/tests/ISA/ISA.Json.Tests/ISA.Json.Tests.fsproj +++ b/tests/ISA/ISA.Json.Tests/ISA.Json.Tests.fsproj @@ -17,7 +17,11 @@ - + + + + + diff --git a/tests/ISA/ISA.Tests/ArcTable.Tests.fs b/tests/ISA/ISA.Tests/ArcTable.Tests.fs index 64b437c9..d7a40565 100644 --- a/tests/ISA/ISA.Tests/ArcTable.Tests.fs +++ b/tests/ISA/ISA.Tests/ArcTable.Tests.fs @@ -2336,7 +2336,7 @@ let private tests_fillMissing = testList "fillMissing" [ ArcTableAux.Unchecked.setCellAt(6,i,(CompositeCell.FreeText $"Sample_{i}")) values let testF = fun () -> ArcTableAux.Unchecked.fillMissingCells headers values #if FABLE_COMPILER_PYTHON - let expectedMs = 8000 + let expectedMs = 10000 #else let expectedMs = 220 #endif diff --git a/tests/JavaScript/ISA.CompositeCell.js b/tests/JavaScript/ISA.CompositeCell.js index 623685e2..8b43bf13 100644 --- a/tests/JavaScript/ISA.CompositeCell.js +++ b/tests/JavaScript/ISA.CompositeCell.js @@ -1,7 +1,7 @@ import { equal, deepEqual, notEqual } from 'assert'; import { CompositeCell } from "./ARCtrl/ISA/ISA/ArcTypes/CompositeCell.js" import { OntologyAnnotation } from './ARCtrl/ISA/ISA/JsonTypes/OntologyAnnotation.js'; -import { assertEqual } from './ARCtrl/fable_modules/fable-library.4.5.0/Util.js'; +import { assertEqual } from './ARCtrl/fable_modules/fable-library-js.4.13.0/Util.js'; diff --git a/tests/JavaScript/ISA.CompositeHeader.js b/tests/JavaScript/ISA.CompositeHeader.js index 06dc49e6..538c5465 100644 --- a/tests/JavaScript/ISA.CompositeHeader.js +++ b/tests/JavaScript/ISA.CompositeHeader.js @@ -1,7 +1,7 @@ import { equal, deepEqual, notEqual } from 'assert'; import { CompositeHeader, IOType } from "./ARCtrl/ISA/ISA/ArcTypes/CompositeHeader.js" import { OntologyAnnotation } from './ARCtrl/ISA/ISA/JsonTypes/OntologyAnnotation.js'; -import { assertEqual } from './ARCtrl/fable_modules/fable-library.4.5.0/Util.js'; +import { assertEqual } from './ARCtrl/fable_modules/fable-library-js.4.13.0/Util.js'; function tests_IOType() { describe('IOType', function () { From 9385d06c4bb728edd6c3d95f528fd0a40a19062d Mon Sep 17 00:00:00 2001 From: HLWeil Date: Fri, 1 Mar 2024 11:45:45 +0100 Subject: [PATCH 14/25] small fixes in python and setup instructions --- README.md | 23 ++++++++-- src/ISA/ISA.Json/ARCtrl.ISA.Json.fsproj | 46 +++++++++---------- .../isa_assay_context.fs | 0 .../isa_comment_context.fs | 0 .../isa_component_context.fs | 0 .../{ro-crate => rocrate}/isa_data_context.fs | 0 .../isa_factor_context.fs | 0 .../isa_factor_value_context.fs | 0 .../isa_investigation_context.fs | 0 .../isa_material_attribute_context.fs | 0 .../isa_material_attribute_value_context.fs | 0 .../isa_material_context.fs | 0 .../isa_ontology_annotation_context.fs | 0 .../isa_ontology_source_reference_context.fs | 0 .../isa_organization_context.fs | 0 .../isa_person_context.fs | 0 .../isa_process_context.fs | 0 .../isa_process_parameter_value_context.fs | 0 .../isa_protocol_context.fs | 0 .../isa_protocol_parameter_context.fs | 0 .../isa_publication_context.fs | 0 .../isa_sample_context.fs | 0 .../isa_source_context.fs | 0 .../isa_study_context.fs | 0 .../{ro-crate => rocrate}/rocrate_context.fs | 0 tests/ISA/ISA.Json.Tests/Json.Tests.fs | 6 +-- 26 files changed, 46 insertions(+), 29 deletions(-) rename src/ISA/ISA.Json/context/{ro-crate => rocrate}/isa_assay_context.fs (100%) rename src/ISA/ISA.Json/context/{ro-crate => rocrate}/isa_comment_context.fs (100%) rename src/ISA/ISA.Json/context/{ro-crate => rocrate}/isa_component_context.fs (100%) rename src/ISA/ISA.Json/context/{ro-crate => rocrate}/isa_data_context.fs (100%) rename src/ISA/ISA.Json/context/{ro-crate => rocrate}/isa_factor_context.fs (100%) rename src/ISA/ISA.Json/context/{ro-crate => rocrate}/isa_factor_value_context.fs (100%) rename src/ISA/ISA.Json/context/{ro-crate => rocrate}/isa_investigation_context.fs (100%) rename src/ISA/ISA.Json/context/{ro-crate => rocrate}/isa_material_attribute_context.fs (100%) rename src/ISA/ISA.Json/context/{ro-crate => rocrate}/isa_material_attribute_value_context.fs (100%) rename src/ISA/ISA.Json/context/{ro-crate => rocrate}/isa_material_context.fs (100%) rename src/ISA/ISA.Json/context/{ro-crate => rocrate}/isa_ontology_annotation_context.fs (100%) rename src/ISA/ISA.Json/context/{ro-crate => rocrate}/isa_ontology_source_reference_context.fs (100%) rename src/ISA/ISA.Json/context/{ro-crate => rocrate}/isa_organization_context.fs (100%) rename src/ISA/ISA.Json/context/{ro-crate => rocrate}/isa_person_context.fs (100%) rename src/ISA/ISA.Json/context/{ro-crate => rocrate}/isa_process_context.fs (100%) rename src/ISA/ISA.Json/context/{ro-crate => rocrate}/isa_process_parameter_value_context.fs (100%) rename src/ISA/ISA.Json/context/{ro-crate => rocrate}/isa_protocol_context.fs (100%) rename src/ISA/ISA.Json/context/{ro-crate => rocrate}/isa_protocol_parameter_context.fs (100%) rename src/ISA/ISA.Json/context/{ro-crate => rocrate}/isa_publication_context.fs (100%) rename src/ISA/ISA.Json/context/{ro-crate => rocrate}/isa_sample_context.fs (100%) rename src/ISA/ISA.Json/context/{ro-crate => rocrate}/isa_source_context.fs (100%) rename src/ISA/ISA.Json/context/{ro-crate => rocrate}/isa_study_context.fs (100%) rename src/ISA/ISA.Json/context/{ro-crate => rocrate}/rocrate_context.fs (100%) diff --git a/README.md b/README.md index c1ca93c7..fa9a85c1 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ and __JavaScript__! ❤️ ``` ```bash - + ``` ### JavaScript @@ -44,10 +44,27 @@ Currently we provide some documentation in form of markdown files in the `/docs` - verify with `npm --version` (Tested with v9.2.0) - [.NET SDK](https://dotnet.microsoft.com/en-us/download) - verify with `dotnet --version` (Tested with 7.0.306) +- [Python](https://www.python.org/downloads/) + - verify with `py --version` (Tested with 3.12.2) ### Local Setup -1. `dotnet tool restore` -3. `npm install` +1. Setup dotnet tools + + `dotnet tool restore` + +2. Install NPM dependencies + + `npm install` + +3. Setup python environment + + `py -m venv .venv` + +4. Install [Poetry](https://python-poetry.org/) and dependencies + + 1. `.\.venv\Scripts\python.exe -m pip install -U pip setuptools` + 2. `.\.venv\Scripts\python.exe -m pip install poetry` + 3. `.\.venv\Scripts\python.exe -m poetry install --no-root` Verify correct setup with `./build.cmd runtests` ✨ diff --git a/src/ISA/ISA.Json/ARCtrl.ISA.Json.fsproj b/src/ISA/ISA.Json/ARCtrl.ISA.Json.fsproj index d10a9482..69ab3c6f 100644 --- a/src/ISA/ISA.Json/ARCtrl.ISA.Json.fsproj +++ b/src/ISA/ISA.Json/ARCtrl.ISA.Json.fsproj @@ -5,29 +5,29 @@ true - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/ISA/ISA.Json/context/ro-crate/isa_assay_context.fs b/src/ISA/ISA.Json/context/rocrate/isa_assay_context.fs similarity index 100% rename from src/ISA/ISA.Json/context/ro-crate/isa_assay_context.fs rename to src/ISA/ISA.Json/context/rocrate/isa_assay_context.fs diff --git a/src/ISA/ISA.Json/context/ro-crate/isa_comment_context.fs b/src/ISA/ISA.Json/context/rocrate/isa_comment_context.fs similarity index 100% rename from src/ISA/ISA.Json/context/ro-crate/isa_comment_context.fs rename to src/ISA/ISA.Json/context/rocrate/isa_comment_context.fs diff --git a/src/ISA/ISA.Json/context/ro-crate/isa_component_context.fs b/src/ISA/ISA.Json/context/rocrate/isa_component_context.fs similarity index 100% rename from src/ISA/ISA.Json/context/ro-crate/isa_component_context.fs rename to src/ISA/ISA.Json/context/rocrate/isa_component_context.fs diff --git a/src/ISA/ISA.Json/context/ro-crate/isa_data_context.fs b/src/ISA/ISA.Json/context/rocrate/isa_data_context.fs similarity index 100% rename from src/ISA/ISA.Json/context/ro-crate/isa_data_context.fs rename to src/ISA/ISA.Json/context/rocrate/isa_data_context.fs diff --git a/src/ISA/ISA.Json/context/ro-crate/isa_factor_context.fs b/src/ISA/ISA.Json/context/rocrate/isa_factor_context.fs similarity index 100% rename from src/ISA/ISA.Json/context/ro-crate/isa_factor_context.fs rename to src/ISA/ISA.Json/context/rocrate/isa_factor_context.fs diff --git a/src/ISA/ISA.Json/context/ro-crate/isa_factor_value_context.fs b/src/ISA/ISA.Json/context/rocrate/isa_factor_value_context.fs similarity index 100% rename from src/ISA/ISA.Json/context/ro-crate/isa_factor_value_context.fs rename to src/ISA/ISA.Json/context/rocrate/isa_factor_value_context.fs diff --git a/src/ISA/ISA.Json/context/ro-crate/isa_investigation_context.fs b/src/ISA/ISA.Json/context/rocrate/isa_investigation_context.fs similarity index 100% rename from src/ISA/ISA.Json/context/ro-crate/isa_investigation_context.fs rename to src/ISA/ISA.Json/context/rocrate/isa_investigation_context.fs diff --git a/src/ISA/ISA.Json/context/ro-crate/isa_material_attribute_context.fs b/src/ISA/ISA.Json/context/rocrate/isa_material_attribute_context.fs similarity index 100% rename from src/ISA/ISA.Json/context/ro-crate/isa_material_attribute_context.fs rename to src/ISA/ISA.Json/context/rocrate/isa_material_attribute_context.fs diff --git a/src/ISA/ISA.Json/context/ro-crate/isa_material_attribute_value_context.fs b/src/ISA/ISA.Json/context/rocrate/isa_material_attribute_value_context.fs similarity index 100% rename from src/ISA/ISA.Json/context/ro-crate/isa_material_attribute_value_context.fs rename to src/ISA/ISA.Json/context/rocrate/isa_material_attribute_value_context.fs diff --git a/src/ISA/ISA.Json/context/ro-crate/isa_material_context.fs b/src/ISA/ISA.Json/context/rocrate/isa_material_context.fs similarity index 100% rename from src/ISA/ISA.Json/context/ro-crate/isa_material_context.fs rename to src/ISA/ISA.Json/context/rocrate/isa_material_context.fs diff --git a/src/ISA/ISA.Json/context/ro-crate/isa_ontology_annotation_context.fs b/src/ISA/ISA.Json/context/rocrate/isa_ontology_annotation_context.fs similarity index 100% rename from src/ISA/ISA.Json/context/ro-crate/isa_ontology_annotation_context.fs rename to src/ISA/ISA.Json/context/rocrate/isa_ontology_annotation_context.fs diff --git a/src/ISA/ISA.Json/context/ro-crate/isa_ontology_source_reference_context.fs b/src/ISA/ISA.Json/context/rocrate/isa_ontology_source_reference_context.fs similarity index 100% rename from src/ISA/ISA.Json/context/ro-crate/isa_ontology_source_reference_context.fs rename to src/ISA/ISA.Json/context/rocrate/isa_ontology_source_reference_context.fs diff --git a/src/ISA/ISA.Json/context/ro-crate/isa_organization_context.fs b/src/ISA/ISA.Json/context/rocrate/isa_organization_context.fs similarity index 100% rename from src/ISA/ISA.Json/context/ro-crate/isa_organization_context.fs rename to src/ISA/ISA.Json/context/rocrate/isa_organization_context.fs diff --git a/src/ISA/ISA.Json/context/ro-crate/isa_person_context.fs b/src/ISA/ISA.Json/context/rocrate/isa_person_context.fs similarity index 100% rename from src/ISA/ISA.Json/context/ro-crate/isa_person_context.fs rename to src/ISA/ISA.Json/context/rocrate/isa_person_context.fs diff --git a/src/ISA/ISA.Json/context/ro-crate/isa_process_context.fs b/src/ISA/ISA.Json/context/rocrate/isa_process_context.fs similarity index 100% rename from src/ISA/ISA.Json/context/ro-crate/isa_process_context.fs rename to src/ISA/ISA.Json/context/rocrate/isa_process_context.fs diff --git a/src/ISA/ISA.Json/context/ro-crate/isa_process_parameter_value_context.fs b/src/ISA/ISA.Json/context/rocrate/isa_process_parameter_value_context.fs similarity index 100% rename from src/ISA/ISA.Json/context/ro-crate/isa_process_parameter_value_context.fs rename to src/ISA/ISA.Json/context/rocrate/isa_process_parameter_value_context.fs diff --git a/src/ISA/ISA.Json/context/ro-crate/isa_protocol_context.fs b/src/ISA/ISA.Json/context/rocrate/isa_protocol_context.fs similarity index 100% rename from src/ISA/ISA.Json/context/ro-crate/isa_protocol_context.fs rename to src/ISA/ISA.Json/context/rocrate/isa_protocol_context.fs diff --git a/src/ISA/ISA.Json/context/ro-crate/isa_protocol_parameter_context.fs b/src/ISA/ISA.Json/context/rocrate/isa_protocol_parameter_context.fs similarity index 100% rename from src/ISA/ISA.Json/context/ro-crate/isa_protocol_parameter_context.fs rename to src/ISA/ISA.Json/context/rocrate/isa_protocol_parameter_context.fs diff --git a/src/ISA/ISA.Json/context/ro-crate/isa_publication_context.fs b/src/ISA/ISA.Json/context/rocrate/isa_publication_context.fs similarity index 100% rename from src/ISA/ISA.Json/context/ro-crate/isa_publication_context.fs rename to src/ISA/ISA.Json/context/rocrate/isa_publication_context.fs diff --git a/src/ISA/ISA.Json/context/ro-crate/isa_sample_context.fs b/src/ISA/ISA.Json/context/rocrate/isa_sample_context.fs similarity index 100% rename from src/ISA/ISA.Json/context/ro-crate/isa_sample_context.fs rename to src/ISA/ISA.Json/context/rocrate/isa_sample_context.fs diff --git a/src/ISA/ISA.Json/context/ro-crate/isa_source_context.fs b/src/ISA/ISA.Json/context/rocrate/isa_source_context.fs similarity index 100% rename from src/ISA/ISA.Json/context/ro-crate/isa_source_context.fs rename to src/ISA/ISA.Json/context/rocrate/isa_source_context.fs diff --git a/src/ISA/ISA.Json/context/ro-crate/isa_study_context.fs b/src/ISA/ISA.Json/context/rocrate/isa_study_context.fs similarity index 100% rename from src/ISA/ISA.Json/context/ro-crate/isa_study_context.fs rename to src/ISA/ISA.Json/context/rocrate/isa_study_context.fs diff --git a/src/ISA/ISA.Json/context/ro-crate/rocrate_context.fs b/src/ISA/ISA.Json/context/rocrate/rocrate_context.fs similarity index 100% rename from src/ISA/ISA.Json/context/ro-crate/rocrate_context.fs rename to src/ISA/ISA.Json/context/rocrate/rocrate_context.fs diff --git a/tests/ISA/ISA.Json.Tests/Json.Tests.fs b/tests/ISA/ISA.Json.Tests/Json.Tests.fs index 713fa809..0a2a6aa1 100644 --- a/tests/ISA/ISA.Json.Tests/Json.Tests.fs +++ b/tests/ISA/ISA.Json.Tests/Json.Tests.fs @@ -1292,7 +1292,7 @@ let testInvestigationFile = Publication.make (Some "12345678") (Some "11.1111/abcdef123456789") - (Some "Lukas Weil, Other Gzúy") + (Some "Lukas Weil, Other Gzuy") // (Some "Lukas Weil, Other Gzúy") (Some "Fair is great") (Some publicationStatus) (Some [|comment|]) @@ -1708,7 +1708,7 @@ let testInvestigationFileLD = Publication.make (Some "12345678") (Some "11.1111/abcdef123456789") - (Some "Lukas Weil, Other Gzúy") + (Some "Lukas Weil, Other Gzuy") // (Some "Lukas Weil, Other Gzúy") (Some "Fair is great") (Some publicationStatus) (Some [|comment|]) @@ -2051,7 +2051,7 @@ let testInvestigationFileLD = Publication.make (Some "12345678") (Some "11.1111/abcdef123456789") - (Some "Lukas Weil, Other Gzúy") + (Some "Lukas Weil, Other Gzuy") // (Some "Lukas Weil, Other Gzúy") (Some "Fair is great") (Some publicationStatus) (Some [|comment|]) From f75ce4d3d9abc84600a769d8ead47a6bd2befe8b Mon Sep 17 00:00:00 2001 From: HLWeil Date: Fri, 1 Mar 2024 11:48:14 +0100 Subject: [PATCH 15/25] small python tests hotfix --- tests/TestingUtils/TestObjects.Json/Investigation.fs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/TestingUtils/TestObjects.Json/Investigation.fs b/tests/TestingUtils/TestObjects.Json/Investigation.fs index e0114098..4d93f8a0 100644 --- a/tests/TestingUtils/TestObjects.Json/Investigation.fs +++ b/tests/TestingUtils/TestObjects.Json/Investigation.fs @@ -58,7 +58,7 @@ let investigationROC = }, "pubMedID": "12345678", "doi": "11.1111/abcdef123456789", - "authorList": "Lukas Weil, Other Gzúy", + "authorList": "Lukas Weil, Other Gzuy", "title": "Fair is great", "status": { "@id": "OntologyTerm/Published", @@ -251,7 +251,7 @@ let investigationROC = }, "pubMedID": "12345678", "doi": "11.1111/abcdef123456789", - "authorList": "Lukas Weil, Other Gzúy", + "authorList": "Lukas Weil, Other Gzuy", "title": "Fair is great", "status": { "@id": "OntologyTerm/Published", @@ -3889,7 +3889,7 @@ let investigationLD = }, "pubMedID": "12345678", "doi": "11.1111/abcdef123456789", - "authorList": "Lukas Weil, Other Gzúy", + "authorList": "Lukas Weil, Other Gzuy", "title": "Fair is great", "status": { "@id": "OntologyTerm/Published", @@ -4072,7 +4072,7 @@ let investigationLD = }, "pubMedID": "12345678", "doi": "11.1111/abcdef123456789", - "authorList": "Lukas Weil, Other Gzúy", + "authorList": "Lukas Weil, Other Gzuy", "title": "Fair is great", "status": { "@id": "OntologyTerm/Published", From 4262833f532c3ed8deedc1ca30b6ad30c04c17b3 Mon Sep 17 00:00:00 2001 From: HLWeil Date: Fri, 1 Mar 2024 14:06:40 +0100 Subject: [PATCH 16/25] hotfix js webrequest --- src/ARCtrl/WebRequest/WebRequest.Node.fs | 2 -- src/ARCtrl/WebRequest/WebRequest.fs | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/ARCtrl/WebRequest/WebRequest.Node.fs b/src/ARCtrl/WebRequest/WebRequest.Node.fs index ad59270b..1a239003 100644 --- a/src/ARCtrl/WebRequest/WebRequest.Node.fs +++ b/src/ARCtrl/WebRequest/WebRequest.Node.fs @@ -14,8 +14,6 @@ open Fetch //// From here: https://github.com/fable-compiler/fable3-samples/blob/25ea2404b28c897988b144f0141bc116da292679/nodejs/src/App.fs#L7 -let importSideEffects() = importSideEffects "isomorphic-fetch" - let isNode() = emitJsExpr () diff --git a/src/ARCtrl/WebRequest/WebRequest.fs b/src/ARCtrl/WebRequest/WebRequest.fs index 0c1a1554..26577303 100644 --- a/src/ARCtrl/WebRequest/WebRequest.fs +++ b/src/ARCtrl/WebRequest/WebRequest.fs @@ -17,7 +17,7 @@ let downloadFile url = if ARCtrl.WebRequestHelpers.NodeJs.isNode() then // From here: https://github.com/fable-compiler/fable3-samples/blob/25ea2404b28c897988b144f0141bc116da292679/nodejs/src/App.fs#L7 - ARCtrl.WebRequestHelpers.NodeJs.importSideEffects() + Fable.Core.JsInterop.importSideEffects "isomorphic-fetch" ARCtrl.WebRequestHelpers.NodeJs.downloadFile url else browserAndDotnet() From 89aa3c2a93cb718ca77433e35599d2d22deb2886 Mon Sep 17 00:00:00 2001 From: Florian Wetzels Date: Tue, 5 Mar 2024 13:38:13 +0100 Subject: [PATCH 17/25] fixed python tests to work on all platforms --- build/BasicTasks.fs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/build/BasicTasks.fs b/build/BasicTasks.fs index 0255088f..dce85b35 100644 --- a/build/BasicTasks.fs +++ b/build/BasicTasks.fs @@ -105,7 +105,13 @@ module Helper = createProcess npmPath - let python = createProcess @".\.venv\Scripts\python.exe" + let python = + if System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.Windows) then + Fake.Core.Trace.log "Detected Windows System." + createProcess @".\.venv\Scripts\python.exe" + else + Fake.Core.Trace.log "Detected Unix System." + createProcess @"./.venv/bin/python" let run proc arg dir = proc arg dir From df663cbfe8cdfaa7ac48690f89539b17715ae0c0 Mon Sep 17 00:00:00 2001 From: HLWeil Date: Tue, 5 Mar 2024 14:25:09 +0100 Subject: [PATCH 18/25] include python setup in CI --- .github/workflows/build-test.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 620f7c3f..b56cb178 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -17,12 +17,16 @@ jobs: steps: - uses: actions/checkout@v3 + + # SETUP .NET - name: Setup .NET uses: actions/setup-dotnet@v3 with: dotnet-version: 6.x.x - name: Restore fable run: dotnet tool restore + + # SETUP NODE - name: Setup Node.js environment uses: actions/setup-node@v3 with: @@ -33,6 +37,28 @@ jobs: - name: install node modules ISA working-directory: ./src/ISA run: npm install + + # SETUP PYTHON + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Setup Virtual Environment + run: python -m venv venv + - name: Setup Poetry Windows + if: matrix.os == 'windows-latest' + run: | + .\.venv\Scripts\python.exe -m pip install -U pip setuptools + .\.venv\Scripts\python.exe -m pip install poetry + .\.venv\Scripts\python.exe -m poetry install + - name: Setup Poetry Unix + if: matrix.os == 'ubuntu-latest' + run: | + ./.venv/bin/python -m pip install -U pip setuptools + ./.venv/bin/python -m pip install poetry + ./.venv/bin/python -m poetry install + + # BUILD - name: make script executable if: matrix.os == 'ubuntu-latest' run: chmod u+x build.sh From b0186f4c3e28fbf9a6678dde5795e6e0723934c0 Mon Sep 17 00:00:00 2001 From: HLWeil Date: Tue, 5 Mar 2024 14:34:48 +0100 Subject: [PATCH 19/25] small fix to CI --- .github/workflows/build-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index b56cb178..7fbeced1 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -44,7 +44,7 @@ jobs: with: python-version: '3.11' - name: Setup Virtual Environment - run: python -m venv venv + run: python -m venv .venv - name: Setup Poetry Windows if: matrix.os == 'windows-latest' run: | From 3e403cdd687be78f1cbadfafb30b148247800bdd Mon Sep 17 00:00:00 2001 From: HLWeil Date: Tue, 5 Mar 2024 14:39:59 +0100 Subject: [PATCH 20/25] small fix to CI --- .github/workflows/build-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 7fbeced1..ef538915 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -50,13 +50,13 @@ jobs: run: | .\.venv\Scripts\python.exe -m pip install -U pip setuptools .\.venv\Scripts\python.exe -m pip install poetry - .\.venv\Scripts\python.exe -m poetry install + .\.venv\Scripts\python.exe -m poetry install --no-root - name: Setup Poetry Unix if: matrix.os == 'ubuntu-latest' run: | ./.venv/bin/python -m pip install -U pip setuptools ./.venv/bin/python -m pip install poetry - ./.venv/bin/python -m poetry install + ./.venv/bin/python -m poetry install --no-root # BUILD - name: make script executable From 890048e84e06a88b4b957fc46e30113c40c014d0 Mon Sep 17 00:00:00 2001 From: HLWeil Date: Wed, 6 Mar 2024 10:03:18 +0100 Subject: [PATCH 21/25] set ci fail-fast to false and fix py encoding in windows --- .github/workflows/build-test.yml | 1 + build.cmd | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index ef538915..2f84b2a2 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -10,6 +10,7 @@ jobs: test: strategy: + fail-fast: false matrix: os: [ubuntu-latest, windows-latest] diff --git a/build.cmd b/build.cmd index 3fba449c..fe247e50 100644 --- a/build.cmd +++ b/build.cmd @@ -1,5 +1,6 @@ @echo off +set PYTHONIOENCODING=utf-8 dotnet tool restore cls dotnet run --project ./build/build.fsproj %* \ No newline at end of file From 8becc672c432eb4399fc3a0e250a87e63044de2b Mon Sep 17 00:00:00 2001 From: HLWeil Date: Wed, 6 Mar 2024 18:43:57 +0100 Subject: [PATCH 22/25] update build project for releasing python package --- build/BasicTasks.fs | 6 +- build/Build.fs | 2 +- build/Build.fsproj | 1 + build/GenerateIndexPy.fs | 49 ++++++++++++ build/PackageTasks.fs | 111 +++++++++++++++------------ build/ProjectInfo.fs | 3 +- build/ReleaseNotesTasks.fs | 13 +++- build/ReleaseTasks.fs | 16 +++- src/ARCtrl/Templates/Template.Web.fs | 2 +- src/ISA/ISA.Json/Data.fs | 1 - src/ISA/ISA.Json/Factor.fs | 1 - src/ISA/ISA.Json/Material.fs | 1 - src/ISA/ISA.Json/Process.fs | 2 - 13 files changed, 142 insertions(+), 66 deletions(-) create mode 100644 build/GenerateIndexPy.fs diff --git a/build/BasicTasks.fs b/build/BasicTasks.fs index dce85b35..d24953d6 100644 --- a/build/BasicTasks.fs +++ b/build/BasicTasks.fs @@ -108,10 +108,10 @@ module Helper = let python = if System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.Windows) then Fake.Core.Trace.log "Detected Windows System." - createProcess @".\.venv\Scripts\python.exe" + createProcess (__SOURCE_DIRECTORY__.Replace(@"\build",@"\.venv\Scripts\python.exe")) else Fake.Core.Trace.log "Detected Unix System." - createProcess @"./.venv/bin/python" + createProcess (__SOURCE_DIRECTORY__.Replace(@"/build",@"/.venv/bin/python")) let run proc arg dir = proc arg dir @@ -137,7 +137,7 @@ let clean = BuildTask.create "Clean" [] { ++ "tests/**/bin" ++ "tests/**/obj" ++ "dist" - ++ ProjectInfo.pkgDir + ++ ProjectInfo.netPkgDir |> Shell.cleanDirs } diff --git a/build/Build.fs b/build/Build.fs index a927c157..bf3cc6d5 100644 --- a/build/Build.fs +++ b/build/Build.fs @@ -21,7 +21,7 @@ open ReleaseTasks let _release = BuildTask.createEmpty "Release" - [clean; build; runTests; pack; createTag; publishNuget; publishNPM] + [clean; build; runTests; pack; createTag; publishNuget; publishNPM; publishPyPi] /// Full release of nuget package for the prerelease version. let _preRelease = diff --git a/build/Build.fsproj b/build/Build.fsproj index 0a1b0056..ecbd44dd 100644 --- a/build/Build.fsproj +++ b/build/Build.fsproj @@ -12,6 +12,7 @@ + diff --git a/build/GenerateIndexPy.fs b/build/GenerateIndexPy.fs new file mode 100644 index 00000000..f8ceb0d1 --- /dev/null +++ b/build/GenerateIndexPy.fs @@ -0,0 +1,49 @@ +module GenerateIndexPy + +open System +open System.IO +open System.Text.RegularExpressions + + +open System.Text + +let createImportStatement path (classes : string []) = + let classes = classes |> Array.reduce (fun acc x -> acc + ", " + x) + sprintf "from %s import %s" path classes + +let writePyIndexfile (path: string) (content: string) = + let filePath = Path.Combine(path, "arctrl.py") + File.WriteAllText(filePath, content) + +let generateIndexFileContent (classes : (string*string) []) = + classes + |> Array.groupBy fst + |> Array.map (fun (p,a) -> createImportStatement p (a |> Array.map snd)) + +let classes = + [| + "__future__", "annotations" + "collections.abc", "Callable" + "typing", "Any" + ".ISA.ISA.JsonTypes.comment", "Comment" + ".ISA.ISA.JsonTypes.ontology_annotation","OntologyAnnotation"; + ".ISA.ISA.JsonTypes.person", "Person"; + ".ISA.ISA.JsonTypes.publication", "Publication"; + ".ISA.ISA.ArcTypes.composite_header", "IOType" + ".ISA.ISA.ArcTypes.composite_header", "CompositeHeader"; + ".ISA.ISA.ArcTypes.composite_cell", "CompositeCell" + ".ISA.ISA.ArcTypes.composite_column", "CompositeColumn" + ".ISA.ISA.ArcTypes.arc_table", "ArcTable" + ".ISA.ISA.ArcTypes.arc_types", "ArcAssay"; + ".ISA.ISA.ArcTypes.arc_types", "ArcStudy"; + ".ISA.ISA.ArcTypes.arc_types", "ArcInvestigation"; + ".Templates.template", "Template" + ".Templates.templates", "Templates" + ".Templates.template", "Organisation" + ".arc","ARC" + |] + +let ARCtrl_generate (rootPath: string) = + generateIndexFileContent classes + |> Array.reduce (fun a b -> a + "\n" + b) + |> writePyIndexfile rootPath \ No newline at end of file diff --git a/build/PackageTasks.fs b/build/PackageTasks.fs index 654c9a27..9fc2b35b 100644 --- a/build/PackageTasks.fs +++ b/build/PackageTasks.fs @@ -15,50 +15,33 @@ let private replaceCommitLink input = let commitLinkPattern = @"\[\[#[a-z0-9]*\]\(.*\)\] " Regex.Replace(input,commitLinkPattern,"") -let packDotNet = BuildTask.create "PackDotNet" [clean; build; runTests] { - if promptYesNo (sprintf "[NUGET] creating stable package with version %s OK?" ProjectInfo.stableVersionTag ) - then - !! "src/**/*.*proj" - -- "src/bin/*" - |> Seq.iter (Fake.DotNet.DotNet.pack (fun p -> - let msBuildParams = - {p.MSBuildParams with - Properties = ([ - "Version", ProjectInfo.stableVersionTag - "PackageReleaseNotes", (ProjectInfo.release.Notes |> List.map replaceCommitLink |> String.concat "\r\n" ) - ] @ p.MSBuildParams.Properties) - } - { - p with - MSBuildParams = msBuildParams - OutputPath = Some ProjectInfo.pkgDir +module BundleDotNet = + let bundle (versionTag : string) (versionSuffix : string option) = + System.IO.Directory.CreateDirectory(ProjectInfo.netPkgDir) |> ignore + !! "src/**/*.*proj" + -- "src/bin/*" + |> Seq.iter (Fake.DotNet.DotNet.pack (fun p -> + let msBuildParams = + {p.MSBuildParams with + Properties = ([ + "Version",versionTag + "PackageReleaseNotes", (ProjectInfo.release.Notes |> List.map replaceCommitLink |> String.toLines ) + ] @ p.MSBuildParams.Properties) } - )) - else failwith "aborted" + { + p with + VersionSuffix = versionSuffix + MSBuildParams = msBuildParams + OutputPath = Some ProjectInfo.netPkgDir + } + )) + +let packDotNet = BuildTask.create "PackDotNet" [clean; build; runTests] { + BundleDotNet.bundle ProjectInfo.stableVersionTag None } -let packDotNetPrerelease = BuildTask.create "PackDotNetPrerelease" [setPrereleaseTag; clean; build; (*runTests*)] { - if promptYesNo (sprintf "[NUGET] package tag will be %s OK?" ProjectInfo.prereleaseTag ) - then - !! "src/**/*.*proj" - -- "src/bin/*" - |> Seq.iter (Fake.DotNet.DotNet.pack (fun p -> - let msBuildParams = - {p.MSBuildParams with - Properties = ([ - "Version", ProjectInfo.prereleaseTag - "PackageReleaseNotes", (ProjectInfo.release.Notes |> List.map replaceCommitLink |> String.toLines ) - ] @ p.MSBuildParams.Properties) - } - { - p with - VersionSuffix = Some ProjectInfo.prereleaseSuffix - OutputPath = Some ProjectInfo.pkgDir - MSBuildParams = msBuildParams - } - )) - else - failwith "aborted" +let packDotNetPrerelease = BuildTask.create "PackDotNetPrerelease" [setPrereleaseTag; clean; build; runTests] { + BundleDotNet.bundle ProjectInfo.prereleaseTag (Some ProjectInfo.prereleaseTag) } module BundleJs = @@ -82,18 +65,46 @@ module BundleJs = }) let packJS = BuildTask.create "PackJS" [clean; build; runTests] { - if promptYesNo (sprintf "[NPM] creating stable package with version %s OK?" ProjectInfo.stableVersionTag ) - then - BundleJs.bundle ProjectInfo.stableVersionTag - else failwith "aborted" + BundleJs.bundle ProjectInfo.stableVersionTag } let packJSPrerelease = BuildTask.create "PackJSPrerelease" [setPrereleaseTag; clean; build; runTests] { - if promptYesNo (sprintf "[NPM] package tag will be %s OK?" ProjectInfo.prereleaseTag ) then - BundleJs.bundle ProjectInfo.prereleaseTag - else failwith "aborted" - } + BundleJs.bundle ProjectInfo.prereleaseTag +} + +module BundlePy = + let bundle (versionTag: string) = + + run dotnet $"fable src/ARCtrl -o {ProjectInfo.pyPkgDir}/arctrl --lang python" "" + run python "-m poetry install --no-root" ProjectInfo.pyPkgDir + GenerateIndexPy.ARCtrl_generate (ProjectInfo.pyPkgDir + "/arctrl") + + Fake.IO.File.readAsString "pyproject.toml" + |> fun t -> + let t = t.Replace(ProjectInfo.stableVersionTag, versionTag) + Fake.IO.File.writeString false $"{ProjectInfo.pyPkgDir}/pyproject.toml" t + + Fake.IO.File.readAsString "README.md" + |> Fake.IO.File.writeString false $"{ProjectInfo.pyPkgDir}/README.md" + + //"" // "fable-library.**/**" + //|> Fake.IO.File.writeString false $"{ProjectInfo.npmPkgDir}/fable_modules/.npmignore" + + run python "-m poetry build" ProjectInfo.pyPkgDir //Remove "-o ." because not compatible with publish + + +let packPy = BuildTask.create "PackPy" [clean; build; (*runTests*)] { + BundlePy.bundle ProjectInfo.stableVersionTag + +} + +//let packPyPrerelease = BuildTask.create "PackJSPrerelease" [setPrereleaseTag; clean; build; runTests] { +// if promptYesNo (sprintf "[NPM] package tag will be %s OK?" ProjectInfo.prereleaseTag ) then +// BundleJs.bundle ProjectInfo.prereleaseTag +// else failwith "aborted" +// } + -let pack = BuildTask.createEmpty "Pack" [packDotNet; packJS] +let pack = BuildTask.createEmpty "Pack" [packDotNet; packJS; packPy] let packPrerelease = BuildTask.createEmpty "PackPrerelease" [packDotNetPrerelease;packJSPrerelease] \ No newline at end of file diff --git a/build/ProjectInfo.fs b/build/ProjectInfo.fs index 4603b20e..9ee30ed3 100644 --- a/build/ProjectInfo.fs +++ b/build/ProjectInfo.fs @@ -30,8 +30,9 @@ let gitHome = $"https://github.com/{gitOwner}" let projectRepo = $"https://github.com/{gitOwner}/{project}" -let pkgDir = "dist/pkg" +let netPkgDir = "dist/pkg" let npmPkgDir = "./dist/js" +let pyPkgDir = "./dist/py" // Create RELEASE_NOTES.md if not existing. Or "release" would throw an error. Fake.Extensions.Release.ReleaseNotes.ensure() diff --git a/build/ReleaseNotesTasks.fs b/build/ReleaseNotesTasks.fs index f9471c49..6b60a779 100644 --- a/build/ReleaseNotesTasks.fs +++ b/build/ReleaseNotesTasks.fs @@ -13,16 +13,23 @@ let createAssemblyVersion = BuildTask.create "createvfs" [] { let updateReleaseNotes = BuildTask.createFn "ReleaseNotes" [] (fun config -> ReleaseNotes.update(ProjectInfo.gitOwner, ProjectInfo.project, config) - let semVer = + let semVerLong,semVerShort = Fake.Core.ReleaseNotes.load "RELEASE_NOTES.md" - |> fun x -> x.SemVer.AsString + |> fun x -> + x.SemVer.AsString, + $"{x.SemVer.Major}.{x.SemVer.Minor}.{x.SemVer.Patch}" Trace.trace "Start updating package.json version" // Update Version in src/Nfdi4Plants.Fornax.Template/package.json let p = "build/release_package.json" let t = System.IO.File.ReadAllText p - let tNew = System.Text.RegularExpressions.Regex.Replace(t, """\"version\": \".*\",""", sprintf "\"version\": \"%s\"," semVer ) + let tNew = System.Text.RegularExpressions.Regex.Replace(t, """\"version\": \".*\",""", sprintf "\"version\": \"%s\"," semVerLong ) System.IO.File.WriteAllText(p, tNew) Trace.trace "Finish updating package.json version" + Trace.trace "Start updating pyproject.toml version" + let p = "pyproject.toml" + let t = System.IO.File.ReadAllText p + let tNew = System.Text.RegularExpressions.Regex.Replace(t, "version = \".*\"", sprintf "version = \"%s\"" semVerShort ) + System.IO.File.WriteAllText(p, tNew) ) diff --git a/build/ReleaseTasks.fs b/build/ReleaseTasks.fs index bef3081b..9086d5cd 100644 --- a/build/ReleaseTasks.fs +++ b/build/ReleaseTasks.fs @@ -31,7 +31,7 @@ let createPrereleaseTag = BuildTask.create "CreatePrereleaseTag" [setPrereleaseT } let publishNuget = BuildTask.create "PublishNuget" [clean; build; runTests; packDotNet] { - let targets = (!! (sprintf "%s/*.*pkg" pkgDir )) + let targets = (!! (sprintf "%s/*.*pkg" netPkgDir )) for target in targets do printfn "%A" target let msg = sprintf "[NUGET] release package with version %s?" stableVersionTag if promptYesNo msg then @@ -44,7 +44,7 @@ let publishNuget = BuildTask.create "PublishNuget" [clean; build; runTests; pack } let publishNugetPrerelease = BuildTask.create "PublishNugetPrerelease" [clean; build; runTests; packDotNetPrerelease] { - let targets = (!! (sprintf "%s/*.*pkg" pkgDir )) + let targets = (!! (sprintf "%s/*.*pkg" netPkgDir )) for target in targets do printfn "%A" target let msg = sprintf "[NUGET] release package with version %s?" prereleaseTag if promptYesNo msg then @@ -87,3 +87,15 @@ let publishNPMPrerelease = BuildTask.create "PublishNPMPrerelease" [clean; build }) else failwith "aborted" } + +let publishPyPi = BuildTask.create "PublishPyPi" [packPy] { + let msg = sprintf "[PyPi] release package with version %s?" stableVersionTag + if promptYesNo msg then + let apikey = Environment.environVarOrNone "PYPI_KEY" + match apikey with + | Some key -> + run python $"-m poetry config pypi-token.pypi {key}" ProjectInfo.pyPkgDir + | None -> () + run python "-m poetry publish" ProjectInfo.pyPkgDir + else failwith "aborted" +} \ No newline at end of file diff --git a/src/ARCtrl/Templates/Template.Web.fs b/src/ARCtrl/Templates/Template.Web.fs index 3a53a3ff..dbba4e5b 100644 --- a/src/ARCtrl/Templates/Template.Web.fs +++ b/src/ARCtrl/Templates/Template.Web.fs @@ -27,4 +27,4 @@ type JsWeb = return templates } |> Async.StartAsPromise -#endif \ No newline at end of file +#endif diff --git a/src/ISA/ISA.Json/Data.fs b/src/ISA/ISA.Json/Data.fs index 97bff049..61cfd18b 100644 --- a/src/ISA/ISA.Json/Data.fs +++ b/src/ISA/ISA.Json/Data.fs @@ -15,7 +15,6 @@ module DataFile = Encode.string "Derived Data File" | DataFile.ImageFile -> Encode.string "Image File" - | _ -> Encode.nil let decoder (options : ConverterOptions) : Decoder = { new Decoder with diff --git a/src/ISA/ISA.Json/Factor.fs b/src/ISA/ISA.Json/Factor.fs index f8675332..97c183f0 100644 --- a/src/ISA/ISA.Json/Factor.fs +++ b/src/ISA/ISA.Json/Factor.fs @@ -18,7 +18,6 @@ module Value = Encode.string s | Value.Ontology s -> OntologyAnnotation.encoder options s - | _ -> Encode.nil let decoder (options : ConverterOptions) : Decoder = { diff --git a/src/ISA/ISA.Json/Material.fs b/src/ISA/ISA.Json/Material.fs index d3837ee0..ee6f8935 100644 --- a/src/ISA/ISA.Json/Material.fs +++ b/src/ISA/ISA.Json/Material.fs @@ -14,7 +14,6 @@ module MaterialType = Encode.string "Extract Name" | MaterialType.LabeledExtractName -> Encode.string "Labeled Extract Name" - | _ -> Encode.nil let decoder (options : ConverterOptions) : Decoder = { new Decoder with diff --git a/src/ISA/ISA.Json/Process.fs b/src/ISA/ISA.Json/Process.fs index ba9cbbfe..dea0892e 100644 --- a/src/ISA/ISA.Json/Process.fs +++ b/src/ISA/ISA.Json/Process.fs @@ -77,7 +77,6 @@ module ProcessInput = Data.encoder options d | ProcessInput.Material m -> Material.encoder options m - | _ -> Encode.nil let decoder (options : ConverterOptions) : Decoder = { new Decoder with @@ -130,7 +129,6 @@ module ProcessOutput = Data.encoder options d | ProcessOutput.Material m -> Material.encoder options m - | _ -> Encode.nil let decoder (options : ConverterOptions) : Decoder = { new Decoder with From 19d850ed5e1474a5e82cd98537db3f398fbacc18 Mon Sep 17 00:00:00 2001 From: HLWeil Date: Thu, 7 Mar 2024 15:13:28 +0100 Subject: [PATCH 23/25] several small cleanups according to PR #317 comments --- README.md | 2 +- build/ProjectInfo.fs | 3 +- pytests/arc_table_tests.ipynb | 238 -------------------------- pytests/composite_header_tests.ipynb | 238 -------------------------- src/ARCtrl/ARCtrl.fsproj | 3 + src/ISA/ISA.Json/ArcTypes/ArcTable.fs | 2 +- src/ISA/ISA/ArcTypes/ArcTable.fs | 6 +- src/ISA/ISA/Fable.fs | 33 +--- testpy.cmd | 2 - 9 files changed, 11 insertions(+), 516 deletions(-) delete mode 100644 pytests/arc_table_tests.ipynb delete mode 100644 pytests/composite_header_tests.ipynb delete mode 100644 testpy.cmd diff --git a/README.md b/README.md index fa9a85c1..37adbe4d 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ Currently we provide some documentation in form of markdown files in the `/docs` - [.NET SDK](https://dotnet.microsoft.com/en-us/download) - verify with `dotnet --version` (Tested with 7.0.306) - [Python](https://www.python.org/downloads/) - - verify with `py --version` (Tested with 3.12.2) + - verify with `py --version` (Tested with 3.12.2, known to work only for >=3.11) ### Local Setup diff --git a/build/ProjectInfo.fs b/build/ProjectInfo.fs index 9ee30ed3..86eeca28 100644 --- a/build/ProjectInfo.fs +++ b/build/ProjectInfo.fs @@ -30,7 +30,8 @@ let gitHome = $"https://github.com/{gitOwner}" let projectRepo = $"https://github.com/{gitOwner}/{project}" -let netPkgDir = "dist/pkg" + +let netPkgDir = "./dist/net" let npmPkgDir = "./dist/js" let pyPkgDir = "./dist/py" diff --git a/pytests/arc_table_tests.ipynb b/pytests/arc_table_tests.ipynb deleted file mode 100644 index 19762e5e..00000000 --- a/pytests/arc_table_tests.ipynb +++ /dev/null @@ -1,238 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "dotnet_interactive": { - "language": "fsharp" - }, - "polyglot_notebook": { - "kernelName": "fsharp" - } - }, - "outputs": [], - "source": [ - "from __future__ import annotations\n", - "from array import array as array_3\n", - "from typing import (Any, Callable, TypeVar)\n", - "from fable_modules.fable_library.array_ import (initialize, add_range_in_place, map as map_1, indexed, reverse as reverse_1, append as append_2, map_indexed, take, fill)\n", - "from fable_modules.fable_library.list import (FSharpList, of_array, is_empty, iterate, append as append_1, cons, singleton as singleton_1)\n", - "from fable_modules.fable_library.map import of_list\n", - "from fable_modules.fable_library.map_util import (add_to_dict, get_item_from_dict)\n", - "from fable_modules.fable_library.mutable_map import Dictionary\n", - "from fable_modules.fable_library.option import value as value_8\n", - "from fable_modules.fable_library.range import range_big_int\n", - "from fable_modules.fable_library.seq import (to_list, delay, collect, singleton, append, to_array, map)\n", - "from fable_modules.fable_library.string_ import (to_text, printf)\n", - "from fable_modules.fable_library.system_text import (StringBuilder__ctor, StringBuilder__AppendLine_Z721C83C5)\n", - "from fable_modules.fable_library.types import (Array, to_string)\n", - "from fable_modules.fable_library.util import (IEnumerable_1, int32_to_string, assert_equal, equals as equals_4, safe_hash, to_enumerable, ignore, compare_arrays, equal_arrays, array_hash)\n", - "from fable_modules.fable_pyxpecto.pyxpecto import Helper_expectError\n", - "from fable_modules.fable_pyxpecto.pyxpecto import TestCase\n", - "from src.ISA.ISA.ArcTypes.arc_table_aux import try_find_duplicate_unique_in_array\n", - "from src.ISA.ISA.ArcTypes.arc_table import ArcTable\n", - "from src.ISA.ISA.ArcTypes.arc_table_aux import (SanityChecks_validateColumnIndex, SanityChecks_validateRowIndex, try_find_duplicate_unique)\n", - "from src.ISA.ISA.ArcTypes.composite_cell import CompositeCell\n", - "from src.ISA.ISA.ArcTypes.composite_column import CompositeColumn\n", - "from src.ISA.ISA.ArcTypes.composite_header import (IOType, CompositeHeader)\n", - "from src.ISA.ISA.ArcTypes.identifier_setters import set_arc_table_name\n", - "from src.ISA.ISA.JsonTypes.ontology_annotation import OntologyAnnotation\n", - "from library import (Test_testList, Test_testCase, Expect_sequenceEqual, Expect_isTrue, Expect_notEqual, Expect_throws, Expect_isNone, Expect_isSome, Expect_containsAll, Expect_isFalse)\n", - "\n", - "__A = TypeVar(\"__A\")\n", - "\n", - "TableName: str = \"Test\"\n", - "\n", - "oa_species: OntologyAnnotation = OntologyAnnotation.from_string(\"species\", \"GO\", \"GO:0123456\")\n", - "\n", - "oa_chlamy: OntologyAnnotation = OntologyAnnotation.from_string(\"Chlamy\", \"NCBI\", \"NCBI:0123456\")\n", - "\n", - "oa_instrument_model: OntologyAnnotation = OntologyAnnotation.from_string(\"instrument model\", \"MS\", \"MS:0123456\")\n", - "\n", - "oa_sciexinstrument_model: OntologyAnnotation = OntologyAnnotation.from_string(\"SCIEX instrument model\", \"MS\", \"MS:654321\")\n", - "\n", - "oa_temperature: OntologyAnnotation = OntologyAnnotation.from_string(\"temperature\", \"NCIT\", \"NCIT:0123210\")\n", - "\n", - "def table_values_printable(table: ArcTable) -> FSharpList[str]:\n", - " def _arrow1027(__unit: None=None, table: Any=table) -> IEnumerable_1[str]:\n", - " def _arrow1026(match_value: Any) -> IEnumerable_1[str]:\n", - " active_pattern_result: tuple[tuple[int, int], CompositeCell] = match_value\n", - " return singleton((((((\"(\" + str(active_pattern_result[0][0])) + \",\") + str(active_pattern_result[0][1])) + \") \") + str(active_pattern_result[1])) + \"\")\n", - "\n", - " return collect(_arrow1026, table.Values)\n", - "\n", - " return to_list(delay(_arrow1027))\n", - "\n", - "\n", - "def create_cells_free_text(pretext: Any, count: int) -> Array[CompositeCell]:\n", - " def _arrow1030(i: int, pretext: Any=pretext, count: Any=count) -> CompositeCell:\n", - " return CompositeCell.create_free_text((((\"\" + str(pretext)) + \"_\") + str(i)) + \"\")\n", - "\n", - " return initialize(count, _arrow1030, None)\n", - "\n", - "\n", - "def create_cells_term(count: int) -> Array[CompositeCell]:\n", - " def _arrow1032(_arg: int, count: Any=count) -> CompositeCell:\n", - " return CompositeCell.create_term(oa_sciexinstrument_model)\n", - "\n", - " return initialize(count, _arrow1032, None)\n", - "\n", - "\n", - "def create_cells_unitized(count: int) -> Array[CompositeCell]:\n", - " def _arrow1035(i: int, count: Any=count) -> CompositeCell:\n", - " return CompositeCell.create_unitized(int32_to_string(i), OntologyAnnotation.empty())\n", - "\n", - " return initialize(count, _arrow1035, None)\n", - "\n", - "\n", - "column_input: CompositeColumn = CompositeColumn.create(CompositeHeader(11, IOType(0)), create_cells_free_text(\"Source\", 5))\n", - "\n", - "column_output: CompositeColumn = CompositeColumn.create(CompositeHeader(12, IOType(1)), create_cells_free_text(\"Sample\", 5))\n", - "\n", - "column_component: CompositeColumn = CompositeColumn.create(CompositeHeader(0, oa_instrument_model), create_cells_term(5))\n", - "\n", - "column_param: CompositeColumn = CompositeColumn.create(CompositeHeader(3, OntologyAnnotation.empty()), create_cells_unitized(5))\n", - "\n", - "def create_test_table(__unit: None=None) -> ArcTable:\n", - " t: ArcTable = ArcTable.init(TableName)\n", - " columns: Array[CompositeColumn] = [column_input, column_output, column_param, column_component, column_param]\n", - " t.AddColumns(columns)\n", - " return t\n" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "table1: ArcTable = create_test_table()" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "8821596988035483298" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# table1.GetHeadersHashCode()\n", - "from src.ISA.ISA.helper import HashCodes_boxHashArray\n", - "\n", - "# HashCodes_boxHashArray(list(table1.Headers))\n", - "\n", - "from fable_modules.fable_library.util import (get_enumerator, dispose, equals, structural_hash, identity_hash, number_hash, IEnumerable_1, ignore, is_iterable)\n", - "_A = TypeVar(\"_A\")\n", - "\n", - "copy_of_struct: _A = table1.Headers[0]\n", - "identity_hash(copy_of_struct)\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "ename": "TypeError", - "evalue": "unhashable type: 'OntologyAnnotation'", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mTypeError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32mc:\\Users\\HLWei\\source\\repos\\ARC_tools\\ARCtrl\\pytests\\arc_table_tests.ipynb Cell 4\u001b[0m line \u001b[0;36m3\n\u001b[0;32m 33\u001b[0m identity_hash(oa)\n\u001b[0;32m 35\u001b[0m p \u001b[39m=\u001b[39m CompositeHeader\u001b[39m.\u001b[39mparameter(oa)\n\u001b[1;32m---> 36\u001b[0m identity_hash(p)\n", - "File \u001b[1;32mc:\\Users\\HLWei\\source\\repos\\ARC_tools\\ARCtrl\\pytests\\fable_modules\\fable_library\\util.py:2642\u001b[0m, in \u001b[0;36midentity_hash\u001b[1;34m(x)\u001b[0m\n\u001b[0;32m 2639\u001b[0m \u001b[39mreturn\u001b[39;00m x\u001b[39m.\u001b[39mGetHashCode()\n\u001b[0;32m 2641\u001b[0m \u001b[39mif\u001b[39;00m is_hashable_py(x):\n\u001b[1;32m-> 2642\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mhash\u001b[39m(x)\n\u001b[0;32m 2644\u001b[0m \u001b[39mreturn\u001b[39;00m physical_hash(x)\n", - "File \u001b[1;32mc:\\Users\\HLWei\\source\\repos\\ARC_tools\\ARCtrl\\pytests\\fable_modules\\fable_library\\types.py:85\u001b[0m, in \u001b[0;36mUnion.__hash__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 83\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__hash__\u001b[39m(\u001b[39mself\u001b[39m) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mint\u001b[39m:\n\u001b[0;32m 84\u001b[0m hashes \u001b[39m=\u001b[39m \u001b[39mmap\u001b[39m(\u001b[39mhash\u001b[39m, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mfields)\n\u001b[1;32m---> 85\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mhash\u001b[39m((\u001b[39mhash\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtag), \u001b[39m*\u001b[39mhashes))\n", - "\u001b[1;31mTypeError\u001b[0m: unhashable type: 'OntologyAnnotation'" - ] - } - ], - "source": [ - "from fable_modules.fable_library.array_ import (contains, fold, partition, exactly_one)\n", - "\n", - "a = table1.Headers\n", - "\n", - "# def folder(acc: int, o: _A, a: Any=a) -> int:\n", - "# def _arrow67(__unit: None=None, acc: Any=acc, o: Any=o) -> int:\n", - "# copy_of_struct: _A = o\n", - "# return identity_hash(copy_of_struct)\n", - "\n", - "# return ((-1640531527 + _arrow67()) + (acc << 6)) + (acc >> 2)\n", - "\n", - "# failt\n", - "# fold(folder, 0, a)\n", - "\n", - "\n", - "# identity_hash(table1.Headers[0])\n", - "# identity_hash(table1.Headers[1])\n", - "# failt\n", - "# identity_hash(table1.Headers[2])\n", - "# identity_hash(table1.Headers[3])\n", - "# identity_hash(table1.Headers[4])\n", - "\n", - "\n", - "# failt\n", - "# identity_hash(column_param.Header)\n", - "\n", - "\n", - "\n", - "table1.Headers[2]\n", - "\n", - "oa = OntologyAnnotation.empty()\n", - "\n", - "identity_hash(oa)\n", - "\n", - "# p = CompositeHeader.parameter(oa)\n", - "\n", - "# identity_hash(p)\n", - "\n", - "\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.0" - }, - "polyglot_notebook": { - "kernelInfo": { - "defaultKernelName": "fsharp", - "items": [ - { - "aliases": [], - "languageName": "fsharp", - "name": "fsharp" - } - ] - } - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/pytests/composite_header_tests.ipynb b/pytests/composite_header_tests.ipynb deleted file mode 100644 index 19762e5e..00000000 --- a/pytests/composite_header_tests.ipynb +++ /dev/null @@ -1,238 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "dotnet_interactive": { - "language": "fsharp" - }, - "polyglot_notebook": { - "kernelName": "fsharp" - } - }, - "outputs": [], - "source": [ - "from __future__ import annotations\n", - "from array import array as array_3\n", - "from typing import (Any, Callable, TypeVar)\n", - "from fable_modules.fable_library.array_ import (initialize, add_range_in_place, map as map_1, indexed, reverse as reverse_1, append as append_2, map_indexed, take, fill)\n", - "from fable_modules.fable_library.list import (FSharpList, of_array, is_empty, iterate, append as append_1, cons, singleton as singleton_1)\n", - "from fable_modules.fable_library.map import of_list\n", - "from fable_modules.fable_library.map_util import (add_to_dict, get_item_from_dict)\n", - "from fable_modules.fable_library.mutable_map import Dictionary\n", - "from fable_modules.fable_library.option import value as value_8\n", - "from fable_modules.fable_library.range import range_big_int\n", - "from fable_modules.fable_library.seq import (to_list, delay, collect, singleton, append, to_array, map)\n", - "from fable_modules.fable_library.string_ import (to_text, printf)\n", - "from fable_modules.fable_library.system_text import (StringBuilder__ctor, StringBuilder__AppendLine_Z721C83C5)\n", - "from fable_modules.fable_library.types import (Array, to_string)\n", - "from fable_modules.fable_library.util import (IEnumerable_1, int32_to_string, assert_equal, equals as equals_4, safe_hash, to_enumerable, ignore, compare_arrays, equal_arrays, array_hash)\n", - "from fable_modules.fable_pyxpecto.pyxpecto import Helper_expectError\n", - "from fable_modules.fable_pyxpecto.pyxpecto import TestCase\n", - "from src.ISA.ISA.ArcTypes.arc_table_aux import try_find_duplicate_unique_in_array\n", - "from src.ISA.ISA.ArcTypes.arc_table import ArcTable\n", - "from src.ISA.ISA.ArcTypes.arc_table_aux import (SanityChecks_validateColumnIndex, SanityChecks_validateRowIndex, try_find_duplicate_unique)\n", - "from src.ISA.ISA.ArcTypes.composite_cell import CompositeCell\n", - "from src.ISA.ISA.ArcTypes.composite_column import CompositeColumn\n", - "from src.ISA.ISA.ArcTypes.composite_header import (IOType, CompositeHeader)\n", - "from src.ISA.ISA.ArcTypes.identifier_setters import set_arc_table_name\n", - "from src.ISA.ISA.JsonTypes.ontology_annotation import OntologyAnnotation\n", - "from library import (Test_testList, Test_testCase, Expect_sequenceEqual, Expect_isTrue, Expect_notEqual, Expect_throws, Expect_isNone, Expect_isSome, Expect_containsAll, Expect_isFalse)\n", - "\n", - "__A = TypeVar(\"__A\")\n", - "\n", - "TableName: str = \"Test\"\n", - "\n", - "oa_species: OntologyAnnotation = OntologyAnnotation.from_string(\"species\", \"GO\", \"GO:0123456\")\n", - "\n", - "oa_chlamy: OntologyAnnotation = OntologyAnnotation.from_string(\"Chlamy\", \"NCBI\", \"NCBI:0123456\")\n", - "\n", - "oa_instrument_model: OntologyAnnotation = OntologyAnnotation.from_string(\"instrument model\", \"MS\", \"MS:0123456\")\n", - "\n", - "oa_sciexinstrument_model: OntologyAnnotation = OntologyAnnotation.from_string(\"SCIEX instrument model\", \"MS\", \"MS:654321\")\n", - "\n", - "oa_temperature: OntologyAnnotation = OntologyAnnotation.from_string(\"temperature\", \"NCIT\", \"NCIT:0123210\")\n", - "\n", - "def table_values_printable(table: ArcTable) -> FSharpList[str]:\n", - " def _arrow1027(__unit: None=None, table: Any=table) -> IEnumerable_1[str]:\n", - " def _arrow1026(match_value: Any) -> IEnumerable_1[str]:\n", - " active_pattern_result: tuple[tuple[int, int], CompositeCell] = match_value\n", - " return singleton((((((\"(\" + str(active_pattern_result[0][0])) + \",\") + str(active_pattern_result[0][1])) + \") \") + str(active_pattern_result[1])) + \"\")\n", - "\n", - " return collect(_arrow1026, table.Values)\n", - "\n", - " return to_list(delay(_arrow1027))\n", - "\n", - "\n", - "def create_cells_free_text(pretext: Any, count: int) -> Array[CompositeCell]:\n", - " def _arrow1030(i: int, pretext: Any=pretext, count: Any=count) -> CompositeCell:\n", - " return CompositeCell.create_free_text((((\"\" + str(pretext)) + \"_\") + str(i)) + \"\")\n", - "\n", - " return initialize(count, _arrow1030, None)\n", - "\n", - "\n", - "def create_cells_term(count: int) -> Array[CompositeCell]:\n", - " def _arrow1032(_arg: int, count: Any=count) -> CompositeCell:\n", - " return CompositeCell.create_term(oa_sciexinstrument_model)\n", - "\n", - " return initialize(count, _arrow1032, None)\n", - "\n", - "\n", - "def create_cells_unitized(count: int) -> Array[CompositeCell]:\n", - " def _arrow1035(i: int, count: Any=count) -> CompositeCell:\n", - " return CompositeCell.create_unitized(int32_to_string(i), OntologyAnnotation.empty())\n", - "\n", - " return initialize(count, _arrow1035, None)\n", - "\n", - "\n", - "column_input: CompositeColumn = CompositeColumn.create(CompositeHeader(11, IOType(0)), create_cells_free_text(\"Source\", 5))\n", - "\n", - "column_output: CompositeColumn = CompositeColumn.create(CompositeHeader(12, IOType(1)), create_cells_free_text(\"Sample\", 5))\n", - "\n", - "column_component: CompositeColumn = CompositeColumn.create(CompositeHeader(0, oa_instrument_model), create_cells_term(5))\n", - "\n", - "column_param: CompositeColumn = CompositeColumn.create(CompositeHeader(3, OntologyAnnotation.empty()), create_cells_unitized(5))\n", - "\n", - "def create_test_table(__unit: None=None) -> ArcTable:\n", - " t: ArcTable = ArcTable.init(TableName)\n", - " columns: Array[CompositeColumn] = [column_input, column_output, column_param, column_component, column_param]\n", - " t.AddColumns(columns)\n", - " return t\n" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "table1: ArcTable = create_test_table()" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "8821596988035483298" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# table1.GetHeadersHashCode()\n", - "from src.ISA.ISA.helper import HashCodes_boxHashArray\n", - "\n", - "# HashCodes_boxHashArray(list(table1.Headers))\n", - "\n", - "from fable_modules.fable_library.util import (get_enumerator, dispose, equals, structural_hash, identity_hash, number_hash, IEnumerable_1, ignore, is_iterable)\n", - "_A = TypeVar(\"_A\")\n", - "\n", - "copy_of_struct: _A = table1.Headers[0]\n", - "identity_hash(copy_of_struct)\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "ename": "TypeError", - "evalue": "unhashable type: 'OntologyAnnotation'", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mTypeError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32mc:\\Users\\HLWei\\source\\repos\\ARC_tools\\ARCtrl\\pytests\\arc_table_tests.ipynb Cell 4\u001b[0m line \u001b[0;36m3\n\u001b[0;32m 33\u001b[0m identity_hash(oa)\n\u001b[0;32m 35\u001b[0m p \u001b[39m=\u001b[39m CompositeHeader\u001b[39m.\u001b[39mparameter(oa)\n\u001b[1;32m---> 36\u001b[0m identity_hash(p)\n", - "File \u001b[1;32mc:\\Users\\HLWei\\source\\repos\\ARC_tools\\ARCtrl\\pytests\\fable_modules\\fable_library\\util.py:2642\u001b[0m, in \u001b[0;36midentity_hash\u001b[1;34m(x)\u001b[0m\n\u001b[0;32m 2639\u001b[0m \u001b[39mreturn\u001b[39;00m x\u001b[39m.\u001b[39mGetHashCode()\n\u001b[0;32m 2641\u001b[0m \u001b[39mif\u001b[39;00m is_hashable_py(x):\n\u001b[1;32m-> 2642\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mhash\u001b[39m(x)\n\u001b[0;32m 2644\u001b[0m \u001b[39mreturn\u001b[39;00m physical_hash(x)\n", - "File \u001b[1;32mc:\\Users\\HLWei\\source\\repos\\ARC_tools\\ARCtrl\\pytests\\fable_modules\\fable_library\\types.py:85\u001b[0m, in \u001b[0;36mUnion.__hash__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 83\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__hash__\u001b[39m(\u001b[39mself\u001b[39m) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mint\u001b[39m:\n\u001b[0;32m 84\u001b[0m hashes \u001b[39m=\u001b[39m \u001b[39mmap\u001b[39m(\u001b[39mhash\u001b[39m, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mfields)\n\u001b[1;32m---> 85\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mhash\u001b[39m((\u001b[39mhash\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtag), \u001b[39m*\u001b[39mhashes))\n", - "\u001b[1;31mTypeError\u001b[0m: unhashable type: 'OntologyAnnotation'" - ] - } - ], - "source": [ - "from fable_modules.fable_library.array_ import (contains, fold, partition, exactly_one)\n", - "\n", - "a = table1.Headers\n", - "\n", - "# def folder(acc: int, o: _A, a: Any=a) -> int:\n", - "# def _arrow67(__unit: None=None, acc: Any=acc, o: Any=o) -> int:\n", - "# copy_of_struct: _A = o\n", - "# return identity_hash(copy_of_struct)\n", - "\n", - "# return ((-1640531527 + _arrow67()) + (acc << 6)) + (acc >> 2)\n", - "\n", - "# failt\n", - "# fold(folder, 0, a)\n", - "\n", - "\n", - "# identity_hash(table1.Headers[0])\n", - "# identity_hash(table1.Headers[1])\n", - "# failt\n", - "# identity_hash(table1.Headers[2])\n", - "# identity_hash(table1.Headers[3])\n", - "# identity_hash(table1.Headers[4])\n", - "\n", - "\n", - "# failt\n", - "# identity_hash(column_param.Header)\n", - "\n", - "\n", - "\n", - "table1.Headers[2]\n", - "\n", - "oa = OntologyAnnotation.empty()\n", - "\n", - "identity_hash(oa)\n", - "\n", - "# p = CompositeHeader.parameter(oa)\n", - "\n", - "# identity_hash(p)\n", - "\n", - "\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.0" - }, - "polyglot_notebook": { - "kernelInfo": { - "defaultKernelName": "fsharp", - "items": [ - { - "aliases": [], - "languageName": "fsharp", - "name": "fsharp" - } - ] - } - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/src/ARCtrl/ARCtrl.fsproj b/src/ARCtrl/ARCtrl.fsproj index 79de8a7e..ab3699c3 100644 --- a/src/ARCtrl/ARCtrl.fsproj +++ b/src/ARCtrl/ARCtrl.fsproj @@ -51,6 +51,9 @@ + + + diff --git a/src/ISA/ISA.Json/ArcTypes/ArcTable.fs b/src/ISA/ISA.Json/ArcTypes/ArcTable.fs index 44bd86c9..9de244c6 100644 --- a/src/ISA/ISA.Json/ArcTypes/ArcTable.fs +++ b/src/ISA/ISA.Json/ArcTypes/ArcTable.fs @@ -160,7 +160,7 @@ module ArcTableExtensions = member this.ToCompressedJsonString(?spaces) : string = let spaces = defaultArg spaces 0 - let stringTable = Dictionary() + let stringTable = Dictionary() let oaTable = Dictionary() let cellTable = Dictionary() let arcTable = ArcTable.compressedEncoder stringTable oaTable cellTable this diff --git a/src/ISA/ISA/ArcTypes/ArcTable.fs b/src/ISA/ISA/ArcTypes/ArcTable.fs index afe35ddb..0c43d20c 100644 --- a/src/ISA/ISA/ArcTypes/ArcTable.fs +++ b/src/ISA/ISA/ArcTypes/ArcTable.fs @@ -271,7 +271,7 @@ type ArcTable(name: string, headers: ResizeArray, values: Syste // Sanity check here too, to avoid removing things from mutable to fail in the middle Array.iter (fun index -> SanityChecks.validateColumnIndex index this.ColumnCount false) indexArr /// go from highest to lowest so no wrong column gets removed after index shift - let indexArr = indexArr |> ARCtrl.ISA.Fable.Array.sortDescending + let indexArr = indexArr |> Array.sortDescending Array.iter (fun index -> this.RemoveColumn index) indexArr static member removeColumns(indexArr:int []) = @@ -476,7 +476,7 @@ type ArcTable(name: string, headers: ResizeArray, values: Syste // Sanity check here too, to avoid removing things from mutable to fail in the middle Array.iter (fun index -> ArcTableAux.SanityChecks.validateRowIndex index this.RowCount false) indexArr /// go from highest to lowest so no wrong column gets removed after index shift - let indexArr = indexArr |> ARCtrl.ISA.Fable.Array.sortDescending + let indexArr = indexArr |> Array.sortDescending Array.iter (fun index -> this.RemoveRow index) indexArr static member removeRows (indexArr:int []) = @@ -757,7 +757,7 @@ type ArcTable(name: string, headers: ResizeArray, values: Syste |> String.concat "\n" member this.StructurallyEquals (other: ArcTable) = - let sort = Array.ofSeq >> ARCtrl.ISA.Fable.Array.sortBy (function |KeyValue (key,_) -> key) + let sort = Array.ofSeq >> Array.sortBy (function |KeyValue (key,_) -> key) let n = this.Name = other.Name let headers = Aux.compareSeq this.Headers other.Headers let values = Aux.compareSeq (sort this.Values) (sort other.Values) diff --git a/src/ISA/ISA/Fable.fs b/src/ISA/ISA/Fable.fs index e4d2c2ec..fb8286e0 100644 --- a/src/ISA/ISA/Fable.fs +++ b/src/ISA/ISA/Fable.fs @@ -118,35 +118,4 @@ let distinct_generic l1 = let hashSeq (s:seq<'a>) = s |> Seq.map (fun x -> x.GetHashCode()) - |> Seq.reduce (fun a b -> a + b) - -module Array = - - - let sort (a : 'T []) = - #if FABLE_COMPILER_PYTHON - a |> List.ofArray |> List.sort |> Array.ofList - #else - a |> Array.sort - #endif - - let sortBy (f : 'T -> 'U) (a : 'T []) = - #if FABLE_COMPILER_PYTHON - a |> List.ofArray |> List.sortBy f |> Array.ofList - #else - a |> Array.sortBy f - #endif - - let sortDescending (a : 'T []) = - #if FABLE_COMPILER_PYTHON - a |> List.ofArray |> List.sortDescending |> Array.ofList - #else - a |> Array.sortDescending - #endif - - let sortDescendingBy (f : 'T -> 'U) (a : 'T []) = - #if FABLE_COMPILER_PYTHON - a |> List.ofArray |> List.sortByDescending f |> Array.ofList - #else - a |> Array.sortByDescending f - #endif + |> Seq.reduce (fun a b -> a + b) \ No newline at end of file diff --git a/testpy.cmd b/testpy.cmd deleted file mode 100644 index 045ac823..00000000 --- a/testpy.cmd +++ /dev/null @@ -1,2 +0,0 @@ -dotnet fable .\tests\ISA\ISA.Tests --lang py -o ./pytests --nocache -python .\pytests\main.py \ No newline at end of file From 78914e0fc5d0b13a5e8f2cfa6380f148d2f13435 Mon Sep 17 00:00:00 2001 From: HLWeil Date: Fri, 8 Mar 2024 13:54:18 +0100 Subject: [PATCH 24/25] bump to 1.2.0 including python release --- RELEASE_NOTES.md | 40 ++++++++++++++++++++++++++++++++++++++ build/PackageTasks.fs | 2 +- build/ReleaseNotesTasks.fs | 1 + build/release_package.json | 2 +- pyproject.toml | 5 +++-- src/ISA/ISA/Helper.fs | 2 ++ 6 files changed, 48 insertions(+), 4 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 6fcebfd2..6ac1e809 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,3 +1,43 @@ +### 1.2.0+19d850e (Released 2024-3-8) +* Additions: + * Added Python compatability + * [[#19d850e](https://github.com/nfdi4plants/ARCtrl/commit/19d850ed5e1474a5e82cd98537db3f398fbacc18)] several small cleanups according to PR #317 comments + * [[#8becc67](https://github.com/nfdi4plants/ARCtrl/commit/8becc672c432eb4399fc3a0e250a87e63044de2b)] update build project for releasing python package + * [[#df663cb](https://github.com/nfdi4plants/ARCtrl/commit/df663cbfe8cdfaa7ac48690f89539b17715ae0c0)] include python setup in CI + * [[#66b83a2](https://github.com/nfdi4plants/ARCtrl/commit/66b83a2292b082e96676a52c1dd176695173f949)] several small changes to test stack + * [[#3dfaad7](https://github.com/nfdi4plants/ARCtrl/commit/3dfaad77a14aa70715388ef6a1fa77cb337bd622)] adjustments to web and validation to allowe for python transpilation + * [[#688c628](https://github.com/nfdi4plants/ARCtrl/commit/688c6283e9b2bd332fd93388b10d8661e185bc1b)] work on json io support in python + * [[#93b6d94](https://github.com/nfdi4plants/ARCtrl/commit/93b6d9490acdf16045bd844f4cec22f2d54c7eca)] replace mocha and expecto with pyxpecto + * [[#aa30389](https://github.com/nfdi4plants/ARCtrl/commit/aa30389f070a248e547373779de64247d0b017cc)] move json schema validation to json tests + * [[#f0bb78d](https://github.com/nfdi4plants/ARCtrl/commit/f0bb78dcdccb1b2bfc31e6cc1c36c6905c950045)] switch towards using .venv for running transpiled python + * [[#2d35cfb](https://github.com/nfdi4plants/ARCtrl/commit/2d35cfbb7b75ca79e5b2d5f28cb6dc56514acab0)] update Fable and pyxpecto + * [[#32a4128](https://github.com/nfdi4plants/ARCtrl/commit/32a4128fcad38d35e695d4dd86db9b19478c1543)] add hashcode tests + * [[#cc8c116](https://github.com/nfdi4plants/ARCtrl/commit/cc8c11661e640ce96c73eb136ae4681f710a65b8)] increase json write test timeout + * [[#3d00621](https://github.com/nfdi4plants/ARCtrl/commit/3d00621ff1feb76f1a4628f6cdad7a3a7cdb49fd)] small adjustments in isa.json + * [[#10c5538](https://github.com/nfdi4plants/ARCtrl/commit/10c55387c420c12804b3f10e737e1d5b1f4b3b43)] cleanup merge of json-ld and thoth.json + * [[#58b3d5a](https://github.com/nfdi4plants/ARCtrl/commit/58b3d5a500c36dd09b34cac2cb2cc62efaf8d3d6)] merge json-ld changes into thoth.json update changes and refactor + * [[#87ab15d](https://github.com/nfdi4plants/ARCtrl/commit/87ab15df0930389a8a9e6bdd7824a89b1ae30790)] increase json parsing test timeout + * [[#90c60b9](https://github.com/nfdi4plants/ARCtrl/commit/90c60b90031b80efd6b23486e7e8061a2b2cc8cb)] rework json encoding + * [[#c482c86](https://github.com/nfdi4plants/ARCtrl/commit/c482c86806d5734755a46b9c5b80bbdc32f5e001)] finish thoth conversion + * [[#d0a99a3](https://github.com/nfdi4plants/ARCtrl/commit/d0a99a314da50eba37f23aa0e0e5c52f97dbd6ce)] start reworking json towards net Thoth.Json + * [[#4f77082](https://github.com/nfdi4plants/ARCtrl/commit/4f77082708909086ec0113e18b2ed8102cd975bd)] Merge pull request #271 from nfdi4plants/jsonld + * [[#3f84e17](https://github.com/nfdi4plants/ARCtrl/commit/3f84e17a2ac724a5fc74a33e5352e34b79b1022a)] increase speed of ARC to Json Type conversion + * [[#5ceb732](https://github.com/nfdi4plants/ARCtrl/commit/5ceb73209026e512b16461ca6ef6788100bdfe18)] bump to 1.1.0 +* Bugfixes: + * [[#890048e](https://github.com/nfdi4plants/ARCtrl/commit/890048e84e06a88b4b957fc46e30113c40c014d0)] set ci fail-fast to false and fix py encoding in windows + * [[#3e403cd](https://github.com/nfdi4plants/ARCtrl/commit/3e403cdd687be78f1cbadfafb30b148247800bdd)] small fix to CI + * [[#b0186f4](https://github.com/nfdi4plants/ARCtrl/commit/b0186f4c3e28fbf9a6678dde5795e6e0723934c0)] small fix to CI + * [[#89aa3c2](https://github.com/nfdi4plants/ARCtrl/commit/89aa3c2a93cb718ca77433e35599d2d22deb2886)] fixed python tests to work on all platforms + * [[#4262833](https://github.com/nfdi4plants/ARCtrl/commit/4262833f532c3ed8deedc1ca30b6ad30c04c17b3)] hotfix js webrequest + * [[#f75ce4d](https://github.com/nfdi4plants/ARCtrl/commit/f75ce4d3d9abc84600a769d8ead47a6bd2befe8b)] small python tests hotfix + * [[#9385d06](https://github.com/nfdi4plants/ARCtrl/commit/9385d06c4bb728edd6c3d95f528fd0a40a19062d)] small fixes in python and setup instructions + * [[#b5eaaed](https://github.com/nfdi4plants/ARCtrl/commit/b5eaaedd24cad0a24423aec39dbdf56bbaac14dd)] hotfix fable python hashing of person + * [[#556e2bf](https://github.com/nfdi4plants/ARCtrl/commit/556e2bf7f0bd85417ce43078ff3c2357819bef1d)] fix http requests in python + * [[#c534acf](https://github.com/nfdi4plants/ARCtrl/commit/c534acf43e16d1e76bb22bbc7e0d1e7134d213e5)] fix and test comment regex handling for python + * [[#2540056](https://github.com/nfdi4plants/ARCtrl/commit/2540056063a107fb18ad30b577214e946a6ae48c)] small fix for compressed json io stringtable conversion https://github.com/fable-compiler/Fable/issues/3771 + * [[#92c72ee](https://github.com/nfdi4plants/ARCtrl/commit/92c72ee0135c712ded59c00c9a0111909ce99c3a)] array fixes in python compilation + * [[#b55d360](https://github.com/nfdi4plants/ARCtrl/commit/b55d3608eb73273a84cca756e0662811ff7b6244)] fix and finish up thoth migration + ### 1.1.0+6309e03 (Released 2024-2-15) * Additions: * [[#544ffdc](https://github.com/nfdi4plants/ARCtrl/commit/544ffdcbcf2e19c12c995e408713b83157cda345)] add some additional information to failing spreadsheet parsers #306 diff --git a/build/PackageTasks.fs b/build/PackageTasks.fs index 9fc2b35b..6d073b24 100644 --- a/build/PackageTasks.fs +++ b/build/PackageTasks.fs @@ -93,7 +93,7 @@ module BundlePy = run python "-m poetry build" ProjectInfo.pyPkgDir //Remove "-o ." because not compatible with publish -let packPy = BuildTask.create "PackPy" [clean; build; (*runTests*)] { +let packPy = BuildTask.create "PackPy" [clean; build; runTests] { BundlePy.bundle ProjectInfo.stableVersionTag } diff --git a/build/ReleaseNotesTasks.fs b/build/ReleaseNotesTasks.fs index 6b60a779..0465244f 100644 --- a/build/ReleaseNotesTasks.fs +++ b/build/ReleaseNotesTasks.fs @@ -30,6 +30,7 @@ let updateReleaseNotes = BuildTask.createFn "ReleaseNotes" [] (fun config -> let t = System.IO.File.ReadAllText p let tNew = System.Text.RegularExpressions.Regex.Replace(t, "version = \".*\"", sprintf "version = \"%s\"" semVerShort ) System.IO.File.WriteAllText(p, tNew) + Trace.trace "Finish updating pyproject.toml version" ) diff --git a/build/release_package.json b/build/release_package.json index 85f36ce1..2786b4fa 100644 --- a/build/release_package.json +++ b/build/release_package.json @@ -1,6 +1,6 @@ { "name": "@nfdi4plants/arctrl", - "version": "1.1.0+6309e03", + "version": "1.2.0+19d850e", "description": "Top level ARC DataModel and API function descriptions.", "type": "module", "main": "index.js", diff --git a/pyproject.toml b/pyproject.toml index ed391162..ed616d1f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,9 @@ [tool.poetry] name = "ARCtrl" -version = "1.0.0" +version = "1.2.0" description = "Library for management of Annotated Research Contexts (ARCs) using an in-memory representation and runtimer agnostic contract systems." -authors = ["Heinrich Lukas Weil "] +authors = ["Heinrich Lukas Weil ", "Kevin Frey "] +maintainers = ["Florian Wetzels"] readme = "README.md" [tool.poetry.dependencies] diff --git a/src/ISA/ISA/Helper.fs b/src/ISA/ISA/Helper.fs index 979edaee..2a259636 100644 --- a/src/ISA/ISA/Helper.fs +++ b/src/ISA/ISA/Helper.fs @@ -78,6 +78,8 @@ module HashCodes = open Fable.Core + // This is necessary until https://github.com/fable-compiler/Fable/issues/3778 is resolved + #if FABLE_COMPILER_PYTHON [] let pyHasCustomHash (obj) : bool = nativeOnly From b6e2ecab5a0a3630e1dd24ae36d34be79ad21d6d Mon Sep 17 00:00:00 2001 From: HLWeil Date: Fri, 8 Mar 2024 14:28:35 +0100 Subject: [PATCH 25/25] small change to semver in npm --- build/ReleaseNotesTasks.fs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/build/ReleaseNotesTasks.fs b/build/ReleaseNotesTasks.fs index 0465244f..2fecf714 100644 --- a/build/ReleaseNotesTasks.fs +++ b/build/ReleaseNotesTasks.fs @@ -13,22 +13,20 @@ let createAssemblyVersion = BuildTask.create "createvfs" [] { let updateReleaseNotes = BuildTask.createFn "ReleaseNotes" [] (fun config -> ReleaseNotes.update(ProjectInfo.gitOwner, ProjectInfo.project, config) - let semVerLong,semVerShort = + let semVer = Fake.Core.ReleaseNotes.load "RELEASE_NOTES.md" - |> fun x -> - x.SemVer.AsString, - $"{x.SemVer.Major}.{x.SemVer.Minor}.{x.SemVer.Patch}" + |> fun x -> $"{x.SemVer.Major}.{x.SemVer.Minor}.{x.SemVer.Patch}" Trace.trace "Start updating package.json version" // Update Version in src/Nfdi4Plants.Fornax.Template/package.json let p = "build/release_package.json" let t = System.IO.File.ReadAllText p - let tNew = System.Text.RegularExpressions.Regex.Replace(t, """\"version\": \".*\",""", sprintf "\"version\": \"%s\"," semVerLong ) + let tNew = System.Text.RegularExpressions.Regex.Replace(t, """\"version\": \".*\",""", sprintf "\"version\": \"%s\"," semVer ) System.IO.File.WriteAllText(p, tNew) Trace.trace "Finish updating package.json version" Trace.trace "Start updating pyproject.toml version" let p = "pyproject.toml" let t = System.IO.File.ReadAllText p - let tNew = System.Text.RegularExpressions.Regex.Replace(t, "version = \".*\"", sprintf "version = \"%s\"" semVerShort ) + let tNew = System.Text.RegularExpressions.Regex.Replace(t, "version = \".*\"", sprintf "version = \"%s\"" semVer) System.IO.File.WriteAllText(p, tNew) Trace.trace "Finish updating pyproject.toml version" )