edgetest sparkfix (#269)

* moving to conda dep

* edgetest action python version

* edgetest action python version

* adding in cloudpickle

* adding in cloudpickle

* [TEMP] changing branch

* [TEMP] changing branch

* reverting some changes

* DEBUG

* DEBUG

* DEBUG

* DEBUG

* DEBUG

* DEBUG

* DEBUG

* DEBUG

* DEBUG

* DEBUG

* DEBUG

* DEBUG

* finally..it works...RIP

.github/workflows/edgetest.yml

@@ -11,12 +11,45 @@ jobs:
     runs-on: ubuntu-latest
     name: running edgetest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           ref: develop
-      - id: run-edgetest
-        uses: edgetest-dev/[email protected]
+
+      - name: Set up Python 3.10
+        uses: conda-incubator/setup-miniconda@v2
         with:
-          edgetest-flags: '-c pyproject.toml --export'
-          base-branch: 'develop'
-          skip-pr: 'false'
+          auto-update-conda: true
+          python-version: '3.10'
+          channels: conda-forge
+
+      - name: Setup Java JDK
+        uses: actions/setup-java@v3
+        with:
+          java-version: '8'
+          distribution: 'adopt'
+
+      - name: Install edgetest
+        shell: bash -el {0}
+        run: |
+          conda install pip
+          conda install edgetest edgetest-conda
+          python -m pip install .[dev]
+      
+      - name: Run edgetest
+        shell: bash -el {0}
+        run: |
+          edgetest -c pyproject.toml --export
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v3
+        with:
+          branch: edgetest-patch
+          base: develop
+          delete-branch: true
+          title: Changes by run-edgetest action
+          commit-message: '[edgetest] automated change'
+          body: Automated changes by [run-edgetest-action](https://github.com/edgetest-dev/run-edgetest-action) GitHub action
+          add-paths: |
+            requirements.txt
+            setup.cfg
+            pyproject.toml

pyproject.toml

@@ -133,7 +133,6 @@ ignore_missing_imports = true
 python_version = "3.10"
 conda_install = ["openjdk=8"]
 extras = ["dev"]
-deps = ["pyspark==3.4.2"]
 command = "pytest tests -m 'not integration'"
 upgrade = [
     "pandas",

pytest.ini

@@ -6,5 +6,4 @@ spark_options =
   spark.default.parallelism: 4
   spark.executor.cores: 4
   spark.sql.execution.arrow.pyspark.enabled: true
-  spark.sql.execution.arrow.enabled: false
   spark.sql.adaptive.enabled: false