From e1b08c398019ec35e55d784fcf87c1cbd4fd4fbc Mon Sep 17 00:00:00 2001
From: Faisal <faisal.dosani@pm.me>
Date: Wed, 21 Feb 2024 11:01:48 -0400
Subject: [PATCH] edgetest sparkfix (#269)
* moving to conda dep
* edgetest action python version
* edgetest action python version
* adding in cloudpickle
* adding in cloudpickle
* [TEMP] changing branch
* [TEMP] changing branch
* reverting some changes
* DEBUG
* DEBUG
* DEBUG
* DEBUG
* DEBUG
* DEBUG
* DEBUG
* DEBUG
* DEBUG
* DEBUG
* DEBUG
* DEBUG
* finally..it works...RIP
---
.github/workflows/edgetest.yml | 45 +++++++++++++++++++++++++++++-----
pyproject.toml | 1 -
pytest.ini | 1 -
3 files changed, 39 insertions(+), 8 deletions(-)
diff --git a/.github/workflows/edgetest.yml b/.github/workflows/edgetest.yml
index 7e0a5524..e47ca6ff 100644
--- a/.github/workflows/edgetest.yml
+++ b/.github/workflows/edgetest.yml
@@ -11,12 +11,45 @@ jobs:
runs-on: ubuntu-latest
name: running edgetest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
with:
ref: develop
- - id: run-edgetest
- uses: edgetest-dev/run-edgetest-action@v1.4
+
+ - name: Set up Python 3.10
+ uses: conda-incubator/setup-miniconda@v2
with:
- edgetest-flags: '-c pyproject.toml --export'
- base-branch: 'develop'
- skip-pr: 'false'
\ No newline at end of file
+ auto-update-conda: true
+ python-version: '3.10'
+ channels: conda-forge
+
+ - name: Setup Java JDK
+ uses: actions/setup-java@v3
+ with:
+ java-version: '8'
+ distribution: 'adopt'
+
+ - name: Install edgetest
+ shell: bash -el {0}
+ run: |
+ conda install pip
+ conda install edgetest edgetest-conda
+ python -m pip install .[dev]
+
+ - name: Run edgetest
+ shell: bash -el {0}
+ run: |
+ edgetest -c pyproject.toml --export
+
+ - name: Create Pull Request
+ uses: peter-evans/create-pull-request@v3
+ with:
+ branch: edgetest-patch
+ base: develop
+ delete-branch: true
+ title: Changes by run-edgetest action
+ commit-message: '[edgetest] automated change'
+ body: Automated changes by [run-edgetest-action](https://github.com/edgetest-dev/run-edgetest-action) GitHub action
+ add-paths: |
+ requirements.txt
+ setup.cfg
+ pyproject.toml
\ No newline at end of file
diff --git a/pyproject.toml b/pyproject.toml
index 6cb436da..dcf8d5c9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -133,7 +133,6 @@ ignore_missing_imports = true
python_version = "3.10"
conda_install = ["openjdk=8"]
extras = ["dev"]
-deps = ["pyspark==3.4.2"]
command = "pytest tests -m 'not integration'"
upgrade = [
"pandas",
diff --git a/pytest.ini b/pytest.ini
index 3036b87b..c780335c 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -6,5 +6,4 @@ spark_options =
spark.default.parallelism: 4
spark.executor.cores: 4
spark.sql.execution.arrow.pyspark.enabled: true
- spark.sql.execution.arrow.enabled: false
spark.sql.adaptive.enabled: false
\ No newline at end of file