From 1a9e2dcd9dfd684aee1686330667dd63efba80ea Mon Sep 17 00:00:00 2001
From: Faisal <faisal.dosani@capitalone.com>
Date: Tue, 20 Feb 2024 17:03:08 -0400
Subject: [PATCH] Edgetest fix (#266)

* updating edgetest config

* adding manual trigger for edgetest

* adding in pyspark edgetest config
---
 .github/workflows/edgetest.yml | 1 +
 README.md                      | 2 +-
 pyproject.toml                 | 3 ++-
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/edgetest.yml b/.github/workflows/edgetest.yml
index b53cf46c..7e0a5524 100644
--- a/.github/workflows/edgetest.yml
+++ b/.github/workflows/edgetest.yml
@@ -5,6 +5,7 @@ name: Run edgetest
 on:
   schedule:
     - cron: '30 17 * * 5'
+  workflow_dispatch:  # allows manual dispatch
 jobs:
   edgetest:
     runs-on: ubuntu-latest
diff --git a/README.md b/README.md
index 4dd0c869..fefe447b 100644
--- a/README.md
+++ b/README.md
@@ -41,7 +41,7 @@ pip install datacompy[ray]
 ### In-scope Spark versions
 Different versions of Spark play nicely with only certain versions of Python below is a matrix of what we test with
 
-|             | Spark 3.1.3  | Spark 3.2.3 | Spark 3.3.4 | Spark 3.4.4 | Spark 3.5.0 |
+|             | Spark 3.1.3  | Spark 3.2.3 | Spark 3.3.4 | Spark 3.4.2 | Spark 3.5.0 |
 |-------------|--------------|-------------|-------------|-------------|-------------|
 | Python 3.8  | ✅           | ✅           | ✅           | ✅          | ✅          |
 | Python 3.9  | ✅           | ✅           | ✅           | ✅          | ✅          |
diff --git a/pyproject.toml b/pyproject.toml
index 602cea8d..5b2eee04 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -130,9 +130,10 @@ module = "pyarrow"
 ignore_missing_imports = true
 
 [edgetest.envs.core]
-python_version = "3.9"
+python_version = "3.10"
 conda_install = ["openjdk=8"]
 extras = ["dev"]
+deps = ["pyspark==3.4.2"]
 command = "pytest tests -m 'not integration'"
 upgrade = [
     "pandas",