Pinned pyspark to match emr on eks base image. (#43)

sbyrne-sf · web-flow · commit 7451fbae6abc · 2025-09-16T17:36:02.000-04:00
This is needed for compatibility.
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -100,7 +100,7 @@ loguru = "^0.7.3"
 numpy = "*"
 pandas = "*"
 pydantic = "^1.8.2 || ^2.0.0"
-pyspark = "*"
+pyspark = "3.5.1"
 python = ">=3.10,<3.12"
 pyyaml = "^6.0"
 salesforce-cdp-connector = "*"
diff --git a/src/datacustomcode/templates/requirements-dev.txt b/src/datacustomcode/templates/requirements-dev.txt
@@ -1,6 +1,6 @@
 # Required packages for the project - Do not modify
 salesforce-cdp-connector>=1.0.16
-pyspark
+pyspark==3.5.1
 pandas
 numpy
 pydantic
