Skip to content

Commit

Permalink
Refactor feature building (#22)
Browse files Browse the repository at this point in the history 
Co-authored-by: mikivee <mikivee>
Co-authored-by: Darren Vengroff <[email protected]>
  • Loading branch information
@mikivee @vengroff
mikivee and vengroff authored Nov 20, 2024
1 parent 4acf8f1 commit 101e337
Show file tree
Hide file tree
Showing 22 changed files with 2,162 additions and 1,545 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/lint.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,9 @@ jobs:
#----------------------------------------------
# run black
#----------------------------------------------
- run: python -m pip install black[jupyter]
- run: python -m pip install black==22.12.0
- run: |
black --version
black ./src ./tests --diff
black ./src ./tests --check
14 changes: 8 additions & 6 deletions install-db-requirements.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,16 @@ cython==0.29.32 ; python_version >= "3.10" and python_version < "3.11"
distro==1.9.0 ; python_version >= "3.10" and python_version < "3.11"
dmlbootstrap==0.2.0 ; python_version >= "3.10" and python_version < "3.11"
dmlutils==0.8.0 ; python_version >= "3.10" and python_version < "3.11"
duckdb==1.1.1 ; python_version >= "3.10" and python_version < "3.11"
et-xmlfile==1.1.0 ; python_version >= "3.10" and python_version < "3.11"
duckdb==1.1.3 ; python_version >= "3.10" and python_version < "3.11"
et-xmlfile==2.0.0 ; python_version >= "3.10" and python_version < "3.11"
flask==2.2.5 ; python_version >= "3.10" and python_version < "3.11"
fsspec[http]==2023.6.0 ; python_version >= "3.10" and python_version < "3.11"
gcsfs==2023.6.0 ; python_version >= "3.10" and python_version < "3.11"
gitpython==3.1.27 ; python_version >= "3.10" and python_version < "3.11"
google-api-core[grpc]==2.15.0 ; python_version >= "3.10" and python_version < "3.11"
google-cloud-secret-manager==2.20.2 ; python_version >= "3.10" and python_version < "3.11"
google-cloud-bigquery-storage==2.27.0 ; python_version >= "3.10" and python_version < "3.11"
google-cloud-bigquery==3.27.0 ; python_version >= "3.10" and python_version < "3.11"
google-cloud-secret-manager==2.21.1 ; python_version >= "3.10" and python_version < "3.11"
googleapis-common-protos[grpc]==1.62.0 ; python_version >= "3.10" and python_version < "3.11"
greenlet==3.1.1 ; python_version >= "3.10" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and python_version < "3.11"
grpc-google-iam-v1==0.13.1 ; python_version >= "3.10" and python_version < "3.11"
Expand All @@ -43,7 +45,7 @@ mlflow-skinny[databricks]==2.9.2 ; python_version >= "3.10" and python_version <
notebook-shim==0.2.2 ; python_version >= "3.10" and python_version < "3.11"
openpyxl==3.1.5 ; python_version >= "3.10" and python_version < "3.11"
pillow==9.4.0 ; python_version >= "3.10" and python_version < "3.11"
proto-plus==1.24.0 ; python_version >= "3.10" and python_version < "3.11"
proto-plus==1.25.0 ; python_version >= "3.10" and python_version < "3.11"
py4j==0.10.9.7 ; python_version >= "3.10" and python_version < "3.11"
py==1.11.0 ; python_version >= "3.10" and python_version < "3.11" and implementation_name == "pypy"
pygments==2.11.2 ; python_version >= "3.10" and python_version < "3.11"
Expand All @@ -53,8 +55,8 @@ pyspark==3.5.3 ; python_version >= "3.10" and python_version < "3.11"
pytoolconfig[global]==1.2.5 ; python_version >= "3.10" and python_version < "3.11"
pywavelets==1.4.1 ; python_version >= "3.10" and python_version < "3.11"
pywin32-ctypes==0.2.3 ; python_version >= "3.10" and python_version < "3.11" and sys_platform == "win32"
pywin32==307 ; sys_platform == "win32" and platform_python_implementation != "PyPy" and python_version >= "3.10" and python_version < "3.11"
pywinpty==2.0.13 ; python_version >= "3.10" and python_version < "3.11" and os_name == "nt"
pywin32==308 ; sys_platform == "win32" and platform_python_implementation != "PyPy" and python_version >= "3.10" and python_version < "3.11"
pywinpty==2.0.14 ; python_version >= "3.10" and python_version < "3.11" and os_name == "nt"
pyyaml==6.0 ; python_version >= "3.10" and python_version < "3.11"
radbcluster==14.3.1 ; python_version >= "3.10" and python_version < "3.11"
secretstorage==3.3.1 ; python_version >= "3.10" and python_version < "3.11"
Expand Down
371 changes: 207 additions & 164 deletions poetry.lock
View file

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@ priority = "primary"
[tool.poetry.dependencies]
python = ">=3.10, <3.11"

google-cloud-bigquery = "^3.26"
google-cloud-bigquery-storage = "^2.26.0"

# These are RA projects in an internal repo.
dmlutils = "^0.8.0"
dmlbootstrap = "^0.2.0"
Expand Down
14 changes: 8 additions & 6 deletions requirements-db-14.3.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,16 @@ cython==0.29.32 ; python_version >= "3.10" and python_version < "3.11"
distro==1.9.0 ; python_version >= "3.10" and python_version < "3.11"
dmlbootstrap==0.2.0 ; python_version >= "3.10" and python_version < "3.11"
dmlutils==0.8.0 ; python_version >= "3.10" and python_version < "3.11"
duckdb==1.1.1 ; python_version >= "3.10" and python_version < "3.11"
et-xmlfile==1.1.0 ; python_version >= "3.10" and python_version < "3.11"
duckdb==1.1.3 ; python_version >= "3.10" and python_version < "3.11"
et-xmlfile==2.0.0 ; python_version >= "3.10" and python_version < "3.11"
flask==2.2.5 ; python_version >= "3.10" and python_version < "3.11"
fsspec[http]==2023.6.0 ; python_version >= "3.10" and python_version < "3.11"
gcsfs==2023.6.0 ; python_version >= "3.10" and python_version < "3.11"
gitpython==3.1.27 ; python_version >= "3.10" and python_version < "3.11"
google-api-core[grpc]==2.15.0 ; python_version >= "3.10" and python_version < "3.11"
google-cloud-secret-manager==2.20.2 ; python_version >= "3.10" and python_version < "3.11"
google-cloud-bigquery-storage==2.27.0 ; python_version >= "3.10" and python_version < "3.11"
google-cloud-bigquery==3.27.0 ; python_version >= "3.10" and python_version < "3.11"
google-cloud-secret-manager==2.21.1 ; python_version >= "3.10" and python_version < "3.11"
googleapis-common-protos[grpc]==1.62.0 ; python_version >= "3.10" and python_version < "3.11"
greenlet==3.1.1 ; python_version >= "3.10" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and python_version < "3.11"
grpc-google-iam-v1==0.13.1 ; python_version >= "3.10" and python_version < "3.11"
Expand All @@ -31,7 +33,7 @@ mlflow-skinny[databricks]==2.9.2 ; python_version >= "3.10" and python_version <
notebook-shim==0.2.2 ; python_version >= "3.10" and python_version < "3.11"
openpyxl==3.1.5 ; python_version >= "3.10" and python_version < "3.11"
pillow==9.4.0 ; python_version >= "3.10" and python_version < "3.11"
proto-plus==1.24.0 ; python_version >= "3.10" and python_version < "3.11"
proto-plus==1.25.0 ; python_version >= "3.10" and python_version < "3.11"
py4j==0.10.9.7 ; python_version >= "3.10" and python_version < "3.11"
py==1.11.0 ; python_version >= "3.10" and python_version < "3.11" and implementation_name == "pypy"
pygments==2.11.2 ; python_version >= "3.10" and python_version < "3.11"
Expand All @@ -41,8 +43,8 @@ pyspark==3.5.3 ; python_version >= "3.10" and python_version < "3.11"
pytoolconfig[global]==1.2.5 ; python_version >= "3.10" and python_version < "3.11"
pywavelets==1.4.1 ; python_version >= "3.10" and python_version < "3.11"
pywin32-ctypes==0.2.3 ; python_version >= "3.10" and python_version < "3.11" and sys_platform == "win32"
pywin32==307 ; sys_platform == "win32" and platform_python_implementation != "PyPy" and python_version >= "3.10" and python_version < "3.11"
pywinpty==2.0.13 ; python_version >= "3.10" and python_version < "3.11" and os_name == "nt"
pywin32==308 ; sys_platform == "win32" and platform_python_implementation != "PyPy" and python_version >= "3.10" and python_version < "3.11"
pywinpty==2.0.14 ; python_version >= "3.10" and python_version < "3.11" and os_name == "nt"
pyyaml==6.0 ; python_version >= "3.10" and python_version < "3.11"
radbcluster==14.3.1 ; python_version >= "3.10" and python_version < "3.11"
secretstorage==3.3.1 ; python_version >= "3.10" and python_version < "3.11"
Expand Down
16 changes: 9 additions & 7 deletions requirements-test-14.3.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ comm==0.1.2 ; python_version >= "3.10" and python_version < "3.11"
confection==0.1.4 ; python_version >= "3.10" and python_version < "3.11"
configparser==5.2.0 ; python_version >= "3.10" and python_version < "3.11"
contourpy==1.0.5 ; python_version >= "3.10" and python_version < "3.11"
coverage[toml]==7.6.1 ; python_version >= "3.10" and python_version < "3.11"
coverage[toml]==7.6.7 ; python_version >= "3.10" and python_version < "3.11"
cryptography==39.0.1 ; python_version >= "3.10" and python_version < "3.11"
cycler==0.11.0 ; python_version >= "3.10" and python_version < "3.11"
cymem==2.0.8 ; python_version >= "3.10" and python_version < "3.11"
Expand All @@ -68,9 +68,9 @@ distro==1.9.0 ; python_version >= "3.10" and python_version < "3.11"
dmlbootstrap==0.2.0 ; python_version >= "3.10" and python_version < "3.11"
dmlutils==0.8.0 ; python_version >= "3.10" and python_version < "3.11"
docstring-to-markdown==0.11 ; python_version >= "3.10" and python_version < "3.11"
duckdb==1.1.1 ; python_version >= "3.10" and python_version < "3.11"
duckdb==1.1.3 ; python_version >= "3.10" and python_version < "3.11"
entrypoints==0.4 ; python_version >= "3.10" and python_version < "3.11"
et-xmlfile==1.1.0 ; python_version >= "3.10" and python_version < "3.11"
et-xmlfile==2.0.0 ; python_version >= "3.10" and python_version < "3.11"
evaluate==0.4.1 ; python_version >= "3.10" and python_version < "3.11"
exceptiongroup==1.2.2 ; python_version >= "3.10" and python_version < "3.11"
executing==0.8.3 ; python_version >= "3.10" and python_version < "3.11"
Expand All @@ -92,8 +92,10 @@ google-api-core==2.15.0 ; python_version >= "3.10" and python_version < "3.11"
google-api-core[grpc]==2.15.0 ; python_version >= "3.10" and python_version < "3.11"
google-auth-oauthlib==1.0.0 ; python_version >= "3.10" and python_version < "3.11"
google-auth==2.21.0 ; python_version >= "3.10" and python_version < "3.11"
google-cloud-bigquery-storage==2.27.0 ; python_version >= "3.10" and python_version < "3.11"
google-cloud-bigquery==3.27.0 ; python_version >= "3.10" and python_version < "3.11"
google-cloud-core==2.4.1 ; python_version >= "3.10" and python_version < "3.11"
google-cloud-secret-manager==2.20.2 ; python_version >= "3.10" and python_version < "3.11"
google-cloud-secret-manager==2.21.1 ; python_version >= "3.10" and python_version < "3.11"
google-cloud-storage==2.11.0 ; python_version >= "3.10" and python_version < "3.11"
google-crc32c==1.5.0 ; python_version >= "3.10" and python_version < "3.11"
google-pasta==0.2.0 ; python_version >= "3.10" and python_version < "3.11"
Expand Down Expand Up @@ -213,7 +215,7 @@ preshed==3.0.9 ; python_version >= "3.10" and python_version < "3.11"
prometheus-client==0.14.1 ; python_version >= "3.10" and python_version < "3.11"
prompt-toolkit==3.0.36 ; python_version >= "3.10" and python_version < "3.11"
prophet==1.1.5 ; python_version >= "3.10" and python_version < "3.11"
proto-plus==1.24.0 ; python_version >= "3.10" and python_version < "3.11"
proto-plus==1.25.0 ; python_version >= "3.10" and python_version < "3.11"
protobuf==4.24.0 ; python_version >= "3.10" and python_version < "3.11"
psutil==5.9.0 ; python_version >= "3.10" and python_version < "3.11"
ptyprocess==0.7.0 ; python_version >= "3.10" and python_version < "3.11"
Expand Down Expand Up @@ -249,8 +251,8 @@ pytoolconfig[global]==1.2.5 ; python_version >= "3.10" and python_version < "3.1
pytz==2022.7 ; python_version >= "3.10" and python_version < "3.11"
pywavelets==1.4.1 ; python_version >= "3.10" and python_version < "3.11"
pywin32-ctypes==0.2.3 ; python_version >= "3.10" and python_version < "3.11" and sys_platform == "win32"
pywin32==307 ; sys_platform == "win32" and platform_python_implementation != "PyPy" and python_version >= "3.10" and python_version < "3.11"
pywinpty==2.0.13 ; python_version >= "3.10" and python_version < "3.11" and os_name == "nt"
pywin32==308 ; sys_platform == "win32" and platform_python_implementation != "PyPy" and python_version >= "3.10" and python_version < "3.11"
pywinpty==2.0.14 ; python_version >= "3.10" and python_version < "3.11" and os_name == "nt"
pyyaml==6.0 ; python_version >= "3.10" and python_version < "3.11"
pyzmq==23.2.0 ; python_version >= "3.10" and python_version < "3.11"
radbcluster==14.3.1 ; python_version >= "3.10" and python_version < "3.11"
Expand Down
Loading

0 comments on commit 101e337

Please sign in to comment.