Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use date instead of datetime #85

Merged
merged 3 commits into from
Feb 22, 2024
Merged

Use date instead of datetime #85

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9230d36
Use date instead of datetime
stinodego Feb 22, 2024
88e9ca4
Refactor data prep
stinodego Feb 22, 2024
2983174
Fix dask
stinodego Feb 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 1 addition & 5 deletions queries/common_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,10 @@
import sys
from pathlib import Path
from subprocess import run
from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar
from typing import Any

from linetimer import CodeTimer

if TYPE_CHECKING:
P = ParamSpec("P")
T = TypeVar("T")

INCLUDE_IO = bool(os.environ.get("INCLUDE_IO", False))
SHOW_RESULTS = bool(os.environ.get("SHOW_RESULTS", False))
LOG_TIMINGS = bool(os.environ.get("LOG_TIMINGS", False))
Expand Down
4 changes: 2 additions & 2 deletions queries/dask/q1.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from __future__ import annotations

from datetime import datetime
from datetime import date
from typing import TYPE_CHECKING

from queries.dask import utils
Expand All @@ -12,7 +12,7 @@


def q() -> None:
VAR1 = datetime(1998, 9, 2)
VAR1 = date(1998, 9, 2)

lineitem = utils.get_line_item_ds
# first call one time to cache in case we don't include the IO times
Expand Down
4 changes: 2 additions & 2 deletions queries/dask/q3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from __future__ import annotations

from datetime import datetime
from datetime import date
from typing import TYPE_CHECKING

from queries.dask import utils
Expand All @@ -12,7 +12,7 @@


def q() -> None:
var1 = datetime(1995, 3, 15)
var1 = date(1995, 3, 15)
var2 = "BUILDING"

line_item_ds = utils.get_line_item_ds
Expand Down
6 changes: 3 additions & 3 deletions queries/dask/q4.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from __future__ import annotations

from datetime import datetime
from datetime import date
from typing import TYPE_CHECKING

from queries.dask import utils
Expand All @@ -12,8 +12,8 @@


def q() -> None:
date1 = datetime(1993, 10, 1)
date2 = datetime(1993, 7, 1)
date1 = date(1993, 10, 1)
date2 = date(1993, 7, 1)

line_item_ds = utils.get_line_item_ds
orders_ds = utils.get_orders_ds
Expand Down
6 changes: 3 additions & 3 deletions queries/dask/q5.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from __future__ import annotations

from datetime import datetime
from datetime import date
from typing import TYPE_CHECKING

from queries.dask import utils
Expand All @@ -12,8 +12,8 @@


def q() -> None:
date1 = datetime(1994, 1, 1)
date2 = datetime(1995, 1, 1)
date1 = date(1994, 1, 1)
date2 = date(1995, 1, 1)

region_ds = utils.get_region_ds
nation_ds = utils.get_nation_ds
Expand Down
6 changes: 3 additions & 3 deletions queries/dask/q6.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from __future__ import annotations

from datetime import datetime
from datetime import date

import pandas as pd

Expand All @@ -10,8 +10,8 @@


def q() -> None:
date1 = datetime(1994, 1, 1)
date2 = datetime(1995, 1, 1)
date1 = date(1994, 1, 1)
date2 = date(1995, 1, 1)
var3 = 24

line_item_ds = utils.get_line_item_ds
Expand Down
6 changes: 3 additions & 3 deletions queries/dask/q7.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from __future__ import annotations

import warnings
from datetime import datetime
from datetime import date
from typing import TYPE_CHECKING

from queries.dask import utils
Expand All @@ -17,8 +17,8 @@


def q() -> None:
var1 = datetime(1995, 1, 1)
var2 = datetime(1997, 1, 1)
var1 = date(1995, 1, 1)
var2 = date(1997, 1, 1)

nation_ds = utils.get_nation_ds
customer_ds = utils.get_customer_ds
Expand Down
8 changes: 5 additions & 3 deletions queries/dask/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
from __future__ import annotations

import os
import timeit
from typing import TYPE_CHECKING, Any

Expand Down Expand Up @@ -29,12 +28,15 @@

def read_ds(path: Path) -> DataFrame:
if INCLUDE_IO:
return dd.read_parquet(path) # type: ignore[attr-defined,no-any-return]
return dd.read_parquet(path, dtype_backend="pyarrow") # type: ignore[attr-defined,no-any-return]
if FILE_TYPE == "feather":
msg = "file type feather not supported for dask queries"
raise ValueError(msg)

return dd.from_pandas(pd.read_parquet(path), npartitions=os.cpu_count()) # type: ignore[attr-defined,no-any-return]
# TODO: Load into memory before returning the Dask DataFrame.
# Code below is tripped up by date types and pyarrow backend is not yet supported
# return dd.from_pandas(pd.read_parquet(path), npartitions=os.cpu_count())
return dd.read_parquet(path, dtype_backend="pyarrow") # type: ignore[attr-defined,no-any-return]


def get_query_answer(query: int, base_dir: Path = ANSWERS_BASE_DIR) -> pd.DataFrame:
Expand Down
4 changes: 2 additions & 2 deletions queries/modin/q1.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from __future__ import annotations

from datetime import datetime
from datetime import date
from typing import TYPE_CHECKING

from queries.modin import utils
Expand All @@ -12,7 +12,7 @@


def q() -> None:
VAR1 = datetime(1998, 9, 2)
VAR1 = date(1998, 9, 2)

lineitem = utils.get_line_item_ds
# first call one time to cache in case we don't include the IO times
Expand Down
4 changes: 2 additions & 2 deletions queries/modin/q3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from __future__ import annotations

from datetime import datetime
from datetime import date
from typing import TYPE_CHECKING

from queries.modin import utils
Expand All @@ -12,7 +12,7 @@


def q() -> None:
var1 = var2 = datetime(1995, 3, 15)
var1 = var2 = date(1995, 3, 15)
var3 = "BUILDING"

customer_ds = utils.get_customer_ds
Expand Down
6 changes: 3 additions & 3 deletions queries/modin/q4.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from __future__ import annotations

from datetime import datetime
from datetime import date
from typing import TYPE_CHECKING

from queries.modin import utils
Expand All @@ -12,8 +12,8 @@


def q() -> None:
date1 = datetime(1993, 10, 1)
date2 = datetime(1993, 7, 1)
date1 = date(1993, 10, 1)
date2 = date(1993, 7, 1)

line_item_ds = utils.get_line_item_ds
orders_ds = utils.get_orders_ds
Expand Down
6 changes: 3 additions & 3 deletions queries/modin/q5.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from __future__ import annotations

from datetime import datetime
from datetime import date
from typing import TYPE_CHECKING

from queries.modin import utils
Expand All @@ -12,8 +12,8 @@


def q() -> None:
date1 = datetime(1994, 1, 1)
date2 = datetime(1995, 1, 1)
date1 = date(1994, 1, 1)
date2 = date(1995, 1, 1)

region_ds = utils.get_region_ds
nation_ds = utils.get_nation_ds
Expand Down
6 changes: 3 additions & 3 deletions queries/modin/q6.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from __future__ import annotations

from datetime import datetime
from datetime import date

import modin.pandas as pd

Expand All @@ -10,8 +10,8 @@


def q() -> None:
date1 = datetime(1994, 1, 1)
date2 = datetime(1995, 1, 1)
date1 = date(1994, 1, 1)
date2 = date(1995, 1, 1)
var3 = 24

line_item_ds = utils.get_line_item_ds
Expand Down
6 changes: 3 additions & 3 deletions queries/modin/q7.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from __future__ import annotations

from datetime import datetime
from datetime import date

import modin.pandas as pd

Expand All @@ -10,8 +10,8 @@


def q() -> None:
var1 = datetime(1995, 1, 1)
var2 = datetime(1997, 1, 1)
var1 = date(1995, 1, 1)
var2 = date(1997, 1, 1)

nation_ds = utils.get_nation_ds
customer_ds = utils.get_customer_ds
Expand Down
4 changes: 2 additions & 2 deletions queries/pandas/q1.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from datetime import datetime
from datetime import date

import pandas as pd

Expand All @@ -8,7 +8,7 @@


def q() -> None:
VAR1 = datetime(1998, 9, 2)
VAR1 = date(1998, 9, 2)

lineitem = utils.get_line_item_ds
# first call one time to cache in case we don't include the IO times
Expand Down
4 changes: 2 additions & 2 deletions queries/pandas/q3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from datetime import datetime
from datetime import date

import pandas as pd

Expand All @@ -8,7 +8,7 @@


def q() -> None:
var1 = var2 = datetime(1995, 3, 15)
var1 = var2 = date(1995, 3, 15)
var3 = "BUILDING"

customer_ds = utils.get_customer_ds
Expand Down
6 changes: 3 additions & 3 deletions queries/pandas/q4.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from datetime import datetime
from datetime import date

import pandas as pd

Expand All @@ -8,8 +8,8 @@


def q() -> None:
date1 = datetime(1993, 10, 1)
date2 = datetime(1993, 7, 1)
date1 = date(1993, 10, 1)
date2 = date(1993, 7, 1)

line_item_ds = utils.get_line_item_ds
orders_ds = utils.get_orders_ds
Expand Down
6 changes: 3 additions & 3 deletions queries/pandas/q5.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from datetime import datetime
from datetime import date

import pandas as pd

Expand All @@ -8,8 +8,8 @@


def q() -> None:
date1 = datetime(1994, 1, 1)
date2 = datetime(1995, 1, 1)
date1 = date(1994, 1, 1)
date2 = date(1995, 1, 1)

region_ds = utils.get_region_ds
nation_ds = utils.get_nation_ds
Expand Down
6 changes: 3 additions & 3 deletions queries/pandas/q6.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from datetime import datetime
from datetime import date

import pandas as pd

Expand All @@ -8,8 +8,8 @@


def q() -> None:
date1 = datetime(1994, 1, 1)
date2 = datetime(1995, 1, 1)
date1 = date(1994, 1, 1)
date2 = date(1995, 1, 1)
var3 = 24

line_item_ds = utils.get_line_item_ds
Expand Down
6 changes: 3 additions & 3 deletions queries/pandas/q7.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from datetime import datetime
from datetime import date

import pandas as pd

Expand Down Expand Up @@ -35,8 +35,8 @@ def query() -> pd.DataFrame:
supplier_ds = supplier_ds()

lineitem_filtered = line_item_ds[
(line_item_ds["l_shipdate"] >= datetime(1995, 1, 1))
& (line_item_ds["l_shipdate"] < datetime(1997, 1, 1))
(line_item_ds["l_shipdate"] >= date(1995, 1, 1))
& (line_item_ds["l_shipdate"] < date(1997, 1, 1))
]
lineitem_filtered["l_year"] = lineitem_filtered["l_shipdate"].dt.year
lineitem_filtered["revenue"] = lineitem_filtered["l_extendedprice"] * (
Expand Down
4 changes: 0 additions & 4 deletions queries/pandas/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,6 @@ def test_results(q_num: int, result_df: PandasDF) -> None:
if is_string_dtype(t):
s1 = s1.apply(lambda x: x.strip())

# TODO: Remove this cast
if s2.dtype == "date32[day][pyarrow]":
s2 = s2.astype("timestamp[us][pyarrow]")

assert_series_equal(left=s1, right=s2, check_index=False, check_dtype=False)


Expand Down
4 changes: 2 additions & 2 deletions queries/polars/q1.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from datetime import datetime
from datetime import date

import polars as pl

Expand All @@ -8,7 +8,7 @@


def q() -> None:
var_1 = datetime(1998, 9, 2)
var_1 = date(1998, 9, 2)
q = utils.get_line_item_ds()
q_final = (
q.filter(pl.col("l_shipdate") <= var_1)
Expand Down
Loading