Skip to content

Commit

Permalink
resolved execution errors in gcs_query.py
Browse files Browse the repository at this point in the history
  • Loading branch information
naishasinha committed Jun 13, 2024
1 parent 5f31035 commit 88e1510
Show file tree
Hide file tree
Showing 6 changed files with 23 additions and 9 deletions.
2 changes: 1 addition & 1 deletion Pipfile
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,4 @@ pre-commit = "*"
python_version = "3.11"

[scripts]
gcs_query = "python ./scripts/1-queries/gcs_query.py"
gcs_query = "./scripts/1-fetched/gcs_query.py"
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ line_length = 79
multi_line_output = 3
no_lines_before = 'LOCALFOLDER'
profile = 'black'
src_paths = ['pre-automation']
src_paths = ['pre-automation', 'scripts']


# [tool.pre-commit]
Expand Down
15 changes: 9 additions & 6 deletions scripts/1-fetched/gcs_query.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@
import json
import os
import sys
import time

# import time
import traceback
from typing import List

Expand All @@ -19,10 +20,10 @@

# Setup paths and LOGGER using shared library
# sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'shared'))
# sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..'))
sys.path.append(".")
sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
# sys.path.append(".")

# Third-party
# First-party/Local
import shared # noqa: E402

(
Expand Down Expand Up @@ -134,11 +135,13 @@ def main():
start_index = state["start_index"]
all_results, start_index = fetch_results(args, start_index)

LOGGER.info(f"PATH_REPO_ROOT: {PATH_REPO_ROOT}")

# Create new directory structure for year and quarter
year = time.strftime("%Y")
quarter = pd.PeriodIndex([TODAY], freq="Q")[0]
quarter_str = str(quarter)
data_directory = os.path.join(
PATH_REPO_ROOT, "data", "1-fetched", f"{year}Q{quarter}"
PATH_REPO_ROOT, "data", "1-fetched", quarter_str
)
os.makedirs(data_directory, exist_ok=True)

Expand Down
1 change: 1 addition & 0 deletions scripts/1-fetched/state.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"start_index": 11}
1 change: 0 additions & 1 deletion scripts/shared/.placeholder

This file was deleted.

Loading

0 comments on commit 88e1510

Please sign in to comment.