Skip to content

Commit

Permalink
Create new branch before transaction
Browse files Browse the repository at this point in the history
Previously, the new branch was a result of implicit branch creation in
`fs.put_file()` _within_ the context.

Now, since we do not allow the base branch to be created in situ, which
would be very confusing, we simply create it explicitly before the
transaction start.

We do this for both NEW_BRANCH (not sure what it is used for in the end)
and TRAINING_BRANCH.
  • Loading branch information
nicholasjng committed Jan 26, 2024
1 parent b103777 commit fcbe093
Showing 1 changed file with 6 additions and 4 deletions.
10 changes: 6 additions & 4 deletions docs/tutorials/demo_data_science_project.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,9 +159,10 @@ def _maybe_urlretrieve(url: str, filename: str) -> str:
"""

# %%
NEW_BRANCH_NAME = "transform-raw-data"
NEW_BRANCH = lakefs.Branch(REPO_NAME, "transform-raw-data", client=fs.client)
NEW_BRANCH.create("main")

with fs.transaction(REPO_NAME, NEW_BRANCH_NAME) as tx:
with fs.transaction(REPO_NAME, NEW_BRANCH) as tx:
fs.put(outfile, f"{REPO_NAME}/{tx.branch.id}/weather-2010.json")
tx.commit(message="Add 2010 weather data")

Expand Down Expand Up @@ -241,7 +242,8 @@ def transform_json_weather_data(filepath):
"""

# %%
TRAINING_BRANCH = "training"
TRAINING_BRANCH = lakefs.Branch(REPO_NAME, "training", client=fs.client)
TRAINING_BRANCH.create("main")

with fs.transaction(REPO_NAME, TRAINING_BRANCH) as tx:
train.to_csv(f"lakefs://{REPO_NAME}/{tx.branch.id}/train_weather.csv")
Expand Down Expand Up @@ -356,7 +358,7 @@ def transform_json_weather_data(filepath):
# %%

# access the data of the previous commit with a lakefs ref expression, in this case the same as in git.
previous_commit = repo.ref(f"{TRAINING_BRANCH}~").get_commit()
previous_commit = repo.ref(f"{TRAINING_BRANCH.id}~").get_commit()
fixed_commit_id = previous_commit.id
print(fixed_commit_id)

Expand Down

0 comments on commit fcbe093

Please sign in to comment.