Skip to content

Commit

Permalink
Add logs for when data-fetching actually starts
Browse files Browse the repository at this point in the history
Summary: Current logs are confusing, they make it look like we first fetch the data quite fast, then we retrieve completed trials, and then we do nothing for a while. This clarifies by rephrasing the "Fetching..." log to "will fetch" and logging the actual start of fetching with "..." at the end, indicating that it will take a bit.

Differential Revision: D64208425
  • Loading branch information
Lena Kashtelyan authored and facebook-github-bot committed Oct 11, 2024
1 parent 81567b7 commit 8d3e95f
Showing 1 changed file with 5 additions and 3 deletions.
8 changes: 5 additions & 3 deletions ax/service/scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -1335,6 +1335,8 @@ def poll_and_process_results(self, poll_all_trial_statuses: bool = False) -> boo
)

# FETCH DATA FOR TRIALS EXPECTING DATA
idcs_str = make_indices_str(indices=trial_indices_to_fetch)
self.logger.info(f"Starting data-fetching for trials {idcs_str}...")
trial_indices_with_new_data = (
self._fetch_data_and_return_trial_indices_with_new_data(
trial_idcs=trial_indices_to_fetch,
Expand Down Expand Up @@ -1483,10 +1485,10 @@ def _get_trial_indices_to_fetch(
)
idcs = make_indices_str(indices=newly_completed)
if newly_completed:
self.logger.info(f"Fetching data for newly completed trials: {idcs}.")
self.logger.info(f"Will fetch data for newly completed trials: {idcs}.")
trial_indices_to_fetch.update(newly_completed)
else:
self.logger.info("No newly completed trials; not fetching data for any.")
self.logger.info("No newly completed trials; will not fetch data for any.")

# Fetch data for running trials that have metrics available while running
if (
Expand All @@ -1499,7 +1501,7 @@ def _get_trial_indices_to_fetch(
# in fetch_trials_data
idcs = make_indices_str(indices=running_trial_indices)
self.logger.info(
f"Fetching data for trials: {idcs} because some metrics "
f"Will fetch data for trials: {idcs} because some metrics "
"on experiment are available while trials are running."
)
trial_indices_to_fetch.update(running_trial_indices)
Expand Down

0 comments on commit 8d3e95f

Please sign in to comment.