diff --git a/docs/example_scripts/acme_track.py b/docs/example_scripts/acme_track.py index 70c13c1b75..347569959c 100644 --- a/docs/example_scripts/acme_track.py +++ b/docs/example_scripts/acme_track.py @@ -9,7 +9,7 @@ from acme.jax import experiments from acme.utils import loggers -from aim.sdk.acme import AimCallback, AimWriter +from aimstack.ml.adapters.acme import AimCallback, AimWriter def make_environment(seed: int) -> dm_env.Environment: diff --git a/docs/example_scripts/catboost_track.py b/docs/example_scripts/catboost_track.py index cade69fde4..bb46263e3e 100644 --- a/docs/example_scripts/catboost_track.py +++ b/docs/example_scripts/catboost_track.py @@ -5,7 +5,7 @@ import numpy as np -from aim.catboost import AimLogger +from aimstack.ml.adapters.catboost import AimLogger from catboost import CatBoostClassifier, Pool # initialize data diff --git a/docs/example_scripts/fastai_track.py b/docs/example_scripts/fastai_track.py index 14c470dc57..79bea6d1e6 100644 --- a/docs/example_scripts/fastai_track.py +++ b/docs/example_scripts/fastai_track.py @@ -7,7 +7,7 @@ from fastai.vision.all import get_image_files, aug_transforms, imagenet_stats from fastai.vision.all import cnn_learner, resnet18, CrossEntropyLossFlat import regex as re -from aim.fastai import AimCallback +from aimstack.ml.adapters.fastai import AimCallback def get_arabic_mnist_labels(file_path): diff --git a/docs/example_scripts/hugging_face_track.py b/docs/example_scripts/hugging_face_track.py index 2437a8dc1b..97a4c8fcd1 100644 --- a/docs/example_scripts/hugging_face_track.py +++ b/docs/example_scripts/hugging_face_track.py @@ -26,7 +26,7 @@ import numpy as np from datasets import load_dataset, load_metric -from aim.hugging_face import AimCallback +from aimstack.ml.adapters.hugging_face import AimCallback import transformers from transformers import ( diff --git a/docs/example_scripts/keras_track.py b/docs/example_scripts/keras_track.py index 61e651f46b..9986a13fb7 100644 --- a/docs/example_scripts/keras_track.py +++ b/docs/example_scripts/keras_track.py @@ -1,4 +1,4 @@ -from aim.keras import AimCallback +from aimstack.ml.adapters.keras import AimCallback import keras from keras.datasets import mnist diff --git a/docs/example_scripts/keras_tuner_track.py b/docs/example_scripts/keras_tuner_track.py index efa7e72c57..deae0b86a8 100644 --- a/docs/example_scripts/keras_tuner_track.py +++ b/docs/example_scripts/keras_tuner_track.py @@ -1,6 +1,6 @@ import tensorflow as tf import tensorflow_datasets as tfds -from aim.keras_tuner import AimCallback +from aimstack.ml.adapters.keras_tuner import AimCallback import kerastuner as kt @@ -55,7 +55,7 @@ def standardize_record(record): tuner.search( train_ds, validation_data=test_ds, - callbacks=[AimCallback(repo='.', experiment_name='keras_tuner_test', tuner=tuner)], + callbacks=[AimCallback(experiment_name='keras_tuner_test', tuner=tuner)], ) best_model = tuner.get_best_models(1)[0] diff --git a/docs/example_scripts/lightgbm_track.py b/docs/example_scripts/lightgbm_track.py index 3c9d71d13a..5ce4d8fdce 100644 --- a/docs/example_scripts/lightgbm_track.py +++ b/docs/example_scripts/lightgbm_track.py @@ -2,7 +2,7 @@ import pandas as pd from sklearn.metrics import mean_squared_error -from aim.lightgbm import AimCallback +from aimstack.ml.adapters.lightgbm import AimCallback import lightgbm as lgb diff --git a/docs/example_scripts/mxnet_track.py b/docs/example_scripts/mxnet_track.py index 24810f9e53..bd3d07548a 100644 --- a/docs/example_scripts/mxnet_track.py +++ b/docs/example_scripts/mxnet_track.py @@ -4,7 +4,7 @@ from mxnet.gluon.model_zoo import vision from mxnet.gluon.contrib.estimator import estimator -from aim.mxnet import AimLoggingHandler +from aimstack.ml.adapters.mxnet import AimLoggingHandler gpu_count = mx.context.num_gpus() diff --git a/docs/example_scripts/optuna_track.py b/docs/example_scripts/optuna_track.py index b8a4d9eab9..6cb6ba21c2 100644 --- a/docs/example_scripts/optuna_track.py +++ b/docs/example_scripts/optuna_track.py @@ -1,5 +1,5 @@ import optuna -from aim.optuna import AimCallback +from aimstack.ml.adapters.optuna import AimCallback # Add Aim callback to Optuna optimization diff --git a/docs/example_scripts/paddle_track.py b/docs/example_scripts/paddle_track.py index 54dddbb34c..dfb3fc7d2c 100644 --- a/docs/example_scripts/paddle_track.py +++ b/docs/example_scripts/paddle_track.py @@ -1,7 +1,7 @@ import paddle import paddle.vision.transforms as T from paddle.static import InputSpec -from aim.paddle import AimCallback +from aimstack.ml.adapters.paddle import AimCallback inputs = [InputSpec([-1, 1, 28, 28], 'float32', 'image')] diff --git a/docs/example_scripts/prophet_track.py b/docs/example_scripts/prophet_track.py index d3f01d504f..a29277bc07 100644 --- a/docs/example_scripts/prophet_track.py +++ b/docs/example_scripts/prophet_track.py @@ -1,4 +1,4 @@ -from aim.prophet import AimLogger +from aimstack.ml.adapters.prophet import AimLogger import numpy as np import pandas as pd from prophet import Prophet diff --git a/docs/example_scripts/pytorch_ignite_track.py b/docs/example_scripts/pytorch_ignite_track.py index 4d3bcb37dc..9bacb37c56 100644 --- a/docs/example_scripts/pytorch_ignite_track.py +++ b/docs/example_scripts/pytorch_ignite_track.py @@ -13,7 +13,7 @@ from ignite.handlers import global_step_from_engine, EarlyStopping from ignite.contrib.handlers import ProgressBar -from aim.pytorch_ignite import AimLogger +from aimstack.ml.adapters.pytorch_ignite import AimLogger # transform to normalize the data transform = transforms.Compose( diff --git a/docs/example_scripts/pytorch_lightning_track.py b/docs/example_scripts/pytorch_lightning_track.py index 2ea2b601b4..61d927585c 100644 --- a/docs/example_scripts/pytorch_lightning_track.py +++ b/docs/example_scripts/pytorch_lightning_track.py @@ -1,4 +1,4 @@ -from aim.pytorch_lightning import AimLogger +from aimstack.ml.adapters.pytorch_lightning import AimLogger from argparse import ArgumentParser @@ -47,7 +47,7 @@ def test_step(self, batch, batch_idx): loss = F.cross_entropy(y_hat, y) self.log('test_loss', loss) # Track metrics manually - self.logger.experiment.track(1, name='manually_tracked_metric') + self.logger.experiment.track_auto(1, name='manually_tracked_metric') def configure_optimizers(self): return torch.optim.Adam(self.parameters(), lr=self.hparams.learning_rate) diff --git a/docs/example_scripts/pytorch_track.py b/docs/example_scripts/pytorch_track.py index 843fb54d50..aadcf9e371 100644 --- a/docs/example_scripts/pytorch_track.py +++ b/docs/example_scripts/pytorch_track.py @@ -1,5 +1,5 @@ -from aim import Run -from aim.pytorch import track_gradients_dists, track_params_dists +from aimstack.asp import Run +from aimstack.ml.adapters.pytorch import track_gradients_dists, track_params_dists import torch import torch.nn as nn @@ -111,8 +111,8 @@ def forward(self, x): acc = 100 * correct / total # aim - Track metrics - items = {'accuracy': acc, 'loss': loss} - aim_run.track(items, epoch=epoch, context={'subset': 'train'}) + aim_run.track_auto(acc, name='accuracy', epoch=epoch, context={'subset': 'train'}) + aim_run.track_auto(loss, name='loss', epoch=epoch, context={'subset': 'train'}) # aim - Track weights and gradients distributions track_params_dists(model, aim_run) @@ -120,10 +120,10 @@ def forward(self, x): # TODO: Do actual validation if i % 300 == 0: - aim_run.track( - loss.item(), name='loss', epoch=epoch, context={'subset': 'val'} + aim_run.track_auto( + loss, name='loss', epoch=epoch, context={'subset': 'val'} ) - aim_run.track( + aim_run.track_auto( acc, name='accuracy', epoch=epoch, context={'subset': 'val'} ) diff --git a/docs/example_scripts/pytorch_track_images.py b/docs/example_scripts/pytorch_track_images.py index 0191ef885e..d0edb9acdd 100644 --- a/docs/example_scripts/pytorch_track_images.py +++ b/docs/example_scripts/pytorch_track_images.py @@ -1,4 +1,4 @@ -from aim import Run +from aimstack.asp import Run from aimstack.asp.models.objects.image import convert_to_aim_image_list import torch @@ -105,7 +105,7 @@ def forward(self, x): ) # aim - Track model loss function - aim_run.track( + aim_run.track_auto( loss.item(), name='loss', epoch=epoch, context={'subset': 'train'} ) @@ -117,23 +117,23 @@ def forward(self, x): acc = 100 * correct / total # aim - Track metrics - aim_run.track( + aim_run.track_auto( acc, name='accuracy', epoch=epoch, context={'subset': 'train'} ) - aim_run.track( + aim_run.track_auto( aim_images, name='images', epoch=epoch, context={'subset': 'train'} ) # TODO: Do actual validation if i % 300 == 0: - aim_run.track( - loss.item(), name='loss', epoch=epoch, context={'subset': 'val'} + aim_run.track_auto( + loss, name='loss', epoch=epoch, context={'subset': 'val'} ) - aim_run.track( + aim_run.track_auto( acc, name='accuracy', epoch=epoch, context={'subset': 'val'} ) - aim_run.track( + aim_run.track_auto( aim_images, name='images', epoch=epoch, context={'subset': 'val'} ) diff --git a/docs/example_scripts/sb3_track.py b/docs/example_scripts/sb3_track.py index 3a9d42a10b..266533ed64 100644 --- a/docs/example_scripts/sb3_track.py +++ b/docs/example_scripts/sb3_track.py @@ -1,4 +1,4 @@ -from aim.sb3 import AimCallback +from aimstack.ml.adapters.sb3 import AimCallback from stable_baselines3 import A2C diff --git a/docs/example_scripts/tensorflow_1_keras_track.py b/docs/example_scripts/tensorflow_1_keras_track.py index 70a7bc43a0..114eb9ff8f 100644 --- a/docs/example_scripts/tensorflow_1_keras_track.py +++ b/docs/example_scripts/tensorflow_1_keras_track.py @@ -1,4 +1,4 @@ -from aim.tensorflow import AimCallback +from aimstack.ml.adapters.tensorflow import AimCallback import tensorflow as tf diff --git a/docs/example_scripts/tensorflow_keras_track.py b/docs/example_scripts/tensorflow_keras_track.py index 44cad93272..c138b6b104 100644 --- a/docs/example_scripts/tensorflow_keras_track.py +++ b/docs/example_scripts/tensorflow_keras_track.py @@ -1,4 +1,4 @@ -from aim.tensorflow import AimCallback +from aimstack.ml.adapters.tensorflow import AimCallback import tensorflow as tf diff --git a/docs/example_scripts/xgboost_track.py b/docs/example_scripts/xgboost_track.py index f2cb2c66f2..5a50cc3a93 100644 --- a/docs/example_scripts/xgboost_track.py +++ b/docs/example_scripts/xgboost_track.py @@ -5,7 +5,7 @@ import numpy as np import xgboost as xgb -from aim.xgboost import AimCallback +from aimstack.ml.adapters.xgboost import AimCallback # label need to be 0 to num_class -1 data = np.loadtxt( diff --git a/pkgs/aimstack/asp/loggers/metric.py b/pkgs/aimstack/asp/loggers/metric.py index f195794938..409a2b9541 100644 --- a/pkgs/aimstack/asp/loggers/metric.py +++ b/pkgs/aimstack/asp/loggers/metric.py @@ -1,6 +1,8 @@ import numbers import json +from typing import Optional, Any + from typing import TYPE_CHECKING if TYPE_CHECKING: from pandas import DataFrame @@ -64,7 +66,11 @@ def dataframe( class Metric(Sequence[numbers.Number], DataframeMixin): - ... + def track(self, value: Any, *, step: Optional[int] = None, **axis): + from aim._sdk import num_utils + if num_utils.is_number(value): + value = num_utils.convert_to_py_number(value) + super().track(value, step=step, **axis) class SystemMetric(Sequence[numbers.Number], DataframeMixin): diff --git a/pkgs/aimstack/asp/loggers/run.py b/pkgs/aimstack/asp/loggers/run.py index 5cede8025e..6ac1d9a38d 100644 --- a/pkgs/aimstack/asp/loggers/run.py +++ b/pkgs/aimstack/asp/loggers/run.py @@ -6,13 +6,14 @@ from functools import partialmethod -from aim import Container, Property +from aim import Container, Sequence, Property from aim._sdk.utils import utc_timestamp from aim._sdk import type_utils from aimcore.callbacks import Caller from aimcore.callbacks import events from aim._ext.system_info import utils as system_utils from aim._sdk.constants import ContainerOpenMode, KeyNames +from aim._sdk.num_utils import is_number from .logging import ( LogLine, @@ -28,7 +29,7 @@ from .distribution import DistributionSequence from .figures import FigureSequence, Figure3DSequence -from typing import Optional, Union, List, Tuple, Dict, Any +from typing import Optional, Union, List, Tuple, Dict, Any, Type from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -135,6 +136,12 @@ def track(self, value, name: str, step: Optional[int] = None, context: dict = Op sequence = self.sequences[name, context] sequence.track(value, step=step, **axis) + def track_auto(self, value, name: str, step: int = None, context: dict = None, **axis): + context = {} if context is None else context + seq_type = self._get_sequence_type_from_value(value) + sequence = self.sequences.typed_sequence(seq_type, name, context) + sequence.track(value, step=step, **axis) + def get_metric(self, name: str, context: Optional[dict] = None) -> Metric: context = {} if context is None else context return self.sequences.typed_sequence(Metric, name, context) @@ -204,3 +211,23 @@ def dataframe( import pandas as pd df = pd.DataFrame(data, index=[0]) return df + + @staticmethod + def _get_sequence_type_from_value(value) -> Type[Sequence]: + val_type = type_utils.get_object_typename(value) + if type_utils.is_allowed_type(val_type, type_utils.get_sequence_value_types(Metric)) \ + or is_number(value): + return Metric + if type_utils.is_allowed_type(val_type, type_utils.get_sequence_value_types(ImageSequence)): + return ImageSequence + if type_utils.is_allowed_type(val_type, type_utils.get_sequence_value_types(AudioSequence)): + return AudioSequence + if type_utils.is_allowed_type(val_type, type_utils.get_sequence_value_types(TextSequence)): + return TextSequence + if type_utils.is_allowed_type(val_type, type_utils.get_sequence_value_types(DistributionSequence)): + return TextSequence + if type_utils.is_allowed_type(val_type, type_utils.get_sequence_value_types(FigureSequence)): + return TextSequence + if type_utils.is_allowed_type(val_type, type_utils.get_sequence_value_types(Figure3DSequence)): + return TextSequence + return Sequence diff --git a/pkgs/aimstack/ml/__init__.py b/pkgs/aimstack/ml/__init__.py index e69de29bb2..75d290c4c1 100644 --- a/pkgs/aimstack/ml/__init__.py +++ b/pkgs/aimstack/ml/__init__.py @@ -0,0 +1,6 @@ +from .loggers.run import Run + + +__all__ = ['Run'] + +__aim_types__ = [Run] diff --git a/pkgs/aimstack/ml/adapters/__init__.py b/pkgs/aimstack/ml/integrations/__init__.py similarity index 100% rename from pkgs/aimstack/ml/adapters/__init__.py rename to pkgs/aimstack/ml/integrations/__init__.py diff --git a/pkgs/aimstack/ml/adapters/acme.py b/pkgs/aimstack/ml/integrations/acme.py similarity index 58% rename from pkgs/aimstack/ml/adapters/acme.py rename to pkgs/aimstack/ml/integrations/acme.py index c05dbb0ca1..1e751d34d3 100644 --- a/pkgs/aimstack/ml/adapters/acme.py +++ b/pkgs/aimstack/ml/integrations/acme.py @@ -1,6 +1,5 @@ from typing import Optional, Dict -from aimstack.asp import Run -from aim._ext.system_info.configs import DEFAULT_SYSTEM_TRACKING_INT +from aimstack.ml import Run from acme.utils.loggers.base import Logger, LoggingData @@ -13,11 +12,8 @@ class AimCallback: If skipped, default Repo is used. experiment_name (:obj:`str`, optional): Sets Run's `experiment` property. 'default' if not specified. Can be used later to query runs/sequences. - system_tracking_interval (:obj:`int`, optional): Sets the tracking interval in seconds for system usage - metrics (CPU, Memory, etc.). Set to `None` to disable system metrics tracking. log_system_params (:obj:`bool`, optional): Enable/Disable logging of system params such as installed packages, git info, environment variables, etc. - capture_terminal_logs (:obj:`bool`, optional): Enable/Disable terminal stdout logging. args (:obj:`dict`, optional): Arguments to set a run parameters """ @@ -25,16 +21,12 @@ def __init__( self, repo: Optional[str] = None, experiment_name: Optional[str] = None, - system_tracking_interval: Optional[int] = DEFAULT_SYSTEM_TRACKING_INT, log_system_params: Optional[bool] = True, - capture_terminal_logs: Optional[bool] = True, args: Optional[Dict] = None, ): self.repo = repo self.experiment_name = experiment_name - self.system_tracking_interval = system_tracking_interval self.log_system_params = log_system_params - self.capture_terminal_logs = capture_terminal_logs self._run = None self._run_hash = None @@ -49,29 +41,22 @@ def experiment(self): def setup(self, args=None): if not self._run: if self._run_hash: - self._run = Run( - self._run_hash, - repo=self.repo, - system_tracking_interval=self.system_tracking_interval, - log_system_params=self.log_system_params, - capture_terminal_logs=self.capture_terminal_logs, - ) + self._run = Run(self._run_hash, repo=self.repo) else: - self._run = Run( - repo=self.repo, - experiment=self.experiment_name, - system_tracking_interval=self.system_tracking_interval, - log_system_params=self.log_system_params, - capture_terminal_logs=self.capture_terminal_logs, - ) + self._run = Run(repo=self.repo) self._run_hash = self._run.hash + if self.experiment_name is not None: + self._run.experiment = self.experiment_name + if self.log_system_params: + self._run.enable_system_monitoring() if args: for key, value in args.items(): self._run.set(key, value, strict=False) def track(self, logs, step=None, context=None): - self._run.track(logs, step=step, context=context) + for name, val in logs.items(): + self._run.track_auto(val, name=name, step=step, context=context) def close(self): if self._run and self._run.active: @@ -86,7 +71,8 @@ def __init__(self, aim_run, logger_label, steps_key, task_id): self.task_id = task_id def write(self, values: LoggingData): - self.aim_run.track(values, context={'logger_label': self.logger_label}) + for name, value in values.items(): + self.aim_run.track_auto(value, name=name, context={'logger_label': self.logger_label}) def close(self): if self.aim_run and self.aim_run.active: diff --git a/pkgs/aimstack/ml/adapters/catboost.py b/pkgs/aimstack/ml/integrations/catboost.py similarity index 69% rename from pkgs/aimstack/ml/adapters/catboost.py rename to pkgs/aimstack/ml/integrations/catboost.py index e06a508611..0bc95bfe4b 100644 --- a/pkgs/aimstack/ml/adapters/catboost.py +++ b/pkgs/aimstack/ml/integrations/catboost.py @@ -1,8 +1,7 @@ from sys import stdout from typing import Optional -from aim import Run -from aim._ext.system_info import DEFAULT_SYSTEM_TRACKING_INT +from aimstack.ml import Run class AimLogger: @@ -12,13 +11,10 @@ class AimLogger: Args: repo (:obj:`str`, optional): Aim repository path or Repo object to which Run object is bound. If skipped, default Repo is used. - experiment_name (:obj:`str`, optional): Sets Run's `experiment` property. 'default' if not specified. + experiment (:obj:`str`, optional): Sets Run's `experiment` property. 'default' if not specified. Can be used later to query runs/sequences. - system_tracking_interval (:obj:`int`, optional): Sets the tracking interval in seconds for system usage - metrics (CPU, Memory, etc.). Set to `None` to disable system metrics tracking. log_system_params (:obj:`bool`, optional): Enable/Disable logging of system params such as installed packages, git info, environment variables, etc. - capture_terminal_logs (:obj:`bool`, optional): Enable/Disable terminal stdout logging. loss_function (:obj:`str`, optional): Loss function log_cout (:obj:`bool`, optional): Enable/Disable stdout logging. """ @@ -27,18 +23,14 @@ def __init__( self, repo: Optional[str] = None, experiment: Optional[str] = None, - system_tracking_interval: Optional[int] = DEFAULT_SYSTEM_TRACKING_INT, log_system_params: Optional[bool] = True, - capture_terminal_logs: Optional[bool] = True, loss_function: Optional[str] = 'Loss', log_cout=stdout, ): super().__init__() self._repo_path = repo self._experiment = experiment - self._system_tracking_interval = system_tracking_interval self._log_system_params = log_system_params - self._capture_terminal_logs = capture_terminal_logs self._run = None self._run_hash = None self._loss_function = loss_function @@ -57,21 +49,15 @@ def setup(self): if self._run: return if self._run_hash: - self._run = Run( - self._run_hash, - repo=self._repo_path, - system_tracking_interval=self._system_tracking_interval, - capture_terminal_logs=self._capture_terminal_logs, - ) + self._run = Run(self._run_hash, repo=self._repo_path) else: - self._run = Run( - repo=self._repo_path, - experiment=self._experiment, - system_tracking_interval=self._system_tracking_interval, - log_system_params=self._log_system_params, - capture_terminal_logs=self._capture_terminal_logs, - ) + self._run = Run(repo=self._repo_path) self._run_hash = self._run.hash + if self._experiment is not None: + self._run.experiment = self._experiment + + if self._log_system_params: + self._run.enable_system_monitoring() def _to_number(self, val): try: @@ -103,21 +89,21 @@ def write(self, log): value_best = self._to_number(log[6]) if any((value_learn, value_test, value_best)): if value_learn: - run.track( + run.track_auto( value_learn, name=self._loss_function, step=value_iter, context={'log': 'learn'}, ) if value_test: - run.track( + run.track_auto( value_test, name=self._loss_function, step=value_iter, context={'log': 'test'}, ) if value_best: - run.track( + run.track_auto( value_best, name=self._loss_function, step=value_iter, diff --git a/pkgs/aimstack/ml/adapters/fastai.py b/pkgs/aimstack/ml/integrations/fastai.py similarity index 76% rename from pkgs/aimstack/ml/adapters/fastai.py rename to pkgs/aimstack/ml/integrations/fastai.py index 3e3c48a2f0..31662722b4 100644 --- a/pkgs/aimstack/ml/adapters/fastai.py +++ b/pkgs/aimstack/ml/integrations/fastai.py @@ -1,8 +1,7 @@ from logging import getLogger from typing import Optional -from aim._sdk.run import Run -from aim._ext.system_info import DEFAULT_SYSTEM_TRACKING_INT +from aimstack.ml import Run try: from fastai.learner import Callback @@ -26,27 +25,20 @@ class AimCallback(Callback): If skipped, default Repo is used. experiment_name (:obj:`str`, optional): Sets Run's `experiment` property. 'default' if not specified. Can be used later to query runs/sequences. - system_tracking_interval (:obj:`int`, optional): Sets the tracking interval in seconds for system usage - metrics (CPU, Memory, etc.). Set to `None` to disable system metrics tracking. log_system_params (:obj:`bool`, optional): Enable/Disable logging of system params such as installed packages, git info, environment variables, etc. - capture_terminal_logs (:obj:`bool`, optional): Enable/Disable terminal stdout logging. """ def __init__( self, repo: Optional[str] = None, experiment_name: Optional[str] = None, - system_tracking_interval: Optional[int] = DEFAULT_SYSTEM_TRACKING_INT, log_system_params: Optional[bool] = True, - capture_terminal_logs: Optional[bool] = True, ): store_attr() self.repo = repo self.experiment_name = experiment_name - self.system_tracking_interval = system_tracking_interval self.log_system_params = log_system_params - self.capture_terminal_logs = capture_terminal_logs self._run = None self._run_hash = None @@ -59,23 +51,14 @@ def experiment(self) -> Run: def setup(self, args=None): if not self._run: if self._run_hash: - self._run = Run( - self._run_hash, - repo=self.repo, - system_tracking_interval=self.system_tracking_interval, - log_system_params=self.log_system_params, - capture_terminal_logs=self.capture_terminal_logs, - ) + self._run = Run(self._run_hash, repo=self.repo) else: - self._run = Run( - repo=self.repo, - experiment=self.experiment_name, - system_tracking_interval=self.system_tracking_interval, - log_system_params=self.log_system_params, - capture_terminal_logs=self.capture_terminal_logs, - ) + self._run = Run(repo=self.repo) self._run_hash = self._run.hash - + if self.experiment_name is not None: + self._run.experiment = self.experiment_name + if self.log_system_params: + self._run.enable_system_monitoring() # Log config parameters if args: try: @@ -93,12 +76,12 @@ def before_fit(self): def after_batch(self): context = {'subset': 'train'} if self.training else {'subset': 'val'} - self._run.track( + self._run.track_auto( self.loss.item(), name='train_loss', step=self.train_iter, context=context ) for i, h in enumerate(self.opt.hypers): for k, v in h.items(): - self._run.track(v, f'{k}_{i}', step=self.train_iter, context=context) + self._run.track_auto(v, f'{k}_{i}', step=self.train_iter, context=context) def before_epoch(self): for metric in self.metrics: @@ -107,7 +90,7 @@ def before_epoch(self): def after_epoch(self): for name, value in zip(self.recorder.metric_names, self.recorder.log): if name not in ['train_loss', 'epoch', 'time'] and value is not None: - self._run.track(value, name=name) + self._run.track_auto(value, name=name) def __del__(self): if self._run and self._run.active: diff --git a/pkgs/aimstack/ml/adapters/hugging_face.py b/pkgs/aimstack/ml/integrations/hugging_face.py similarity index 80% rename from pkgs/aimstack/ml/adapters/hugging_face.py rename to pkgs/aimstack/ml/integrations/hugging_face.py index 099886ec1e..787f8e67a8 100644 --- a/pkgs/aimstack/ml/adapters/hugging_face.py +++ b/pkgs/aimstack/ml/integrations/hugging_face.py @@ -3,9 +3,8 @@ from difflib import SequenceMatcher from collections import defaultdict -from aim._ext.system_info import DEFAULT_SYSTEM_TRACKING_INT from aim._sdk.num_utils import is_number -from aim._sdk.run import Run +from aimstack.ml import Run try: from transformers.trainer_callback import TrainerCallback @@ -27,26 +26,19 @@ class AimCallback(TrainerCallback): If skipped, default Repo is used. experiment_name (:obj:`str`, optional): Sets Run's `experiment` property. 'default' if not specified. Can be used later to query runs/sequences. - system_tracking_interval (:obj:`int`, optional): Sets the tracking interval in seconds for system usage - metrics (CPU, Memory, etc.). Set to `None` to disable system metrics tracking. log_system_params (:obj:`bool`, optional): Enable/Disable logging of system params such as installed packages, git info, environment variables, etc. - capture_terminal_logs (:obj:`bool`, optional): Enable/Disable terminal stdout logging. """ def __init__( self, repo: Optional[str] = None, experiment_name: Optional[str] = None, - system_tracking_interval: Optional[int] = DEFAULT_SYSTEM_TRACKING_INT, log_system_params: Optional[bool] = True, - capture_terminal_logs: Optional[bool] = True, ): self._repo_path = repo self._experiment_name = experiment_name - self._system_tracking_interval = system_tracking_interval self._log_system_params = log_system_params - self._capture_terminal_logs = capture_terminal_logs self._run = None self._run_hash = None self._log_value_warned = False @@ -63,21 +55,14 @@ def setup(self, args=None, state=None, model=None): if not self._run: if self._run_hash: - self._run = Run( - self._run_hash, - repo=self._repo_path, - system_tracking_interval=self._system_tracking_interval, - capture_terminal_logs=self._capture_terminal_logs, - ) + self._run = Run(self._run_hash, repo=self._repo_path) else: - self._run = Run( - repo=self._repo_path, - experiment=self._experiment_name, - system_tracking_interval=self._system_tracking_interval, - log_system_params=self._log_system_params, - capture_terminal_logs=self._capture_terminal_logs, - ) + self._run = Run(repo=self._repo_path) self._run_hash = self._run.hash + if self._experiment_name is not None: + self._run.experiment = self._experiment_name + if self._log_system_params: + self._run.enable_system_monitoring() if args: combined_dict = {**args.to_sanitized_dict()} @@ -141,7 +126,7 @@ def on_log(self, args, state, control, model=None, logs=None, **kwargs): ) continue - self._run.track( + self._run.track_auto( log_value, name=log_name, context=context, diff --git a/pkgs/aimstack/ml/adapters/keras.py b/pkgs/aimstack/ml/integrations/keras.py similarity index 52% rename from pkgs/aimstack/ml/adapters/keras.py rename to pkgs/aimstack/ml/integrations/keras.py index c326ab3959..82cbfc6b01 100644 --- a/pkgs/aimstack/ml/adapters/keras.py +++ b/pkgs/aimstack/ml/integrations/keras.py @@ -1,8 +1,7 @@ from typing import Optional -from aim._sdk.run import Run -from aimstack.ml.adapters.keras_mixins import TrackerKerasCallbackMetricsEpochEndMixin -from aim._ext.system_info import DEFAULT_SYSTEM_TRACKING_INT +from aimstack.ml import Run +from aimstack.ml.integrations.keras_mixins import TrackerKerasCallbackMetricsEpochEndMixin try: from keras.callbacks import Callback @@ -22,41 +21,25 @@ class AimCallback(TrackerKerasCallbackMetricsEpochEndMixin, Callback): If skipped, default Repo is used. experiment_name (:obj:`str`, optional): Sets Run's `experiment` property. 'default' if not specified. Can be used later to query runs/sequences. - system_tracking_interval (:obj:`int`, optional): Sets the tracking interval in seconds for system usage - metrics (CPU, Memory, etc.). Set to `None` to disable system metrics tracking. log_system_params (:obj:`bool`, optional): Enable/Disable logging of system params such as installed packages, git info, environment variables, etc. - capture_terminal_logs (:obj:`bool`, optional): Enable/Disable terminal stdout logging. """ def __init__( self, repo: Optional[str] = None, experiment_name: Optional[str] = None, - system_tracking_interval: Optional[int] = DEFAULT_SYSTEM_TRACKING_INT, log_system_params: Optional[bool] = True, - capture_terminal_logs: Optional[bool] = True, ): super(Callback, self).__init__() - self._system_tracking_interval = system_tracking_interval self._log_system_params = log_system_params - self._capture_terminal_logs = capture_terminal_logs - if repo is None and experiment_name is None: - self._run = Run( - system_tracking_interval=self._system_tracking_interval, - log_system_params=self._log_system_params, - capture_terminal_logs=self._capture_terminal_logs, - ) - else: - self._run = Run( - repo=repo, - experiment=experiment_name, - system_tracking_interval=self._system_tracking_interval, - log_system_params=self._log_system_params, - capture_terminal_logs=self._capture_terminal_logs, - ) + self._run = Run(repo=repo) + if experiment_name is not None: + self._run.experiment = experiment_name + if log_system_params: + self._run.enable_system_monitoring() self._run_hash = self._run.hash self._repo_path = repo @@ -64,12 +47,9 @@ def __init__( @property def experiment(self) -> Run: if not self._run: - self._run = Run( - self._run_hash, - repo=self._repo_path, - system_tracking_interval=self._system_tracking_interval, - capture_terminal_logs=self._capture_terminal_logs, - ) + self._run = Run(self._run_hash, repo=self._repo_path) + if self._log_system_params: + self._run.enable_system_monitoring() return self._run @classmethod @@ -84,7 +64,6 @@ def metrics( def close(self) -> None: if self._run: - self._run.close() del self._run self._run = None diff --git a/pkgs/aimstack/ml/adapters/keras_mixins.py b/pkgs/aimstack/ml/integrations/keras_mixins.py similarity index 73% rename from pkgs/aimstack/ml/adapters/keras_mixins.py rename to pkgs/aimstack/ml/integrations/keras_mixins.py index b6ed8308d7..36b340f15a 100644 --- a/pkgs/aimstack/ml/adapters/keras_mixins.py +++ b/pkgs/aimstack/ml/integrations/keras_mixins.py @@ -14,16 +14,14 @@ def _log_epoch_metrics(self, epoch, logs): if not logs: return - track_func = self._run.track - train_logs = {k: v for k, v in logs.items() if not k.startswith('val_')} for name, value in train_logs.items(): - track_func(value, name=name, epoch=epoch, context={'subset': 'train'}) + self._run.track_auto(value, name=name, context={'subset': 'train'}, epoch=epoch) val_logs = {k: v for k, v in logs.items() if k.startswith('val_')} for name, value in val_logs.items(): - track_func(value, name=name[4:], epoch=epoch, context={'subset': 'val'}) + self._run.track_auto(value, name=name[4:], context={'subset': 'val'}, epoch=epoch) lr = self._get_learning_rate() if lr is not None: - track_func(lr, name='lr', epoch=epoch, context={'subset': 'train'}) + self._run.track_auto(lr, name='lr', context={'subset': 'train'}, epoch=epoch) diff --git a/pkgs/aimstack/ml/adapters/keras_tuner.py b/pkgs/aimstack/ml/integrations/keras_tuner.py similarity index 58% rename from pkgs/aimstack/ml/adapters/keras_tuner.py rename to pkgs/aimstack/ml/integrations/keras_tuner.py index 519b3b4c5b..8f7ec38f08 100644 --- a/pkgs/aimstack/ml/adapters/keras_tuner.py +++ b/pkgs/aimstack/ml/integrations/keras_tuner.py @@ -1,7 +1,6 @@ from typing import Optional, List -from aim._sdk.run import Run -from aim._ext.system_info import DEFAULT_SYSTEM_TRACKING_INT +from aimstack.ml import Run try: from kerastuner.engine.tuner_utils import TunerCallback @@ -21,11 +20,8 @@ class AimCallback(TunerCallback): If skipped, default Repo is used. experiment_name (:obj:`str`, optional): Sets Run's `experiment` property. 'default' if not specified. Can be used later to query runs/sequences. - system_tracking_interval (:obj:`int`, optional): Sets the tracking interval in seconds for system usage - metrics (CPU, Memory, etc.). Set to `None` to disable system metrics tracking. log_system_params (:obj:`bool`, optional): Enable/Disable logging of system params such as installed packages, git info, environment variables, etc. - capture_terminal_logs (:obj:`bool`, optional): Enable/Disable terminal stdout logging. tuner (:obj:`any`): keras_tuner instance """ @@ -33,16 +29,12 @@ def __init__( self, repo: Optional[str] = None, experiment_name: Optional[str] = None, - system_tracking_interval: Optional[int] = DEFAULT_SYSTEM_TRACKING_INT, log_system_params: Optional[bool] = True, - capture_terminal_logs: Optional[bool] = True, tuner=None, ): self._repo_path = repo self._experiment_name = experiment_name - self._system_tracking_interval = system_tracking_interval self._log_system_params = log_system_params - self._capture_terminal_logs = capture_terminal_logs self.tuner = tuner @@ -60,20 +52,11 @@ def on_epoch_begin(self, epoch, logs=None): tuner_key = next(iter(trial_dict)) self._current_trial_id = trial_dict[tuner_key].trial_id if self._current_trial_id not in self._started_trials: - if self._repo_path is None and self._experiment_name is None: - self._run = Run( - system_tracking_interval=self._system_tracking_interval, - log_system_params=self._log_system_params, - capture_terminal_logs=self._capture_terminal_logs, - ) - else: - self._run = Run( - repo=self._repo_path, - experiment=self._experiment_name, - system_tracking_interval=self._system_tracking_interval, - log_system_params=self._log_system_params, - capture_terminal_logs=self._capture_terminal_logs, - ) + self._run = Run(repo=self._repo_path) + if self._experiment_name is not None: + self._run.experiment = self._experiment_name + if self._log_system_params: + self._run.enable_system_monitoring() self._run['trial_id'] = self._current_trial_id self._started_trials.append(self._current_trial_id) trial = self.tuner.oracle.get_trial(self._current_trial_id) @@ -84,8 +67,4 @@ def on_epoch_begin(self, epoch, logs=None): def on_batch_end(self, batch, logs=None): if logs: for log_name, log_value in logs.items(): - self._run.track(log_value, name=log_name) - - def __del__(self): - if self._run is not None and self._run.active: - self._run.close() + self._run.get_metric(name=log_name, context={}).track_auto(log_value) diff --git a/pkgs/aimstack/ml/adapters/lightgbm.py b/pkgs/aimstack/ml/integrations/lightgbm.py similarity index 64% rename from pkgs/aimstack/ml/adapters/lightgbm.py rename to pkgs/aimstack/ml/integrations/lightgbm.py index fe0dd85aca..26c8595fcf 100644 --- a/pkgs/aimstack/ml/adapters/lightgbm.py +++ b/pkgs/aimstack/ml/integrations/lightgbm.py @@ -1,7 +1,6 @@ from typing import Optional -from aim import Run -from aim._ext.system_info import DEFAULT_SYSTEM_TRACKING_INT +from aimstack.ml import Run try: from lightgbm.callback import CallbackEnv @@ -21,26 +20,19 @@ class AimCallback: If skipped, default Repo is used. experiment_name (:obj:`str`, optional): Sets Run's `experiment` property. 'default' if not specified. Can be used later to query runs/sequences. - system_tracking_interval (:obj:`int`, optional): Sets the tracking interval in seconds for system usage - metrics (CPU, Memory, etc.). Set to `None` to disable system metrics tracking. log_system_params (:obj:`bool`, optional): Enable/Disable logging of system params such as installed packages, git info, environment variables, etc. - capture_terminal_logs (:obj:`bool`, optional): Enable/Disable terminal stdout logging. """ def __init__( self, repo: Optional[str] = None, experiment_name: Optional[str] = None, - system_tracking_interval: Optional[int] = DEFAULT_SYSTEM_TRACKING_INT, log_system_params: Optional[bool] = True, - capture_terminal_logs: Optional[bool] = True, ): self._repo_path = repo self._experiment = experiment_name - self._system_tracking_interval = system_tracking_interval self._log_system_params = log_system_params - self._capture_terminal_logs = capture_terminal_logs self._run = None self._run_hash = None @@ -58,21 +50,14 @@ def setup(self): if self._run: return if self._run_hash: - self._run = Run( - self._run_hash, - repo=self._repo_path, - system_tracking_interval=self._system_tracking_interval, - capture_terminal_logs=self._capture_terminal_logs, - ) + self._run = Run(self._run_hash, repo=self._repo_path) else: - self._run = Run( - repo=self._repo_path, - experiment=self._experiment, - system_tracking_interval=self._system_tracking_interval, - log_system_params=self._log_system_params, - capture_terminal_logs=self._capture_terminal_logs, - ) + self._run = Run(repo=self._repo_path) self._run_hash = self._run.hash + if self._experiment is not None: + self._run.experiment = self._experiment + if self._log_system_params: + self._run.enable_system_monitoring() def __call__(self, env: CallbackEnv): if env.iteration == env.begin_iteration: @@ -83,17 +68,17 @@ def __call__(self, env: CallbackEnv): for item in env.evaluation_result_list: if len(item) == 4: data_name, eval_name, result, _ = item - self._run.track( + self._run.track_auto( result, name=eval_name, context={'data_name': data_name} ) else: data_name, eval_name = item[1].split() res_mean = item[2] res_stdv = item[4] - self._run.track( + self._run.track_auto( res_mean, name=f'{eval_name}-mean', context={'data_name': data_name} ) - self._run.track( + self._run.track_auto( res_stdv, name=f'{eval_name}-stdv', context={'data_name': data_name} ) @@ -101,7 +86,6 @@ def __call__(self, env: CallbackEnv): def close(self): if self._run: - self._run.close() del self._run self._run = None diff --git a/pkgs/aimstack/ml/adapters/mxnet.py b/pkgs/aimstack/ml/integrations/mxnet.py similarity index 83% rename from pkgs/aimstack/ml/adapters/mxnet.py rename to pkgs/aimstack/ml/integrations/mxnet.py index 129199a682..108cf89cbf 100644 --- a/pkgs/aimstack/ml/adapters/mxnet.py +++ b/pkgs/aimstack/ml/integrations/mxnet.py @@ -11,8 +11,7 @@ Estimator, ) from typing import Optional, Union, Any, List -from aim._sdk.run import Run -from aim._ext.system_info import DEFAULT_SYSTEM_TRACKING_INT +from aimstack.ml import Run class AimLoggingHandler( @@ -26,11 +25,8 @@ class AimLoggingHandler( If skipped, default Repo is used. experiment_name (:obj:`str`, optional): Sets Run's `experiment` property. 'default' if not specified. Can be used later to query runs/sequences. - system_tracking_interval (:obj:`int`, optional): Sets the tracking interval in seconds for system usage - metrics (CPU, Memory, etc.). Set to `None` to disable system metrics tracking. log_system_params (:obj:`bool`, optional): Enable/Disable logging of system params such as installed packages, git info, environment variables, etc. - capture_terminal_logs (:obj:`bool`, optional): Enable/Disable terminal stdout logging. log_interval: (:obj:`int` or `str`) default: 'epoch': Logging interval during training. log_interval='epoch': display metrics every epoch log_interval=integer k: display metrics every interval of k batches @@ -44,9 +40,7 @@ def __init__( self, repo: Optional[str] = None, experiment_name: Optional[str] = None, - system_tracking_interval: Optional[int] = DEFAULT_SYSTEM_TRACKING_INT, log_system_params: Optional[bool] = True, - capture_terminal_logs: Optional[bool] = True, log_interval: Union[int, str] = 'epoch', metrics: Optional[List[Any]] = None, priority=np.Inf, @@ -67,9 +61,7 @@ def __init__( self.repo = repo self.experiment_name = experiment_name - self.system_tracking_interval = system_tracking_interval self.log_system_params = log_system_params - self.capture_terminal_logs = capture_terminal_logs self._run = None self._run_hash = None @@ -143,7 +135,7 @@ def epoch_end(self, estimator: Optional[Estimator], *args, **kwargs): context_name, metric_name = name.split(' ') context = {'subset': context_name} - self._run.track( + self._run.track_auto( value, metric_name, step=self.batch_index, context=context ) estimator.logger.info(msg.rstrip(', ')) @@ -171,7 +163,7 @@ def batch_end(self, estimator: Optional[Estimator], *args, **kwargs): context_name, metric_name = name.split(' ') context = {'subset': context_name} - self._run.track( + self._run.track_auto( value, metric_name, step=self.batch_index, context=context ) estimator.logger.info(msg.rstrip(', ')) @@ -186,22 +178,14 @@ def experiment(self) -> Run: def setup(self, estimator: Optional[Estimator] = None, args=None): if not self._run: if self._run_hash: - self._run = Run( - self._run_hash, - repo=self.repo, - system_tracking_interval=self.system_tracking_interval, - log_system_params=self.log_system_params, - capture_terminal_logs=self.capture_terminal_logs, - ) + self._run = Run(self._run_hash, repo=self.repo) else: - self._run = Run( - repo=self.repo, - experiment=self.experiment_name, - system_tracking_interval=self.system_tracking_interval, - log_system_params=self.log_system_params, - capture_terminal_logs=self.capture_terminal_logs, - ) + self._run = Run(repo=self.repo) self._run_hash = self._run.hash + if self.experiment_name is not None: + self._run.experiment = self.experiment_name + if self.log_system_params: + self._run.enable_system_monitoring() # Log config parameters if args: @@ -212,5 +196,6 @@ def setup(self, estimator: Optional[Estimator] = None, args=None): estimator.logger.warning(f'Aim could not log config parameters -> {e}') def __del__(self): - if self._run and self._run.active: - self._run.close() + if self._run is not None: + del self._run + self._run = None diff --git a/pkgs/aimstack/ml/adapters/optuna.py b/pkgs/aimstack/ml/integrations/optuna.py similarity index 80% rename from pkgs/aimstack/ml/adapters/optuna.py rename to pkgs/aimstack/ml/integrations/optuna.py index 0d9ad3b7e6..bab3860617 100644 --- a/pkgs/aimstack/ml/adapters/optuna.py +++ b/pkgs/aimstack/ml/integrations/optuna.py @@ -8,8 +8,7 @@ with try_import() as _imports: - from aim._sdk.run import Run - from aim._ext.system_info import DEFAULT_SYSTEM_TRACKING_INT + from aimstack.ml import Run @experimental_class('2.9.0') @@ -22,11 +21,8 @@ class AimCallback: If skipped, default Repo is used. experiment_name (:obj:`str`, optional): Sets Run's `experiment` property. 'default' if not specified. Can be used later to query runs/sequences. - system_tracking_interval (:obj:`int`, optional): Sets the tracking interval in seconds for system usage - metrics (CPU, Memory, etc.). Set to `None` to disable system metrics tracking. log_system_params (:obj:`bool`, optional): Enable/Disable logging of system params such as installed packages, git info, environment variables, etc. - capture_terminal_logs (:obj:`bool`, optional): Enable/Disable terminal stdout logging. metric_name (:obj:`str`, optional): Name assigned to optimized metric. In case of multi-objective optimization, list of names can be passed. Those names will be assigned to metrics in the order returned by objective function. @@ -45,9 +41,7 @@ def __init__( self, repo: Optional[str] = None, experiment_name: Optional[str] = None, - system_tracking_interval: Optional[int] = DEFAULT_SYSTEM_TRACKING_INT, log_system_params: Optional[bool] = True, - capture_terminal_logs: Optional[bool] = True, metric_name: Union[str, Sequence[str]] = 'value', as_multirun: bool = False, ) -> None: @@ -62,9 +56,7 @@ def __init__( self._as_multirun = as_multirun self._repo_path = repo self._experiment_name = experiment_name - self._system_tracking_interval = system_tracking_interval self._log_system_params = log_system_params - self._capture_terminal_logs = capture_terminal_logs self._run = None self._run_hash = None @@ -118,10 +110,10 @@ def __call__( self.close() else: for key, value in trial.params.items(): - self._run.track(value, name=key, step=step) + self._run.track_auto(value, name=key, step=step) for key, value in metrics.items(): - self._run.track(value, name=key, step=step) + self._run.track_auto(value, name=key, step=step) @property def experiment(self) -> Run: @@ -131,21 +123,14 @@ def experiment(self) -> Run: def setup(self, args=None): if not self._run: if self._run_hash: - self._run = Run( - self._run_hash, - repo=self._repo_path, - system_tracking_interval=self._system_tracking_interval, - capture_terminal_logs=self._capture_terminal_logs, - ) + self._run = Run(self._run_hash, repo=self._repo_path) else: - self._run = Run( - repo=self._repo_path, - experiment=self._experiment_name, - system_tracking_interval=self._system_tracking_interval, - log_system_params=self._log_system_params, - capture_terminal_logs=self._capture_terminal_logs, - ) + self._run = Run(repo=self._repo_path) self._run_hash = self._run.hash + if self._experiment_name is not None: + self._run.experiment = self._experiment_name + if self._log_system_params: + self._run.enable_system_monitoring() if args: for key, value in args.items(): diff --git a/pkgs/aimstack/ml/adapters/paddle.py b/pkgs/aimstack/ml/integrations/paddle.py similarity index 58% rename from pkgs/aimstack/ml/adapters/paddle.py rename to pkgs/aimstack/ml/integrations/paddle.py index 9c15252d90..92e907861d 100644 --- a/pkgs/aimstack/ml/adapters/paddle.py +++ b/pkgs/aimstack/ml/integrations/paddle.py @@ -1,7 +1,6 @@ from paddle.hapi.callbacks import Callback from typing import Optional -from aim._sdk.run import Run -from aim._ext.system_info import DEFAULT_SYSTEM_TRACKING_INT +from aimstack.ml import Run class AimCallback(Callback): @@ -13,26 +12,19 @@ class AimCallback(Callback): If skipped, default Repo is used. experiment_name (:obj:`str`, optional): Sets Run's `experiment` property. 'default' if not specified. Can be used later to query runs/sequences. - system_tracking_interval (:obj:`int`, optional): Sets the tracking interval in seconds for system usage - metrics (CPU, Memory, etc.). Set to `None` to disable system metrics tracking. log_system_params (:obj:`bool`, optional): Enable/Disable logging of system params such as installed packages, git info, environment variables, etc. - capture_terminal_logs (:obj:`bool`, optional): Enable/Disable terminal stdout logging. """ def __init__( self, repo: Optional[str] = None, experiment_name: Optional[str] = None, - system_tracking_interval: Optional[int] = DEFAULT_SYSTEM_TRACKING_INT, log_system_params: Optional[bool] = True, - capture_terminal_logs: Optional[bool] = True, ): self.repo = repo self.experiment_name = experiment_name - self.system_tracking_interval = system_tracking_interval self.log_system_params = log_system_params - self.capture_terminal_logs = capture_terminal_logs self._run = None self._run_hash = None @@ -56,7 +48,7 @@ def _track(self, logs, context, step=None): v = v[0] else: raise NotImplementedError(f'number of items in {k} are more than 1') - self._run.track(v, k, step=step, context=context, epoch=self.epoch) + self._run.track_auto(v, k, step=step, context=context, epoch=self.epoch) @property def experiment(self): @@ -67,28 +59,20 @@ def experiment(self): def setup(self, args=None): if not self._run: if self._run_hash: - self._run = Run( - self._run_hash, - repo=self.repo, - system_tracking_interval=self.system_tracking_interval, - log_system_params=self.log_system_params, - capture_terminal_logs=self.capture_terminal_logs, - ) + self._run = Run(self._run_hash, repo=self.repo) else: - self._run = Run( - repo=self.repo, - experiment=self.experiment_name, - system_tracking_interval=self.system_tracking_interval, - log_system_params=self.log_system_params, - capture_terminal_logs=self.capture_terminal_logs, - ) + self._run = Run(repo=self.repo) self._run_hash = self._run.hash - + if self.experiment_name is not None: + self._run.experiment = self.experiment_name + if self.log_system_params: + self._run.enable_system_monitoring() # Log config parameters if args: for key in args: self._run.set(key, args[key], strict=False) def __del__(self): - if self._run and self._run.active: - self._run.close() + if self._run is not None: + del self._run + self._run = None diff --git a/pkgs/aimstack/ml/adapters/prophet.py b/pkgs/aimstack/ml/integrations/prophet.py similarity index 63% rename from pkgs/aimstack/ml/adapters/prophet.py rename to pkgs/aimstack/ml/integrations/prophet.py index e99726262f..25bd250d01 100644 --- a/pkgs/aimstack/ml/adapters/prophet.py +++ b/pkgs/aimstack/ml/integrations/prophet.py @@ -1,7 +1,6 @@ from typing import Any, Dict, Optional, TypeVar -from aimstack.asp import Run -from aim._ext.system_info import DEFAULT_SYSTEM_TRACKING_INT +from aimstack.ml import Run Prophet = TypeVar('Prophet') @@ -18,11 +17,8 @@ class AimLogger: If skipped, default Repo is used. experiment_name (:obj:`str`, optional): Sets Run's `experiment` property. 'default' if not specified. Can be used later to query runs/sequences. - system_tracking_interval (:obj:`int`, optional): Sets the tracking interval in seconds for system usage - metrics (CPU, Memory, etc.). Set to `None` to disable system metrics tracking. log_system_params (:obj:`bool`, optional): Enable/Disable logging of system params such as installed packages, git info, environment variables, etc. - capture_terminal_logs (:obj:`bool`, optional): Enable/Disable terminal stdout logging. prophet_model (:obj: `Any`, optional): An instance of Prophet model. """ @@ -30,16 +26,12 @@ def __init__( self, repo: Optional[str] = None, experiment_name: Optional[str] = None, - system_tracking_interval: Optional[int] = DEFAULT_SYSTEM_TRACKING_INT, log_system_params: Optional[bool] = True, - capture_terminal_logs: Optional[bool] = True, prophet_model: Prophet = None, ): self._repo_path = repo self._experiment = experiment_name - self._system_tracking_interval = system_tracking_interval self._log_system_params = log_system_params - self._capture_terminal_logs = capture_terminal_logs self._run = None self._run_hash = None self.setup(prophet_model.__dict__) @@ -57,28 +49,22 @@ def setup(self, hparams: Dict[str, Any]) -> None: if self._run: return if self._run_hash: - self._run = Run( - self._run_hash, - repo=self._repo_path, - system_tracking_interval=self._system_tracking_interval, - capture_terminal_logs=self._capture_terminal_logs, - ) + self._run = Run(self._run_hash, repo=self._repo_path) else: - self._run = Run( - repo=self._repo_path, - experiment=self._experiment, - system_tracking_interval=self._system_tracking_interval, - log_system_params=self._log_system_params, - capture_terminal_logs=self._capture_terminal_logs, - ) + self._run = Run(repo=self._repo_path) self._run_hash = self._run.hash + if self._experiment is not None: + self._run.experiment = self._experiment + if self._log_system_params: + self._run.enable_system_monitoring() for hparam, value in hparams.items(): self._run.set(hparam, value, strict=False) def __del__(self) -> None: - if self._run and self._run.active: - self._run.close() + if self._run is not None: + del self._run + self._run = None def track_metrics( self, @@ -94,4 +80,4 @@ def track_metrics( if context is None: context = {'subset': 'val'} for metric, value in metrics.items(): - self._run.track(value, name=metric, context=context) + self._run.track_auto(value, name=metric, context=context) diff --git a/pkgs/aimstack/ml/adapters/pytorch.py b/pkgs/aimstack/ml/integrations/pytorch.py similarity index 98% rename from pkgs/aimstack/ml/adapters/pytorch.py rename to pkgs/aimstack/ml/integrations/pytorch.py index 4d3f20d9ee..18fa1954ed 100644 --- a/pkgs/aimstack/ml/adapters/pytorch.py +++ b/pkgs/aimstack/ml/integrations/pytorch.py @@ -1,4 +1,4 @@ -from aimstack.asp import Run +from aimstack.ml import Run def track_params_dists(model, run: Run): diff --git a/pkgs/aimstack/ml/adapters/pytorch_ignite.py b/pkgs/aimstack/ml/integrations/pytorch_ignite.py similarity index 85% rename from pkgs/aimstack/ml/adapters/pytorch_ignite.py rename to pkgs/aimstack/ml/integrations/pytorch_ignite.py index 6ede363ba1..72b65b6b60 100644 --- a/pkgs/aimstack/ml/adapters/pytorch_ignite.py +++ b/pkgs/aimstack/ml/integrations/pytorch_ignite.py @@ -1,7 +1,6 @@ from typing import Any, Callable, Dict, List, Optional, Union -from aim import Run -from aim._ext.system_info import DEFAULT_SYSTEM_TRACKING_INT +from aimstack.ml import Run try: from torch.optim import Optimizer @@ -33,11 +32,8 @@ class AimLogger(BaseLogger): If skipped, default Repo is used. experiment_name (:obj:`str`, optional): Sets Run's `experiment` property. 'default' if not specified. Can be used later to query runs/sequences. - system_tracking_interval (:obj:`int`, optional): Sets the tracking interval in seconds for system usage - metrics (CPU, Memory, etc.). Set to `None` to disable system metrics tracking. log_system_params (:obj:`bool`, optional): Enable/Disable logging of system params such as installed packages, git info, environment variables, etc. - capture_terminal_logs (:obj:`bool`, optional): Enable/Disable terminal stdout logging. train_metric_prefix (:obj:`str`, optional): Training metric prefix. val_metric_prefix (:obj:`str`, optional): validation metric prefix. test_metric_prefix (:obj:`str`, optional): testing metric prefix. @@ -48,9 +44,7 @@ def __init__( self, repo: Optional[str] = None, experiment_name: Optional[str] = None, - system_tracking_interval: Optional[int] = DEFAULT_SYSTEM_TRACKING_INT, log_system_params: Optional[bool] = True, - capture_terminal_logs: Optional[bool] = True, train_metric_prefix: Optional[str] = 'train_', val_metric_prefix: Optional[str] = 'val_', test_metric_prefix: Optional[str] = 'test_', @@ -60,9 +54,7 @@ def __init__( self._repo_path = repo self._experiment_name = experiment_name - self._system_tracking_interval = system_tracking_interval self._log_system_params = log_system_params - self._capture_terminal_logs = capture_terminal_logs self._train_metric_prefix = train_metric_prefix self._val_metric_prefix = val_metric_prefix @@ -76,21 +68,15 @@ def __init__( def experiment(self) -> Run: if self._run is None: if self._run_hash: - self._run = Run( - self._run_hash, - repo=self._repo_path, - system_tracking_interval=self._system_tracking_interval, - capture_terminal_logs=self._capture_terminal_logs, - ) + self._run = Run(self._run_hash, repo=self._repo_path) else: - self._run = Run( - repo=self._repo_path, - experiment=self._experiment_name, - system_tracking_interval=self._system_tracking_interval, - log_system_params=self._log_system_params, - capture_terminal_logs=self._capture_terminal_logs, - ) + self._run = Run(repo=self._repo_path) self._run_hash = self._run.hash + if self._experiment_name is not None: + self._run.experiment = self._experiment_name + if self._log_system_params: + self._run.enable_system_monitoring() + return self._run def log_params(self, params: dict): @@ -121,7 +107,7 @@ def log_metrics(self, metrics: Dict[str, float], step: Optional[int] = None): name = name[len(self._val_metric_prefix):] context['subset'] = 'val' context.update(self._context) - self.experiment.track(v, step=step, name=name, context=context) + self.experiment.track_auto(v, step=step, name=name, context=context) @property def save_dir(self) -> str: @@ -136,8 +122,7 @@ def version(self) -> str: return self.experiment.hash def close(self) -> None: - if self._run: - self._run.close() + if self._run is not None: del self._run self._run = None diff --git a/pkgs/aimstack/ml/adapters/pytorch_lightning.py b/pkgs/aimstack/ml/integrations/pytorch_lightning.py similarity index 76% rename from pkgs/aimstack/ml/adapters/pytorch_lightning.py rename to pkgs/aimstack/ml/integrations/pytorch_lightning.py index 479b7f9a6d..c361b91c27 100644 --- a/pkgs/aimstack/ml/adapters/pytorch_lightning.py +++ b/pkgs/aimstack/ml/integrations/pytorch_lightning.py @@ -25,10 +25,9 @@ 'Please install it with command: \n pip install pytorch-lightning' ) -from aim._sdk.run import Run -from aim._sdk.repo import Repo +from aimstack.ml import Run +from aim import Repo from aim._sdk.utils import clean_repo_path, get_aim_repo_name -from aim._ext.system_info import DEFAULT_SYSTEM_TRACKING_INT class AimLogger(Logger): @@ -40,11 +39,8 @@ class AimLogger(Logger): If skipped, default Repo is used. experiment_name (:obj:`str`, optional): Sets Run's `experiment` property. 'default' if not specified. Can be used later to query runs/sequences. - system_tracking_interval (:obj:`int`, optional): Sets the tracking interval in seconds for system usage - metrics (CPU, Memory, etc.). Set to `None` to disable system metrics tracking. log_system_params (:obj:`bool`, optional): Enable/Disable logging of system params such as installed packages, git info, environment variables, etc. - capture_terminal_logs (:obj:`bool`, optional): Enable/Disable terminal stdout logging. train_metric_prefix (:obj:`str`, optional): Training metric prefix. val_metric_prefix (:obj:`str`, optional): validation metric prefix. test_metric_prefix (:obj:`str`, optional): testing metric prefix. @@ -56,9 +52,7 @@ def __init__( self, repo: Optional[str] = None, experiment_name: Optional[str] = None, - system_tracking_interval: Optional[int] = DEFAULT_SYSTEM_TRACKING_INT, log_system_params: Optional[bool] = True, - capture_terminal_logs: Optional[bool] = True, train_metric_prefix: Optional[str] = 'train_', val_metric_prefix: Optional[str] = 'val_', test_metric_prefix: Optional[str] = 'test_', @@ -69,9 +63,7 @@ def __init__( self._repo_path = repo self._experiment_name = experiment_name - self._system_tracking_interval = system_tracking_interval self._log_system_params = log_system_params - self._capture_terminal_logs = capture_terminal_logs self._train_metric_prefix = train_metric_prefix self._val_metric_prefix = val_metric_prefix @@ -98,23 +90,17 @@ def _convert_params(params: Union[Dict[str, Any], Namespace]) -> Dict[str, Any]: def experiment(self) -> Run: if self._run is None: if self._run_hash: - self._run = Run( - self._run_hash, - repo=self._repo_path, - system_tracking_interval=self._system_tracking_interval, - capture_terminal_logs=self._capture_terminal_logs, - ) + self._run = Run(self._run_hash, repo=self._repo_path) if self._run_name is not None: self._run.name = self._run_name else: - self._run = Run( - repo=self._repo_path, - experiment=self._experiment_name, - system_tracking_interval=self._system_tracking_interval, - log_system_params=self._log_system_params, - capture_terminal_logs=self._capture_terminal_logs, - ) + self._run = Run(repo=self._repo_path) self._run_hash = self._run.hash + if self._experiment_name is not None: + self._run.experiment = self._experiment_name + if self._log_system_params: + self._run.enable_system_monitoring() + return self._run @rank_zero_only @@ -157,13 +143,12 @@ def log_metrics(self, metrics: Dict[str, float], step: Optional[int] = None): elif self._val_metric_prefix and name.startswith(self._val_metric_prefix): name = name[len(self._val_metric_prefix):] context['subset'] = 'val' - self.experiment.track(v, name=name, step=step, epoch=epoch, context=context) + self.experiment.track_auto(v, name=name, step=step, epoch=epoch, context=context) @rank_zero_only def finalize(self, status: str = '') -> None: super().finalize(status) - if self._run: - self._run.close() + if self._run is not None: del self._run self._run = None @@ -171,9 +156,11 @@ def __del__(self): self.finalize() @property - def save_dir(self) -> str: - repo_path = clean_repo_path(self._repo_path) or Repo.default_repo_path() - return os.path.join(repo_path, get_aim_repo_name()) + def save_dir(self) -> Optional[str]: + if self._repo_path is not None and not Repo.is_remote_path(self._repo_path): + repo_path = clean_repo_path(self._repo_path) + return os.path.join(repo_path, get_aim_repo_name()) + return None @property def name(self) -> str: diff --git a/pkgs/aimstack/ml/adapters/sb3.py b/pkgs/aimstack/ml/integrations/sb3.py similarity index 70% rename from pkgs/aimstack/ml/adapters/sb3.py rename to pkgs/aimstack/ml/integrations/sb3.py index 67b5a88900..59f2855646 100644 --- a/pkgs/aimstack/ml/adapters/sb3.py +++ b/pkgs/aimstack/ml/integrations/sb3.py @@ -5,8 +5,7 @@ from stable_baselines3.common.logger import KVWriter, Logger from stable_baselines3.common.callbacks import BaseCallback # type: ignore -from aim._sdk.run import Run -from aim._ext.system_info import DEFAULT_SYSTEM_TRACKING_INT +from aimstack.ml import Run logger = logging.getLogger(__name__) @@ -40,7 +39,7 @@ def write( else: context = {'tag': tag} - self.aim_callback.experiment.track( + self.aim_callback.experiment.track_auto( value, key, step=step, context=context ) @@ -54,11 +53,8 @@ class AimCallback(BaseCallback): If skipped, default Repo is used. experiment_name (:obj:`str`, optional): Sets Run's `experiment` property. 'default' if not specified. Can be used later to query runs/sequences. - system_tracking_interval (:obj:`int`, optional): Sets the tracking interval in seconds for system usage - metrics (CPU, Memory, etc.). Set to `None` to disable system metrics tracking. log_system_params (:obj:`bool`, optional): Enable/Disable logging of system params such as installed packages, git info, environment variables, etc. - capture_terminal_logs (:obj:`bool`, optional): Enable/Disable terminal stdout logging. verbose (:obj:`bool`, optional): Enable/Disable verbose """ @@ -66,18 +62,14 @@ def __init__( self, repo: Optional[str] = None, experiment_name: Optional[str] = None, - system_tracking_interval: Optional[int] = DEFAULT_SYSTEM_TRACKING_INT, log_system_params: Optional[bool] = True, - capture_terminal_logs: Optional[bool] = True, verbose: int = 0, ) -> None: super().__init__(verbose) self.repo = repo self.experiment_name = experiment_name - self.system_tracking_interval = system_tracking_interval self.log_system_params = log_system_params - self.capture_terminal_logs = capture_terminal_logs self._run = None self._run_hash = None @@ -110,22 +102,14 @@ def experiment(self): def setup(self, args=None): if not self._run: if self._run_hash: - self._run = Run( - self._run_hash, - repo=self.repo, - system_tracking_interval=self.system_tracking_interval, - log_system_params=self.log_system_params, - capture_terminal_logs=self.capture_terminal_logs, - ) + self._run = Run(self._run_hash, repo=self.repo) else: - self._run = Run( - repo=self.repo, - experiment=self.experiment_name, - system_tracking_interval=self.system_tracking_interval, - log_system_params=self.log_system_params, - capture_terminal_logs=self.capture_terminal_logs, - ) + self._run = Run(repo=self.repo) self._run_hash = self._run.hash + if self.experiment_name is not None: + self._run.experiment = self.experiment_name + if self.log_system_params: + self._run.enable_system_monitoring() # Log config parameters if args: diff --git a/pkgs/aimstack/ml/adapters/tensorflow.py b/pkgs/aimstack/ml/integrations/tensorflow.py similarity index 58% rename from pkgs/aimstack/ml/adapters/tensorflow.py rename to pkgs/aimstack/ml/integrations/tensorflow.py index c5d0ee9915..4ad44bd62d 100644 --- a/pkgs/aimstack/ml/adapters/tensorflow.py +++ b/pkgs/aimstack/ml/integrations/tensorflow.py @@ -1,8 +1,7 @@ from typing import Optional -from aim._sdk.run import Run -from aimstack.ml.adapters.keras_mixins import TrackerKerasCallbackMetricsEpochEndMixin -from aim._ext.system_info import DEFAULT_SYSTEM_TRACKING_INT +from aimstack.ml import Run +from aimstack.ml.integrations.keras_mixins import TrackerKerasCallbackMetricsEpochEndMixin try: from tensorflow.keras.callbacks import Callback @@ -22,8 +21,6 @@ class AimCallback(TrackerKerasCallbackMetricsEpochEndMixin, Callback): If skipped, default Repo is used. experiment_name (:obj:`str`, optional): Sets Run's `experiment` property. 'default' if not specified. Can be used later to query runs/sequences. - system_tracking_interval (:obj:`int`, optional): Sets the tracking interval in seconds for system usage - metrics (CPU, Memory, etc.). Set to `None` to disable system metrics tracking. log_system_params (:obj:`bool`, optional): Enable/Disable logging of system params such as installed packages, git info, environment variables, etc. capture_terminal_logs (:obj:`bool`, optional): Enable/Disable terminal stdout logging. @@ -33,30 +30,19 @@ def __init__( self, repo: Optional[str] = None, experiment_name: Optional[str] = None, - system_tracking_interval: Optional[int] = DEFAULT_SYSTEM_TRACKING_INT, log_system_params: Optional[bool] = True, capture_terminal_logs: Optional[bool] = True, ): super(Callback, self).__init__() - self._system_tracking_interval = system_tracking_interval self._log_system_params = log_system_params self._capture_terminal_logs = capture_terminal_logs - if repo is None and experiment_name is None: - self._run = Run( - system_tracking_interval=self._system_tracking_interval, - log_system_params=self._log_system_params, - capture_terminal_logs=self._capture_terminal_logs, - ) - else: - self._run = Run( - repo=repo, - experiment=experiment_name, - system_tracking_interval=self._system_tracking_interval, - log_system_params=self._log_system_params, - capture_terminal_logs=self._capture_terminal_logs, - ) + self._run = Run(repo=repo) + if experiment_name is not None: + self._run.experiment = experiment_name + if log_system_params: + self._run.enable_system_monitoring() self._run_hash = self._run.hash self._repo_path = repo @@ -64,12 +50,9 @@ def __init__( @property def experiment(self) -> Run: if not self._run: - self._run = Run( - self._run_hash, - repo=self._repo_path, - system_tracking_interval=self._system_tracking_interval, - capture_terminal_logs=self._capture_terimanl_logs, - ) + self._run = Run(self._run_hash, repo=self._repo_path) + if self._log_system_params: + self._run.enable_system_monitoring() return self._run @classmethod @@ -83,8 +66,7 @@ def metrics( return cls(repo, experiment, run) def close(self) -> None: - if self._run: - self._run.close() + if self._run is not None: del self._run self._run = None diff --git a/pkgs/aimstack/ml/adapters/xgboost.py b/pkgs/aimstack/ml/integrations/xgboost.py similarity index 64% rename from pkgs/aimstack/ml/adapters/xgboost.py rename to pkgs/aimstack/ml/integrations/xgboost.py index 58a6c17841..8a10a32cbe 100644 --- a/pkgs/aimstack/ml/adapters/xgboost.py +++ b/pkgs/aimstack/ml/integrations/xgboost.py @@ -1,7 +1,6 @@ from typing import Optional -from aim._ext.system_info import DEFAULT_SYSTEM_TRACKING_INT -from aim._sdk.run import Run +from aimstack.ml import Run try: from xgboost.callback import TrainingCallback @@ -21,27 +20,20 @@ class AimCallback(TrainingCallback): If skipped, default Repo is used. experiment_name (:obj:`str`, optional): Sets Run's `experiment` property. 'default' if not specified. Can be used later to query runs/sequences. - system_tracking_interval (:obj:`int`, optional): Sets the tracking interval in seconds for system usage - metrics (CPU, Memory, etc.). Set to `None` to disable system metrics tracking. log_system_params (:obj:`bool`, optional): Enable/Disable logging of system params such as installed packages, git info, environment variables, etc. - capture_terminal_logs (:obj:`bool`, optional): Enable/Disable terminal stdout logging. """ def __init__( self, repo: Optional[str] = None, experiment_name: Optional[str] = None, - system_tracking_interval: Optional[int] = DEFAULT_SYSTEM_TRACKING_INT, log_system_params: Optional[bool] = True, - capture_terminal_logs: Optional[bool] = True, ): super().__init__() self._repo_path = repo self._experiment = experiment_name - self._system_tracking_interval = system_tracking_interval self._log_system_params = log_system_params - self._capture_terminal_logs = capture_terminal_logs self._run = None self._run_hash = None @@ -55,21 +47,14 @@ def setup(self): if self._run: return if self._run_hash: - self._run = Run( - self._run_hash, - repo=self._repo_path, - system_tracking_interval=self._system_tracking_interval, - capture_terminal_logs=self._capture_terminal_logs, - ) + self._run = Run(self._run_hash, repo=self._repo_path) else: - self._run = Run( - repo=self._repo_path, - experiment=self._experiment, - system_tracking_interval=self._system_tracking_interval, - log_system_params=self._log_system_params, - capture_terminal_logs=self._capture_terminal_logs, - ) + self._run = Run(repo=self._repo_path) self._run_hash = self._run.hash + if self._experiment is not None: + self._run.experiment = self._experiment + if self._log_system_params: + self._run.enable_system_monitoring() def before_training(self, model): self.setup() @@ -88,11 +73,11 @@ def after_iteration(self, model, epoch: int, evals_log) -> bool: else: score = log[-1] - self._run.track( + self._run.track_auto( score, step=0, name=metric_name, context={'stdv': False} ) if stdv is not None: - self._run.track( + self._run.track_auto( score, step=0, name=metric_name, context={'stdv': True} ) @@ -103,8 +88,7 @@ def after_training(self, model): return model def close(self): - if self._run: - self._run.close() + if self._run is not None: del self._run self._run = None diff --git a/pkgs/aimstack/ml/models/__init__.py b/pkgs/aimstack/ml/loggers/__init__.py similarity index 100% rename from pkgs/aimstack/ml/models/__init__.py rename to pkgs/aimstack/ml/loggers/__init__.py diff --git a/pkgs/aimstack/ml/models/deeplake_dataset.py b/pkgs/aimstack/ml/loggers/deeplake_dataset.py similarity index 99% rename from pkgs/aimstack/ml/models/deeplake_dataset.py rename to pkgs/aimstack/ml/loggers/deeplake_dataset.py index f729d122b5..cdf2dc367b 100644 --- a/pkgs/aimstack/ml/models/deeplake_dataset.py +++ b/pkgs/aimstack/ml/loggers/deeplake_dataset.py @@ -3,7 +3,7 @@ import logging from deeplake.util.exceptions import ReadOnlyModeError -from aim._sdk.record import Record +from aim import Record logger = logging.getLogger(__name__) diff --git a/pkgs/aimstack/ml/models/dvc_metadata.py b/pkgs/aimstack/ml/loggers/dvc_metadata.py similarity index 98% rename from pkgs/aimstack/ml/models/dvc_metadata.py rename to pkgs/aimstack/ml/loggers/dvc_metadata.py index 12bb55fc56..6c73730c6a 100644 --- a/pkgs/aimstack/ml/models/dvc_metadata.py +++ b/pkgs/aimstack/ml/loggers/dvc_metadata.py @@ -2,7 +2,7 @@ from pathlib import Path import yaml -from aim._sdk.record import Record +from aim import Record try: from dvc.repo import Repo diff --git a/pkgs/aimstack/ml/models/hf_datasets_metadata.py b/pkgs/aimstack/ml/loggers/hf_datasets_metadata.py similarity index 99% rename from pkgs/aimstack/ml/models/hf_datasets_metadata.py rename to pkgs/aimstack/ml/loggers/hf_datasets_metadata.py index 342489ebd1..1b215c3b25 100644 --- a/pkgs/aimstack/ml/models/hf_datasets_metadata.py +++ b/pkgs/aimstack/ml/loggers/hf_datasets_metadata.py @@ -1,7 +1,7 @@ from datasets import Dataset, DatasetDict from logging import getLogger -from aim._sdk.record import Record +from aim import Record logger = getLogger(__name__) diff --git a/pkgs/aimstack/ml/models/hub_dataset.py b/pkgs/aimstack/ml/loggers/hub_dataset.py similarity index 97% rename from pkgs/aimstack/ml/models/hub_dataset.py rename to pkgs/aimstack/ml/loggers/hub_dataset.py index 079e72b72e..e0af742da5 100644 --- a/pkgs/aimstack/ml/models/hub_dataset.py +++ b/pkgs/aimstack/ml/loggers/hub_dataset.py @@ -1,4 +1,4 @@ -from aim._sdk.record import Record +from aim import Record import hub diff --git a/pkgs/aimstack/ml/loggers/run.py b/pkgs/aimstack/ml/loggers/run.py new file mode 100644 index 0000000000..fdbbe88ac3 --- /dev/null +++ b/pkgs/aimstack/ml/loggers/run.py @@ -0,0 +1,6 @@ +from aimstack.asp import Run as BaseRun +from aim import Property + + +class Run(BaseRun): + experiment = Property(default='default') diff --git a/pkgs/aimstack/ml/training_flow.py b/pkgs/aimstack/ml/training_flow.py index d1682f4ce7..e564bbdb0c 100644 --- a/pkgs/aimstack/ml/training_flow.py +++ b/pkgs/aimstack/ml/training_flow.py @@ -1,6 +1,6 @@ from typing import Dict, Any -from aimstack.asp import Run +from aimstack.ml import Run from aimcore.callbacks import Caller, event diff --git a/src/aimcore/cli/conatiners/commands.py b/src/aimcore/cli/conatiners/commands.py index 7ed957eea5..03d68bebfd 100644 --- a/src/aimcore/cli/conatiners/commands.py +++ b/src/aimcore/cli/conatiners/commands.py @@ -19,7 +19,7 @@ def containers(ctx, repo): @click.pass_context def list_containers(ctx): """List Containers available in Repo.""" - #TODO [MV]: add more useful information + # TODO [MV]: add more useful information repo_path = ctx.obj['repo'] if not Repo.is_remote_path(repo_path): if not Repo.exists(repo_path): diff --git a/src/aimcore/cli/conatiners/utils.py b/src/aimcore/cli/conatiners/utils.py index e8a44447b0..8d32755732 100644 --- a/src/aimcore/cli/conatiners/utils.py +++ b/src/aimcore/cli/conatiners/utils.py @@ -23,4 +23,3 @@ def match_runs(repo: 'Repo', hashes: List[str]) -> List[str]: matched_hashes.add(run_hash) return list(matched_hashes) - diff --git a/src/aimcore/cli/migrate/utils.py b/src/aimcore/cli/migrate/utils.py index bda43ab264..54c90e0572 100644 --- a/src/aimcore/cli/migrate/utils.py +++ b/src/aimcore/cli/migrate/utils.py @@ -15,7 +15,7 @@ logger = logging.getLogger(__name__) RUN_DATA_QUERY_TEMPLATE = """ -SELECT +SELECT run.hash as hash, run.name as name, run.description as description, @@ -24,7 +24,7 @@ json_group_array(tag.name) as tags, {select_notes} as notes FROM - run + run LEFT OUTER JOIN experiment ON run.experiment_id = experiment.id {notes_join_clause} LEFT OUTER JOIN run_tag ON run.id = run_tag.run_id LEFT JOIN tag ON run_tag.tag_id = tag.id @@ -48,7 +48,7 @@ } -def migrate_v1_sequence_data(run: Run, trace_data_tree: TreeView, length: int, item_type: str, name: str, context: Dict): +def migrate_v1_sequence_data(run: Run, trace_tree: TreeView, length: int, item_type: str, name: str, context: Dict): if name.startswith('__system'): seq = run.sequences.typed_sequence(SystemMetric, name, context) elif item_type == 'aim.log_line': @@ -59,9 +59,9 @@ def migrate_v1_sequence_data(run: Run, trace_data_tree: TreeView, length: int, i logger.warning(f'Unknown type of sequence element \'{item_type}\'. Skipping.') return seq = get_seq_method(run, name, context) - trace_iter = zip(trace_data_tree.subtree('val').items(), - trace_data_tree.subtree('time').items(), - trace_data_tree.subtree('epoch').items()) + trace_iter = zip(trace_tree.subtree('val').items(), + trace_tree.subtree('time').items(), + trace_tree.subtree('epoch').items()) trace_iter = tqdm.tqdm(trace_iter, leave=False, total=length) context_str = str(context) if len(context_str) > 20: @@ -142,8 +142,8 @@ def table_exists(tbl): try: import sqlite3 except ModuleNotFoundError: - if not click.confirm(f'Missing package \'sqlite3\'. Cannot migrate Run experiment, tags and notes info. ' - f'Would you like to proceed?'): + if not click.confirm('Missing package \'sqlite3\'. Cannot migrate Run experiment, tags and notes info. ' + 'Would you like to proceed?'): exit(0) return {} else: diff --git a/src/aimcore/cli/package/commands.py b/src/aimcore/cli/package/commands.py index 50d914a61f..1dca018b6d 100644 --- a/src/aimcore/cli/package/commands.py +++ b/src/aimcore/cli/package/commands.py @@ -81,7 +81,7 @@ def sync_package(name, repo): exit(1) else: if repo and not Repo.is_remote_path(repo): - click.echo(f'Failed to run package sync. \'--repo\' must point to Aim remote server path.') + click.echo('Failed to run package sync. \'--repo\' must point to Aim remote server path.') exit(1) repo_inst = Repo.from_path(repo) if repo else Repo.default() diff --git a/src/aimcore/cli/server/commands.py b/src/aimcore/cli/server/commands.py index faf966c1ef..173a9a5282 100644 --- a/src/aimcore/cli/server/commands.py +++ b/src/aimcore/cli/server/commands.py @@ -3,7 +3,6 @@ from aimcore.cli.utils import set_log_level, start_uvicorn_app from aim._sdk.repo import Repo -from aim._sdk.package_utils import Package from aimcore.transport.config import ( AIM_SERVER_DEFAULT_HOST, AIM_SERVER_DEFAULT_PORT, diff --git a/src/aimcore/cli/ui/commands.py b/src/aimcore/cli/ui/commands.py index 9ea5089b3d..54dafcb194 100644 --- a/src/aimcore/cli/ui/commands.py +++ b/src/aimcore/cli/ui/commands.py @@ -126,7 +126,8 @@ def ui(dev, host, port, workers, uds, import aimstack import aimcore - reload_dirs = [os.path.dirname(aim.__file__), os.path.dirname(aimcore.__file__), os.path.dirname(aimstack.__file__), dev_package_dir] + reload_dirs = [os.path.dirname(aim.__file__), os.path.dirname(aimcore.__file__), + os.path.dirname(aimstack.__file__), dev_package_dir] else: reload_dirs = [] diff --git a/src/aimcore/web/api/reports/views.py b/src/aimcore/web/api/reports/views.py index 544300dccb..bbfc152655 100644 --- a/src/aimcore/web/api/reports/views.py +++ b/src/aimcore/web/api/reports/views.py @@ -1,10 +1,8 @@ -from aimcore.web.api.utils import APIRouter +from aimcore.web.api.utils import APIRouter # wrapper for fastapi.APIRouter from aimcore.web.api.reports.serializers import report_response_serializer from aimcore.web.api.reports.models import Report -from typing import List from sqlalchemy.orm import Session from fastapi import Depends, HTTPException -from aimcore.web.api.utils import APIRouter # wrapper for fastapi.APIRouter from aimcore.web.api.reports.pydantic_models import ( diff --git a/src/aimcore/web/api/runs/utils.py b/src/aimcore/web/api/runs/utils.py index 79a3c50471..774cb55ece 100644 --- a/src/aimcore/web/api/runs/utils.py +++ b/src/aimcore/web/api/runs/utils.py @@ -394,5 +394,3 @@ async def run_log_records_streamer(run: Run, record_range: str) -> bytes: yield collect_streamable_data(encoded_tree) except asyncio.CancelledError: pass - - diff --git a/src/python/aim/_sdk/container.py b/src/python/aim/_sdk/container.py index f184f8620b..bdae6d1539 100644 --- a/src/python/aim/_sdk/container.py +++ b/src/python/aim/_sdk/container.py @@ -228,6 +228,10 @@ def __setitem__(self, key, value): self._attrs_tree[key] = value self._meta_attrs_tree.merge(key, value) + def set(self, key, value, strict: bool): + self._attrs_tree.set(key, value, strict) + self._meta_attrs_tree.set(key, value, strict) + def __getitem__(self, key): return self._attrs_tree.collect(key, strict=True) diff --git a/src/python/aim/_sdk/interfaces/container.py b/src/python/aim/_sdk/interfaces/container.py index 64d89ff51c..9826126d25 100644 --- a/src/python/aim/_sdk/interfaces/container.py +++ b/src/python/aim/_sdk/interfaces/container.py @@ -23,6 +23,10 @@ def __delitem__(self, key): def get(self, key, default: Any = None, strict: bool = False): ... + @abstractmethod + def set(self, key, value: Any, strict: bool): + ... + @property @abstractmethod def sequences(self) -> 'SequenceCollection': diff --git a/src/python/aim/_sdk/repo.py b/src/python/aim/_sdk/repo.py index 03e06ab016..0a240abc2e 100644 --- a/src/python/aim/_sdk/repo.py +++ b/src/python/aim/_sdk/repo.py @@ -419,7 +419,7 @@ def copy_trees(): for ctx_idx in source_meta_container_tree_collected['sequences'].keys(): for seq_name in source_meta_container_tree_collected['sequences'][ctx_idx].keys(): - seq_typename = source_meta_container_tree_collected['sequences'][ctx_idx][seq_name][KeyNames.INFO_PREFIX][KeyNames.SEQUENCE_TYPE] + seq_typename = source_meta_container_tree_collected['sequences'][ctx_idx][seq_name][KeyNames.INFO_PREFIX][KeyNames.SEQUENCE_TYPE] # noqa for typename in seq_typename.split('->'): dest_meta_tree[(KeyNames.SEQUENCES, typename, ctx_idx, seq_name)] = 1 diff --git a/src/python/aim/utils.py b/src/python/aim/utils.py index 3460fa4ddc..1b8e4ff431 100644 --- a/src/python/aim/utils.py +++ b/src/python/aim/utils.py @@ -1,9 +1,12 @@ import random -from typing import Optional, Dict, Type +from typing import Optional, Dict, Type, TYPE_CHECKING from aim import Sequence, Container from aim._core.storage.encoding import encode_path, decode_path +if TYPE_CHECKING: + from aim import Repo + def _process_sequence_values(repo: 'Repo', values_list: list, steps_list: list, sequence: Sequence) -> list: # TODO V4: Move blobs uri handling to dump