diff --git a/README.md b/README.md index 549a2230c..0a3e9c704 100644 --- a/README.md +++ b/README.md @@ -23,8 +23,7 @@ Here is the list of notebooks in this repo: | 3 | XGBoost | Agaricus (Scala) | Uses XGBoost classifier function to create model that can accurately differentiate between edible and poisonous mushrooms with the [agaricus dataset](https://archive.ics.uci.edu/ml/datasets/mushroom) | 4 | XGBoost | Mortgage (Scala) | End-to-end ETL + XGBoost example to predict mortgage default with [Fannie Mae Single-Family Loan Performance Data](https://capitalmarkets.fanniemae.com/credit-risk-transfer/single-family-credit-risk-transfer/fannie-mae-single-family-loan-performance-data) | 5 | XGBoost | Taxi (Scala) | End-to-end ETL + XGBoost example to predict taxi trip fare amount with [NYC taxi trips data set](https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page) -| 6 | ML/DL | Criteo Training | ETL and deep learning training of the Criteo 1TB Click Logs dataset -| 7 | ML/DL | PCA End-to-End | Spark MLlib based PCA example to train and transform with a synthetic dataset +| 6 | ML/DL | PCA End-to-End | Spark MLlib based PCA example to train and transform with a synthetic dataset Here is the list of Apache Spark applications (Scala and PySpark) that can be built for running on GPU with RAPIDS Accelerator in this repo: diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/Dockerfile b/examples/ML+DL-Examples/Spark-DL/criteo_train/Dockerfile deleted file mode 100644 index 0e56925b1..000000000 --- a/examples/ML+DL-Examples/Spark-DL/criteo_train/Dockerfile +++ /dev/null @@ -1,229 +0,0 @@ -# -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -ARG IMAGE=nvcr.io/nvidia/tensorflow:21.03-tf2-py3 -FROM ${IMAGE} -ENV CUDA_SHORT_VERSION=11.2 - -SHELL ["/bin/bash", "-c"] -ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/lib:/repos/dist/lib - -ENV DEBIAN_FRONTEND=noninteractive - -ARG RELEASE=true -ARG RMM_VER=v21.10.00 -ARG CUDF_VER=v21.10.00 -ARG NVTAB_VER=v0.6.0 -ARG SM="60;61;70;75;80" - -ENV CUDA_HOME=/usr/local/cuda -ENV CUDA_PATH=$CUDA_HOME -ENV CUDA_CUDA_LIBRARY=${CUDA_HOME}/lib64/stubs -ENV PATH=${CUDA_HOME}/lib64/:${PATH}:${CUDA_HOME}/bin - -# Build env variables for rmm -ENV INSTALL_PREFIX=/usr - -RUN apt update -y --fix-missing && \ - apt upgrade -y && \ - apt install -y --no-install-recommends software-properties-common && \ - apt update -y --fix-missing - -RUN apt install -y --no-install-recommends \ - git \ - libboost-all-dev \ - python3.8-dev \ - build-essential \ - autoconf \ - bison \ - flex \ - libboost-filesystem-dev \ - libboost-system-dev \ - libboost-regex-dev \ - libjemalloc-dev \ - wget \ - libssl-dev \ - protobuf-compiler \ - clang-format \ - aptitude \ - numactl \ - libnuma-dev \ - libaio-dev \ - libibverbs-dev \ - libtool && \ - apt-get autoremove -y && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - #update-alternatives --install /usr/bin/python python /usr/bin/python3.8 1 && \ - #wget https://bootstrap.pypa.io/get-pip.py && \ - #python get-pip.py - -# Install cmake -RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null && \ - apt-add-repository 'deb https://apt.kitware.com/ubuntu/ focal main' && \ - apt-get update && \ - apt-get install -y cmake - -# Install arrow from source -ENV ARROW_HOME=/usr/local -RUN git clone --branch apache-arrow-4.0.1 --recurse-submodules https://github.com/apache/arrow.git build-env && \ - pushd build-env && \ - export PARQUET_TEST_DATA="${PWD}/cpp/submodules/parquet-testing/data" && \ - export ARROW_TEST_DATA="${PWD}/testing/data" && \ - pip install -r python/requirements-build.txt && \ - mkdir cpp/release && \ - pushd cpp/release && \ - cmake -DCMAKE_INSTALL_PREFIX=${ARROW_HOME} \ - -DCMAKE_INSTALL_LIBDIR=lib \ - -DCMAKE_LIBRARY_PATH=${CUDA_CUDA_LIBRARY} \ - -DARROW_FLIGHT=ON \ - -DARROW_GANDIVA=OFF \ - -DARROW_ORC=ON \ - -DARROW_WITH_BZ2=ON \ - -DARROW_WITH_ZLIB=ON \ - -DARROW_WITH_ZSTD=ON \ - -DARROW_WITH_LZ4=ON \ - -DARROW_WITH_SNAPPY=ON \ - -DARROW_WITH_BROTLI=ON \ - -DARROW_PARQUET=ON \ - -DARROW_PYTHON=ON \ - -DARROW_PLASMA=ON \ - -DARROW_BUILD_TESTS=ON \ - -DARROW_CUDA=ON \ - -DARROW_DATASET=ON \ - .. && \ - make -j$(nproc) && \ - make install && \ - popd && \ - pushd python && \ - export PYARROW_WITH_PARQUET=ON && \ - export PYARROW_WITH_CUDA=ON && \ - export PYARROW_WITH_ORC=ON && \ - export PYARROW_WITH_DATASET=ON && \ - python setup.py build_ext --build-type=release bdist_wheel && \ - pip install dist/*.whl && \ - popd && \ - popd && \ - rm -rf build-env - - -# Install rmm from source -RUN git clone https://github.com/rapidsai/rmm.git build-env && cd build-env/ && \ - if [ "$RELEASE" == "true" ] && [ ${RMM_VER} != "vnightly" ] ; then git fetch --all --tags && git checkout tags/${RMM_VER}; else git checkout main; fi; \ - sed -i '/11.2/ a "11.4": "11.x",' python/setup.py && \ - cd ..; \ - pushd build-env && \ - ./build.sh librmm && \ - pip install python/. && \ - popd && \ - rm -rf build-env - - - -# Build env for CUDF build -RUN git clone https://github.com/rapidsai/cudf.git build-env && cd build-env/ && \ - if [ "$RELEASE" == "true" ] && [ ${CUDF_VER} != "vnightly" ] ; then git fetch --all --tags && git checkout tags/${CUDF_VER}; else git checkout main; fi; \ - git submodule update --init --recursive && \ - cd .. && \ - pushd build-env && \ - export CUDF_HOME=${PWD} && \ - export CUDF_ROOT=${PWD}/cpp/build/ && \ - export CMAKE_LIBRARY_PATH=${CUDA_CUDA_LIBRARY} && \ - ./build.sh libcudf cudf dask_cudf && \ - protoc -I=python/cudf/cudf/utils/metadata --python_out=/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata python/cudf/cudf/utils/metadata/orc_column_statistics.proto && \ - popd && \ - rm -rf build-env - -RUN apt-get update -y && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - vim gdb git wget unzip tar python3.8-dev \ - zlib1g-dev lsb-release clang-format libboost-all-dev \ - openssl curl zip\ - slapd && \ - rm -rf /var/lib/apt/lists/* - -ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION='python' - - -RUN pip install pandas sklearn ortools nvtx-plugins pydot && \ - pip cache purge - -# tf-nightly for performance test -# more details: https://github.com/tensorflow/tensorflow/issues/44194 -RUN pip uninstall tensorflow -y; pip install tf-nightly==2.7.0.dev20210722 -RUN pip uninstall keras-nightly -y; pip install keras-nightly==2.7.0.dev2021072200 - - -RUN mkdir -p /usr/local/nvidia/lib64 && \ - ln -s /usr/local/cuda/lib64/libcusolver.so /usr/local/nvidia/lib64/libcusolver.so.10 - -RUN pip install pybind11 -SHELL ["/bin/bash", "-c"] - -# prepare nccl -RUN apt remove -y libnccl2 libnccl-dev -RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin \ - && mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600 \ - && apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/7fa2af80.pub \ - && add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /" \ - && apt-get update \ - && apt install libnccl2=2.8.4-1+cuda11.2 libnccl-dev=2.8.4-1+cuda11.2 - -# install Horovod -RUN pip uninstall horovod -y -RUN HOROVOD_WITH_MPI=1 HOROVOD_WITH_TENSORFLOW=1 HOROVOD_GPU_OPERATIONS=NCCL \ - pip install horovod[spark] --no-cache-dir - -# Install NVTabular -RUN git clone https://github.com/NVIDIA/NVTabular.git /nvtabular/ && \ - cd /nvtabular/; if [ "$RELEASE" == "true" ] && [ ${NVTAB_VER} != "vnightly" ] ; then git fetch --all --tags && git checkout tags/${NVTAB_VER}; else git checkout main; fi; \ - python setup.py develop --user; - - -RUN pip install pynvml pytest graphviz sklearn scipy matplotlib -RUN pip install nvidia-pyindex; pip install tritonclient[all] grpcio-channelz -RUN pip install nvtx mpi4py==3.0.3 cupy-cuda112 cachetools typing_extensions fastavro - -RUN apt-get update; apt-get install -y graphviz - -RUN pip uninstall numpy -y; pip install numpy -RUN pip install dask==2021.04.1 distributed==2021.04.1 dask-cuda -RUN pip install dask[dataframe]==2021.04.1 -RUN pip uninstall pandas -y; pip install pandas==1.1.5 -RUN echo $(du -h --max-depth=1 /) - - -# install spark-3.1.2-bin-hadoop3.2 -RUN wget \ - https://mirror-hk.koddos.net/apache/spark/spark-3.1.2/spark-3.1.2-bin-hadoop3.2.tgz \ - && tar -xzf spark-3.1.2-bin-hadoop3.2.tgz -C /opt/ \ - && rm spark-3.1.2-bin-hadoop3.2.tgz - -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get install openjdk-8-jdk openjdk-8-jre lsb-release -y --allow-downgrades --allow-change-held-packages --no-install-recommends -ENV JAVA_HOME /usr/lib/jvm/java-1.8.0-openjdk-amd64 -ENV PATH $PATH:/usr/lib/jvm/java-1.8.0-openjdk-amd64/jre/bin:/usr/lib/jvm/java-1.8.0-openjdk-amd64/bin - -# add spark env to conf -ADD spark-env.sh /opt/spark-3.1.2-bin-hadoop3.2/conf/ - -ADD start-spark.sh /workspace/ -ADD submit.sh /workspace/ -ADD criteo_keras.py /workspace/ - -HEALTHCHECK NONE -ENTRYPOINT [] -CMD ["/bin/bash"] diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/Dockerfile.conda_db b/examples/ML+DL-Examples/Spark-DL/criteo_train/Dockerfile.conda_db deleted file mode 100644 index 475b99149..000000000 --- a/examples/ML+DL-Examples/Spark-DL/criteo_train/Dockerfile.conda_db +++ /dev/null @@ -1,78 +0,0 @@ -# -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -FROM nvidia/cuda:11.4.3-cudnn8-devel-ubuntu20.04 - -ENV DEBIAN_FRONTEND=noninteractive -# Disable NVIDIA repos to prevent accidental upgrades. -RUN cd /etc/apt/sources.list.d && \ - mv cuda.list cuda.list.disabled - -# See https://github.com/databricks/containers/blob/master/ubuntu/minimal/Dockerfile -RUN apt-get update && \ - apt-get install --yes --no-install-recommends \ - openjdk-8-jdk \ - openjdk-8-jre \ - lsb-release \ - iproute2 \ - bash \ - sudo \ - coreutils \ - procps \ - wget && \ - /var/lib/dpkg/info/ca-certificates-java.postinst configure && \ - rm -rf /var/lib/apt/lists/* - - -ENV PATH /databricks/conda/bin:$PATH - -RUN wget -q https://repo.continuum.io/miniconda/Miniconda3-py38_4.9.2-Linux-x86_64.sh -O miniconda.sh && \ - bash miniconda.sh -b -p /databricks/conda && \ - rm miniconda.sh && \ - # Source conda.sh for all login and interactive shells. - ln -s /databricks/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ - echo ". /etc/profile.d/conda.sh" >> ~/.bashrc && \ - # Set always_yes for non-interactive shells. - conda config --system --set always_yes True && \ - conda clean --all - -# install openjdk8, cmake, openmpi openmpi-mpicc -RUN conda install cmake openmpi openmpi-mpicc -y -ENV JAVA_HOME /usr/lib/jvm/java-1.8.0-openjdk-amd64 -ENV PATH $PATH:/usr/lib/jvm/java-1.8.0-openjdk-amd64/jre/bin:/usr/lib/jvm/java-1.8.0-openjdk-amd64/bin - -RUN conda install -y -c nvidia -c rapidsai -c numba -c conda-forge nvtabular=1.2.2 python=3.8 cudatoolkit=11.4 scikit-learn - -RUN pip uninstall tensorflow -y; pip install tensorflow-gpu==2.8 -RUN pip install torch==1.11.0+cu115 torchvision==0.12.0+cu115 torchaudio===0.11.0+cu115 -f https://download.pytorch.org/whl/cu115/torch_stable.html -RUN rm -rf /databricks/conda/include/google -RUN HOROVOD_WITH_MPI=1 HOROVOD_GPU_OPERATIONS=NCCL HOROVOD_WITH_TENSORFLOW=1 HOROVOD_WITH_PYTORCH=1 \ - pip install horovod[spark] --no-cache-dir -RUN pip install pynvml jupyter matplotlib - - -RUN apt-get update && apt-get install wget openssh-client openssh-server \ - -y --allow-downgrades --allow-change-held-packages --no-install-recommends -RUN useradd --create-home --shell /bin/bash --groups sudo ubuntu - -ENV PYSPARK_PYTHON=/databricks/conda/bin/python -ENV USER root -ENV DEFAULT_DATABRICKS_ROOT_CONDA_ENV=base -ENV DATABRICKS_ROOT_CONDA_ENV=base -# disable gds due to errors -ENV LIBCUDF_CUFILE_POLICY=OFF -# required by DB -RUN pip install virtualenv -RUN pip install adlfs diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/README.md b/examples/ML+DL-Examples/Spark-DL/criteo_train/README.md deleted file mode 100644 index 083894055..000000000 --- a/examples/ML+DL-Examples/Spark-DL/criteo_train/README.md +++ /dev/null @@ -1,101 +0,0 @@ -# Criteo - -ETL and deep learning training of the Criteo 1TB Click Logs dataset. Users can prepare their dataset accordingly. - -_Please note: The following demo is dedicated for DGX-2 machine(with V100 GPUs)._ We optimized the whole workflow on DGX-2 and it's not guaranteed that it can run successfully on other type of machines. - -## Dataset - -The dataset used here is from Criteo clicklog dataset. -It's preprocessed by [DLRM](https://github.com/NVIDIA/DeepLearningExamples/tree/master/TensorFlow2/Recommendation/DLRM_and_DCNv2/preproc) -ETL job on Spark. We also provide a small size sample data in sample_data folder. -All 40 columns(1 label + 39 features) are already numeric. - -In the following parts, we assume that the data are mounted as docker volumn at `/data/parquet` - -## Run Criteo example benchmark using Dockerfile - -1. Build the docker image -``` -nvidia-docker build -f Dockerfile -t nvspark/tf-hvd-train:0.1 . -``` - -2. Enter into it (also mount necessary dataset volume and devices) -``` - nvidia-docker run \ - --network host \ - --device /dev/infiniband \ - --privileged \ - -v /raid/spark-team/criteo/parquet:/data/parquet \ - -it nvspark/tf-hvd-train:0.1 bash -``` - -3. when you are inside the container -``` -cd /workspace -# start standalone Spark -./start-spark.sh - -# start training -./submit.sh -``` - -## Notebook demo - -We also provide a Notebook demo for quick test, user can set it up by the following command: - -```bash -SPARK_HOME= $PATH_TO_SPARK_HOME -SPARK_URL=spark://$SPARK_MASTER_IP:7077 -export PYSPARK_DRIVER_PYTHON=jupyter -export PYSPARK_DRIVER_PYTHON_OPTS='notebook' - -$SPARK_HOME/bin/pyspark --master $SPARK_URL --deploy-mode client \ ---driver-memory 20G \ ---executor-memory 50G \ ---executor-cores 6 \ ---conf spark.cores.max=96 \ ---conf spark.task.cpus=6 \ ---conf spark.locality.wait=0 \ ---conf spark.yarn.maxAppAttempts=1 \ ---conf spark.sql.shuffle.partitions=4 \ ---conf spark.sql.files.maxPartitionBytes=1024m \ ---conf spark.sql.warehouse.dir=$OUT \ ---conf spark.task.resource.gpu.amount=0.08 \ ---conf spark.executor.resource.gpu.amount=1 \ ---conf spark.executor.resource.gpu.discoveryScript=./getGpusResources.sh \ ---files $SPARK_HOME/examples/src/main/scripts/getGpusResources.sh - -``` - -## Note: - -If you want to try in a node with only 1 GPU, please modify the GPU number per worker in `$SPARK_HOME/conf/spark-env.sh` before you launch spark workers becasue the docker image is targeted for DGX-2 with 16 GPUs - -## file decription: - -`Dockerfile`: consistent environment, main components are build from source directly. But this file take a while to build an image. - -`spark-env.sh`: Spark config changes, we set 16 GPU for a work for DGX-2 box. It is in SPARK_HOME/conf/ - -`start-spark.sh`: launch Spark in standalone mode. In a DGX-2 box, it will launch Spark Master and a Spark Worker which contains 16 GPUs - -`submit.sh`: commands used to submit the job - -`criteo_keras.py`: Python script to train the Criteo model. Please run `python criteo_keras.py --help` to see parameter details - -## workspace folder in docker container: - -`/workspace/` - -## Run in Databricks Runtime -Some extra packages are required to run the example, we provide a Dockerfile [Dockerfile.conda_db](Dockerfile.conda_db) -to use [Customize containers with Databricks Container Services](https://docs.databricks.com/clusters/custom-containers.html) -in Databricks cloud environment. - -To use it: -1. build the docker image locally -2. push the image to a DB supported Docker hub. -3. set the image url in DB cluster setup page. - -![microbenchmark-speedup](/docs/img/guides/criteo-perf.png) \ No newline at end of file diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/create-conda-env.sh b/examples/ML+DL-Examples/Spark-DL/criteo_train/create-conda-env.sh deleted file mode 100755 index f3ba315f3..000000000 --- a/examples/ML+DL-Examples/Spark-DL/criteo_train/create-conda-env.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash --login - -export HOROVOD_CUDA_HOME=/usr/local/cuda -export HOROVOD_NCCL_LINK=SHARED -export HOROVOD_GPU_OPERATIONS=NCCL - -# Create the conda environment: -conda env create --file environment.yml --force - -# Activate the conda environment: -eval "$(conda shell.bash hook)" -conda activate spark_dl - -# Install tensorflow: -pip install tensorflow-gpu - -# Install pytorch: -pip install torch==1.10.2+cu113 torchvision==0.11.3+cu113 torchaudio==0.10.2+cu113\ - -f https://download.pytorch.org/whl/cu113/torch_stable.html - -# Install horovod: -pip install horovod[tensorflow,keras,pytorch,spark] - -# Check the build: -horovodrun --check-build - -echo -echo "Run 'conda activate spark_dl' to activate the environment" -echo diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/criteo_keras.py b/examples/ML+DL-Examples/Spark-DL/criteo_train/criteo_keras.py deleted file mode 100644 index bb653f9e6..000000000 --- a/examples/ML+DL-Examples/Spark-DL/criteo_train/criteo_keras.py +++ /dev/null @@ -1,309 +0,0 @@ -# -# Copyright (c) 2021-2021, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import argparse -import math -import pprint -import sys - -# This needs to happen first to avoid pyarrow serialization errors. -from pyspark.sql import SparkSession - -# Make sure pyarrow is referenced before anything else to avoid segfault due to conflict -# with TensorFlow libraries. Use `pa` package reference to ensure it's loaded before -# functions like `deserialize_model` which are implemented at the top level. -# See https://jira.apache.org/jira/browse/ARROW-3346 -import pyarrow as pa - -import horovod -import horovod.tensorflow.keras as hvd -import tensorflow as tf -from horovod.spark.common.backend import SparkBackend -from tensorflow.keras.layers import BatchNormalization, Input, Embedding, Concatenate, Dense, Flatten -from tensorflow.keras.layers.experimental.preprocessing import CategoryEncoding - -PETASTORM_DATALOADER = 'petastorm' -NVTABULAR_DATALOADER = 'nvtabular' - -CONTINUOUS_COLUMNS = [f'i{i}' for i in range(13)] -CATEGORICAL_COLUMNS = [f'c{c}' for c in range(26)] -ALL_COLUMNS = CONTINUOUS_COLUMNS + CATEGORICAL_COLUMNS -LABEL_COLUMNS = ['clicked'] - - -def get_category_dimensions(spark, data_dir): - df = spark.read.csv(f'{data_dir}/dimensions/*.csv', header=True).toPandas() - dimensions = df.to_dict('records')[0] - pprint.pprint(dimensions) - return dimensions - - -def build_model(dimensions, args): - inputs = { - **{i: Input(shape=(1,), name=i, dtype=tf.float32) for i in CONTINUOUS_COLUMNS}, - **{c: Input(shape=(1,), name=c, dtype=tf.int32) for c in CATEGORICAL_COLUMNS} - } - - one_hots = [] - embeddings = [] - for c in CATEGORICAL_COLUMNS: - dimension = int(dimensions[c]) + 1 - if dimension <= 128: - one_hots.append(CategoryEncoding(num_tokens=dimension, name=f'one_hot_{c}')(inputs[c])) - else: - embedding_size = int(math.floor(0.6 * dimension ** 0.25)) - embeddings.append(Embedding(input_dim=dimension, - output_dim=embedding_size, - input_length=1, - name=f'embedding_{c}')(inputs[c])) - - x = Concatenate(name='embeddings_concat')(embeddings) - x = Flatten(name='embeddings_flatten')(x) - x = Concatenate(name='inputs_concat')([x] + one_hots + [inputs[i] for i in CONTINUOUS_COLUMNS]) - x = BatchNormalization()(x) - x = Dense(1024, activation='relu')(x) - x = BatchNormalization()(x) - x = Dense(1024, activation='relu')(x) - x = BatchNormalization()(x) - x = Dense(1024, activation='relu')(x) - x = BatchNormalization()(x) - x = Dense(512, activation='relu')(x) - output = Dense(1, activation='sigmoid', name='output')(x) - model = tf.keras.Model(inputs=[inputs[c] for c in ALL_COLUMNS], outputs=output) - if hvd.rank() == 0: - model.summary() - - opt = tf.keras.optimizers.Adam(learning_rate=args.learning_rate) - opt = hvd.DistributedOptimizer(opt) - model.compile(optimizer=opt, loss='binary_crossentropy', metrics=[tf.keras.metrics.AUC()]) - - return model - - -def train_fn(dimensions, train_rows, val_rows, args): - # Make sure pyarrow is referenced before anything else to avoid segfault due to conflict - # with TensorFlow libraries. Use `pa` package reference to ensure it's loaded before - # functions like `deserialize_model` which are implemented at the top level. - # See https://jira.apache.org/jira/browse/ARROW-3346 - pa - - import atexit - import horovod.tensorflow.keras as hvd - from horovod.spark.task import get_available_devices - import os - import tempfile - import tensorflow as tf - import tensorflow.keras.backend as K - import shutil - - gpus = get_available_devices() - if gpus: - os.environ['CUDA_VISIBLE_DEVICES'] = gpus[0] - if args.dataloader == NVTABULAR_DATALOADER: - os.environ['TF_MEMORY_ALLOCATION'] = '0.85' - from nvtabular.loader.tensorflow import KerasSequenceLoader - - # Horovod: initialize Horovod inside the trainer. - hvd.init() - - # Horovod: restore from checkpoint, use hvd.load_model under the hood. - model = build_model(dimensions, args) - - # Horovod: adjust learning rate based on number of processes. - scaled_lr = K.get_value(model.optimizer.lr) * hvd.size() - K.set_value(model.optimizer.lr, scaled_lr) - - # Horovod: print summary logs on the first worker. - verbose = 1 if hvd.rank() == 0 else 0 - - callbacks = [ - # Horovod: broadcast initial variable states from rank 0 to all other processes. - # This is necessary to ensure consistent initialization of all workers when - # training is started with random weights or restored from a checkpoint. - hvd.callbacks.BroadcastGlobalVariablesCallback(root_rank=0), - - # Horovod: average metrics among workers at the end of every epoch. - # - # Note: This callback must be in the list before the ReduceLROnPlateau, - # TensorBoard, or other metrics-based callbacks. - hvd.callbacks.MetricAverageCallback(), - - # Horovod: using `lr = 1.0 * hvd.size()` from the very beginning leads to worse final - # accuracy. Scale the learning rate `lr = 1.0` ---> `lr = 1.0 * hvd.size()` during - # the first five epochs. See https://arxiv.org/abs/1706.02677 for details. - hvd.callbacks.LearningRateWarmupCallback(initial_lr=scaled_lr, warmup_epochs=5, verbose=verbose), - - # Reduce LR if the metric is not improved for 10 epochs, and stop training - # if it has not improved for 20 epochs. - tf.keras.callbacks.ReduceLROnPlateau(monitor='val_auc', patience=10, verbose=verbose), - tf.keras.callbacks.EarlyStopping(monitor='val_auc', mode='min', patience=20, verbose=verbose), - tf.keras.callbacks.TerminateOnNaN(), - - # Log Tensorboard events. - tf.keras.callbacks.TensorBoard(log_dir=args.logs_dir, write_steps_per_second=True, update_freq=10) - ] - - # Horovod: save checkpoints only on the first worker to prevent other workers from corrupting them. - if hvd.rank() == 0: - ckpt_dir = tempfile.mkdtemp() - ckpt_file = os.path.join(ckpt_dir, 'checkpoint.h5') - atexit.register(lambda: shutil.rmtree(ckpt_dir)) - callbacks.append(tf.keras.callbacks.ModelCheckpoint( - ckpt_file, monitor='val_auc', mode='min', save_best_only=True)) - - if args.dataloader == PETASTORM_DATALOADER: - from petastorm import make_batch_reader - from petastorm.tf_utils import make_petastorm_dataset - - # Make Petastorm readers. - with make_batch_reader(f'{args.data_dir}/train', - num_epochs=None, - cur_shard=hvd.rank(), - shard_count=hvd.size(), - hdfs_driver='libhdfs') as train_reader: - with make_batch_reader(f'{args.data_dir}/val', - num_epochs=None, - cur_shard=hvd.rank(), - shard_count=hvd.size(), - hdfs_driver='libhdfs') as val_reader: - # Convert readers to tf.data.Dataset. - train_ds = make_petastorm_dataset(train_reader) \ - .unbatch() \ - .shuffle(10 * args.batch_size) \ - .batch(args.batch_size) \ - .map(lambda x: (tuple(getattr(x, c) for c in ALL_COLUMNS), x.clicked)) - - val_ds = make_petastorm_dataset(val_reader) \ - .unbatch() \ - .batch(args.batch_size) \ - .map(lambda x: (tuple(getattr(x, c) for c in ALL_COLUMNS), x.clicked)) - - history = model.fit(train_ds, - validation_data=val_ds, - steps_per_epoch=int(train_rows / args.batch_size / hvd.size()), - validation_steps=int(val_rows / args.batch_size / hvd.size()), - callbacks=callbacks, - verbose=verbose, - epochs=args.epochs) - - else: - import cupy - - def seed_fn(): - """ - Generate consistent dataloader shuffle seeds across workers - Reseeds each worker's dataloader each epoch to get fresh a shuffle - that's consistent across workers. - """ - min_int, max_int = tf.int32.limits - max_rand = max_int // hvd.size() - # Generate a seed fragment on each worker - seed_fragment = cupy.random.randint(0, max_rand).get() - # Aggregate seed fragments from all Horovod workers - seed_tensor = tf.constant(seed_fragment) - reduced_seed = hvd.allreduce(seed_tensor, name="shuffle_seed", op=hvd.Sum) - return reduced_seed % max_rand - - train_ds = KerasSequenceLoader( - f'{args.data_dir}/train', - batch_size=args.batch_size, - label_names=LABEL_COLUMNS, - cat_names=CATEGORICAL_COLUMNS, - cont_names=CONTINUOUS_COLUMNS, - engine="parquet", - shuffle=True, - buffer_size=0.06, # how many batches to load at once - parts_per_chunk=1, - global_size=hvd.size(), - global_rank=hvd.rank(), - seed_fn=seed_fn) - - val_ds = KerasSequenceLoader( - f'{args.data_dir}/val', - batch_size=args.batch_size, - label_names=LABEL_COLUMNS, - cat_names=CATEGORICAL_COLUMNS, - cont_names=CONTINUOUS_COLUMNS, - engine="parquet", - shuffle=False, - buffer_size=0.06, # how many batches to load at once - parts_per_chunk=1, - global_size=hvd.size(), - global_rank=hvd.rank()) - - history = model.fit(train_ds, - validation_data=val_ds, - steps_per_epoch=int(train_rows / args.batch_size / hvd.size()), - validation_steps=int(val_rows / args.batch_size / hvd.size()), - callbacks=callbacks, - verbose=verbose, - epochs=args.epochs) - - if hvd.rank() == 0: - return history.history - - -def train(dimensions, train_rows, val_rows, args): - # Horovod: run training. - history = horovod.spark.run(train_fn, - args=(dimensions, train_rows, val_rows, args), - num_proc=args.num_proc, - extra_mpi_args='-mca btl_tcp_if_include enp134s0f0 -x NCCL_IB_GID_INDEX=3', - stdout=sys.stdout, - stderr=sys.stderr, - verbose=2, - nics={}, - prefix_output_with_timestamp=True)[0] - - best_val_loss = min(history['val_loss']) - print('Best Loss: %f' % best_val_loss) - - -def main(): - parser = argparse.ArgumentParser(description='Criteo Spark Keras Training Example', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('--data-dir', default='file:///opt/data/criteo/parquet', - help='location of the transformed Criteo dataset in Parquet format') - parser.add_argument('--logs-dir', default='/opt/experiments/criteo', help='location of TensorFlow logs') - parser.add_argument('--dataloader', default=PETASTORM_DATALOADER, - choices=[PETASTORM_DATALOADER, NVTABULAR_DATALOADER], - help='dataloader to use') - parser.add_argument('--num-proc', type=int, default=1, help='number of worker processes for training') - parser.add_argument('--learning-rate', type=float, default=0.0001, help='initial learning rate') - parser.add_argument('--batch-size', type=int, default=64 * 1024, help='batch size') - parser.add_argument('--epochs', type=int, default=3, help='number of epochs to train') - parser.add_argument('--local-checkpoint-file', default='checkpoint', help='model checkpoint') - args = parser.parse_args() - - spark = SparkSession.builder.appName('Criteo Keras Training').getOrCreate() - - dimensions = get_category_dimensions(spark, args.data_dir) - - train_df = spark.read.parquet(f'{args.data_dir}/train') - val_df = spark.read.parquet(f'{args.data_dir}/val') - test_df = spark.read.parquet(f'{args.data_dir}/test') - train_rows, val_rows, test_rows = train_df.count(), val_df.count(), test_df.count() - print('Training: %d' % train_rows) - print('Validation: %d' % val_rows) - print('Test: %d' % test_rows) - - train(dimensions, train_rows, val_rows, args) - - spark.stop() - - -if __name__ == '__main__': - main() diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/environment.yml b/examples/ML+DL-Examples/Spark-DL/criteo_train/environment.yml deleted file mode 100644 index 9bb5e3a52..000000000 --- a/examples/ML+DL-Examples/Spark-DL/criteo_train/environment.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: spark_dl -channels: - - rapidsai - - nvidia - - pytorch - - conda-forge -dependencies: - - nvtabular - - python>=3.7 - - cudatoolkit=11.2 - - cudf>=21.10.* - - dask-cuda>=21.10.* - - dask-cudf>=21.10.* - - rmm>=21.10.* - - dask==2021.9.1 - - distributed>=2021.9.1 - - fsspec>=0.6.0 - - nvtx>=0.2.1 - - numba>=0.53.0 - - dlpack - - scikit-learn - - asvdb - - pynvml - - cmake - - cudnn - - cxx-compiler - - nccl - - nvcc_linux-64 - - openmpi - - pip diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/notebooks/Criteo-Training.ipynb b/examples/ML+DL-Examples/Spark-DL/criteo_train/notebooks/Criteo-Training.ipynb deleted file mode 100644 index d760106b4..000000000 --- a/examples/ML+DL-Examples/Spark-DL/criteo_train/notebooks/Criteo-Training.ipynb +++ /dev/null @@ -1,12675 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "dd4592b6", - "metadata": {}, - "source": [ - "# Introduction\n", - "\n", - "This notebook contains the same content as \"criteo_keras.py\" but in a notebook(interactive) form.\n", - "\n", - "The dataset used here is from Criteo clicklog dataset. It's preprocessed by DLRM(https://github.com/NVIDIA/DeepLearningExamples/tree/master/TensorFlow2/Recommendation/DLRM_and_DCNv2/preproc) ETL job on Spark.\n", - "\n", - "We provide a small size sample data in `sample_data` folder.\n", - "\n", - "The data schema after the DLRM ETL: also 40 columns -- 1 label column, 39 numerical feature columns. (Categorical columns are processed)\n", - "\n", - "User can either take the same routine to get the data or use your own preprocessed data and change the DL model accordingly.\n", - "\n", - "`Please note: The following demo is dedicated for DGX-2 machine(with V100 GPUs).` We optimized the whole workflow on DGX-2 and it's not guaranteed that it can run successfully on other type of machines." - ] - }, - { - "cell_type": "markdown", - "id": "e8bdbe10", - "metadata": {}, - "source": [ - "### import necessary libraries" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "cd1fb946", - "metadata": {}, - "outputs": [], - "source": [ - "import argparse\n", - "import math\n", - "import pprint\n", - "import sys\n", - "# This needs to happen first to avoid pyarrow serialization errors.\n", - "from pyspark.sql import SparkSession\n", - "\n", - "# Make sure pyarrow is referenced before anything else to avoid segfault due to conflict\n", - "# with TensorFlow libraries. Use `pa` package reference to ensure it's loaded before\n", - "# functions like `deserialize_model` which are implemented at the top level.\n", - "# See https://jira.apache.org/jira/browse/ARROW-3346\n", - "import pyarrow as pa\n", - "\n", - "import horovod\n", - "import horovod.tensorflow.keras as hvd\n", - "import tensorflow as tf\n", - "from horovod.spark.common.backend import SparkBackend\n", - "from tensorflow.keras.layers import BatchNormalization, Input, Embedding, Concatenate, Dense, Flatten\n", - "from tensorflow.keras.layers.experimental.preprocessing import CategoryEncoding" - ] - }, - { - "cell_type": "markdown", - "id": "ffda2b11", - "metadata": {}, - "source": [ - "### set some macros\n", - "\n", - "xxx_DATALOADER is the switch to control which dataloader we will use,\n", - "xxx_COLUMNS are only used in NVTabular dataloader." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "d690dd8b", - "metadata": {}, - "outputs": [], - "source": [ - "PETASTORM_DATALOADER = 'petastorm'\n", - "NVTABULAR_DATALOADER = 'nvtabular'\n", - "\n", - "CONTINUOUS_COLUMNS = [f'i{i}' for i in range(13)]\n", - "CATEGORICAL_COLUMNS = [f'c{c}' for c in range(26)]\n", - "ALL_COLUMNS = CONTINUOUS_COLUMNS + CATEGORICAL_COLUMNS\n", - "LABEL_COLUMNS = ['clicked']" - ] - }, - { - "cell_type": "markdown", - "id": "cdd51a2a", - "metadata": {}, - "source": [ - "The \"dimensions\" contains the count of distinct value for those categorical column 'after' DLRM ETL.\n", - "\n", - "This will be used to build embedding layers for our model." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "92e6d359", - "metadata": {}, - "outputs": [], - "source": [ - "def get_category_dimensions(spark, data_dir):\n", - " df = spark.read.csv(f'{data_dir}/dimensions/*.csv', header=True).toPandas()\n", - " dimensions = df.to_dict('records')[0]\n", - " pprint.pprint(dimensions)\n", - " return dimensions" - ] - }, - { - "cell_type": "markdown", - "id": "66e38f1a", - "metadata": {}, - "source": [ - "### Build the model\n", - "\n", - "The model is composed by mainly 2 parts: embedding layers and fully-connected layers." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "0c29876c", - "metadata": {}, - "outputs": [], - "source": [ - "def build_model(dimensions, args):\n", - " \n", - " inputs = {\n", - " **{i: Input(shape=(1,), name=i, dtype=tf.float32) for i in CONTINUOUS_COLUMNS},\n", - " **{c: Input(shape=(1,), name=c, dtype=tf.int32) for c in CATEGORICAL_COLUMNS}\n", - " }\n", - "\n", - " one_hots = []\n", - " embeddings = []\n", - " for c in CATEGORICAL_COLUMNS:\n", - " dimension = int(dimensions[c]) + 1\n", - " # dimension <= 128, smaller size for demo\n", - " if dimension <= 8:\n", - " one_hots.append(CategoryEncoding(num_tokens=dimension, name=f'one_hot_{c}')(inputs[c]))\n", - " else:\n", - " # embedding_size = int(math.floor(0.6 * dimension ** 0.25)), smaller model size for demo\n", - " embedding_size = 8\n", - " embeddings.append(Embedding(input_dim=dimension,\n", - " output_dim=embedding_size,\n", - " input_length=1,\n", - " name=f'embedding_{c}')(inputs[c]))\n", - "\n", - " x = Concatenate(name='embeddings_concat')(embeddings)\n", - " x = Flatten(name='embeddings_flatten')(x)\n", - " x = Concatenate(name='inputs_concat')([x] + one_hots + [inputs[i] for i in CONTINUOUS_COLUMNS])\n", - " x = BatchNormalization()(x)\n", - " x = Dense(64, activation='relu')(x)\n", - " x = BatchNormalization()(x)\n", - " x = Dense(64, activation='relu')(x)\n", - " x = BatchNormalization()(x)\n", - " x = Dense(64, activation='relu')(x)\n", - " x = BatchNormalization()(x)\n", - " x = Dense(32, activation='relu')(x)\n", - " output = Dense(1, activation='sigmoid', name='output')(x)\n", - " model = tf.keras.Model(inputs=[inputs[c] for c in ALL_COLUMNS], outputs=output)\n", - " if hvd.rank() == 0:\n", - " model.summary()\n", - "\n", - " opt = tf.keras.optimizers.Adam(learning_rate=args.learning_rate)\n", - " opt = hvd.DistributedOptimizer(opt)\n", - " model.compile(optimizer=opt, loss='binary_crossentropy', metrics=[tf.keras.metrics.AUC()])\n", - "\n", - " return model" - ] - }, - { - "cell_type": "markdown", - "id": "7dcc291d", - "metadata": {}, - "source": [ - "### Set train function\n", - "\n", - "The train_fn is the function that will execute across all Horovod workers(also say Spark executors in our case).\n", - "\n", - "We use `CUDA_VISIBLE_DEVICES` to control the GPU resources to avoid overlapping." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "3bc96eb8", - "metadata": {}, - "outputs": [], - "source": [ - "def train_fn(dimensions, train_rows, val_rows, args):\n", - " # Make sure pyarrow is referenced before anything else to avoid segfault due to conflict\n", - " # with TensorFlow libraries. Use `pa` package reference to ensure it's loaded before\n", - " # functions like `deserialize_model` which are implemented at the top level.\n", - " # See https://jira.apache.org/jira/browse/ARROW-3346\n", - " pa\n", - "\n", - " import atexit\n", - " import horovod.tensorflow.keras as hvd\n", - " from horovod.spark.task import get_available_devices\n", - " import os\n", - " import tempfile\n", - " import tensorflow as tf\n", - " import tensorflow.keras.backend as K\n", - " import shutil\n", - "\n", - " gpus = get_available_devices()\n", - " if gpus:\n", - " os.environ['CUDA_VISIBLE_DEVICES'] = gpus[0]\n", - " if args.dataloader == NVTABULAR_DATALOADER:\n", - " os.environ['TF_MEMORY_ALLOCATION'] = '0.85'\n", - " from nvtabular.loader.tensorflow import KerasSequenceLoader\n", - "\n", - " # Horovod: initialize Horovod inside the trainer.\n", - " hvd.init()\n", - "\n", - " # Horovod: restore from checkpoint, use hvd.load_model under the hood.\n", - " model = build_model(dimensions, args)\n", - "\n", - " # Horovod: adjust learning rate based on number of processes.\n", - " scaled_lr = K.get_value(model.optimizer.lr) * hvd.size()\n", - " K.set_value(model.optimizer.lr, scaled_lr)\n", - "\n", - " # Horovod: print summary logs on the first worker.\n", - " verbose = 1 if hvd.rank() == 0 else 0\n", - "\n", - " callbacks = [\n", - " # Horovod: broadcast initial variable states from rank 0 to all other processes.\n", - " # This is necessary to ensure consistent initialization of all workers when\n", - " # training is started with random weights or restored from a checkpoint.\n", - " hvd.callbacks.BroadcastGlobalVariablesCallback(root_rank=0),\n", - "\n", - " # Horovod: average metrics among workers at the end of every epoch.\n", - " #\n", - " # Note: This callback must be in the list before the ReduceLROnPlateau,\n", - " # TensorBoard, or other metrics-based callbacks.\n", - " hvd.callbacks.MetricAverageCallback(),\n", - "\n", - " # Horovod: using `lr = 1.0 * hvd.size()` from the very beginning leads to worse final\n", - " # accuracy. Scale the learning rate `lr = 1.0` ---> `lr = 1.0 * hvd.size()` during\n", - " # the first five epochs. See https://arxiv.org/abs/1706.02677 for details.\n", - " hvd.callbacks.LearningRateWarmupCallback(initial_lr=scaled_lr, warmup_epochs=5, verbose=verbose),\n", - "\n", - " # Reduce LR if the metric is not improved for 10 epochs, and stop training\n", - " # if it has not improved for 20 epochs.\n", - " tf.keras.callbacks.ReduceLROnPlateau(monitor='val_auc', patience=10, verbose=verbose),\n", - " tf.keras.callbacks.EarlyStopping(monitor='val_auc', mode='min', patience=20, verbose=verbose),\n", - " tf.keras.callbacks.TerminateOnNaN(),\n", - "\n", - " # Log Tensorboard events.\n", - " tf.keras.callbacks.TensorBoard(log_dir=args.logs_dir, write_steps_per_second=True, update_freq=10)\n", - " ]\n", - "\n", - " # Horovod: save checkpoints only on the first worker to prevent other workers from corrupting them.\n", - " if hvd.rank() == 0:\n", - " ckpt_dir = tempfile.mkdtemp()\n", - " ckpt_file = os.path.join(ckpt_dir, 'checkpoint.h5')\n", - " atexit.register(lambda: shutil.rmtree(ckpt_dir))\n", - " callbacks.append(tf.keras.callbacks.ModelCheckpoint(\n", - " ckpt_file, monitor='val_auc', mode='min', save_best_only=True))\n", - "\n", - " if args.dataloader == PETASTORM_DATALOADER:\n", - " from petastorm import make_batch_reader\n", - " from petastorm.tf_utils import make_petastorm_dataset\n", - "\n", - " # Make Petastorm readers.\n", - " with make_batch_reader(f'{args.data_dir}/train',\n", - " num_epochs=None,\n", - " cur_shard=hvd.rank(),\n", - " shard_count=hvd.size(),\n", - " hdfs_driver='libhdfs') as train_reader:\n", - " with make_batch_reader(f'{args.data_dir}/val',\n", - " num_epochs=None,\n", - " cur_shard=hvd.rank(),\n", - " shard_count=hvd.size(),\n", - " hdfs_driver='libhdfs') as val_reader:\n", - " # Convert readers to tf.data.Dataset.\n", - " train_ds = make_petastorm_dataset(train_reader) \\\n", - " .unbatch() \\\n", - " .shuffle(10 * args.batch_size) \\\n", - " .batch(args.batch_size) \\\n", - " .map(lambda x: (tuple(getattr(x, c) for c in ALL_COLUMNS), x.clicked))\n", - "\n", - " val_ds = make_petastorm_dataset(val_reader) \\\n", - " .unbatch() \\\n", - " .batch(args.batch_size) \\\n", - " .map(lambda x: (tuple(getattr(x, c) for c in ALL_COLUMNS), x.clicked))\n", - "\n", - " history = model.fit(train_ds,\n", - " validation_data=val_ds,\n", - " steps_per_epoch=int(train_rows / args.batch_size / hvd.size()),\n", - " validation_steps=int(val_rows / args.batch_size / hvd.size()),\n", - " callbacks=callbacks,\n", - " verbose=verbose,\n", - " epochs=args.epochs)\n", - "\n", - " else:\n", - " import cupy\n", - "\n", - " def seed_fn():\n", - " \"\"\"\n", - " Generate consistent dataloader shuffle seeds across workers\n", - " Reseeds each worker's dataloader each epoch to get fresh a shuffle\n", - " that's consistent across workers.\n", - " \"\"\"\n", - " min_int, max_int = tf.int32.limits\n", - " max_rand = max_int // hvd.size()\n", - " # Generate a seed fragment on each worker\n", - " seed_fragment = cupy.random.randint(0, max_rand).get()\n", - " # Aggregate seed fragments from all Horovod workers\n", - " seed_tensor = tf.constant(seed_fragment)\n", - " reduced_seed = hvd.allreduce(seed_tensor, name=\"shuffle_seed\", op=hvd.Sum)\n", - " return reduced_seed % max_rand\n", - "\n", - " train_ds = KerasSequenceLoader(\n", - " f'{args.data_dir}/train',\n", - " batch_size=args.batch_size,\n", - " label_names=LABEL_COLUMNS,\n", - " cat_names=CATEGORICAL_COLUMNS,\n", - " cont_names=CONTINUOUS_COLUMNS,\n", - " engine=\"parquet\",\n", - " shuffle=True,\n", - " buffer_size=0.06, # how many batches to load at once\n", - " parts_per_chunk=1,\n", - " global_size=hvd.size(),\n", - " global_rank=hvd.rank(),\n", - " seed_fn=seed_fn)\n", - "\n", - " val_ds = KerasSequenceLoader(\n", - " f'{args.data_dir}/val',\n", - " batch_size=args.batch_size,\n", - " label_names=LABEL_COLUMNS,\n", - " cat_names=CATEGORICAL_COLUMNS,\n", - " cont_names=CONTINUOUS_COLUMNS,\n", - " engine=\"parquet\",\n", - " shuffle=False,\n", - " buffer_size=0.06, # how many batches to load at once\n", - " parts_per_chunk=1,\n", - " global_size=hvd.size(),\n", - " global_rank=hvd.rank())\n", - "\n", - " history = model.fit(train_ds,\n", - " validation_data=val_ds,\n", - " steps_per_epoch=int(train_rows / args.batch_size / hvd.size()),\n", - " validation_steps=int(val_rows / args.batch_size / hvd.size()),\n", - " callbacks=callbacks,\n", - " verbose=verbose,\n", - " epochs=args.epochs)\n", - "\n", - " if hvd.rank() == 0:\n", - " return history.history" - ] - }, - { - "cell_type": "markdown", - "id": "392ac264", - "metadata": {}, - "source": [ - "### Wrapper function to train\n", - "\n", - "Here we call the `horovod.spark.run` to start the training process in Horovod on Spark." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "f9b8459d", - "metadata": {}, - "outputs": [], - "source": [ - "def train(dimensions, train_rows, val_rows, args):\n", - " # Horovod: run training.\n", - " history = horovod.spark.run(train_fn,\n", - " args=(dimensions, train_rows, val_rows, args),\n", - " num_proc=args.num_proc,\n", - " extra_mpi_args='-mca btl_tcp_if_include enp134s0f0 -x NCCL_IB_GID_INDEX=3',\n", - " stdout=sys.stdout,\n", - " stderr=sys.stderr,\n", - " verbose=2,\n", - " nics={},\n", - " prefix_output_with_timestamp=True)[0]\n", - "\n", - " best_val_loss = min(history['val_loss'])\n", - " print('Best Loss: %f' % best_val_loss)" - ] - }, - { - "cell_type": "markdown", - "id": "3e4ac6b8", - "metadata": {}, - "source": [ - "## Use NVTabular\n", - "\n", - "Here we set `--dataloader` to `nvtabular` to force NVTabular run." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "c2afc3a4", - "metadata": {}, - "outputs": [], - "source": [ - "def main():\n", - " parser = argparse.ArgumentParser(description='Criteo Spark Keras Training Example',\n", - " formatter_class=argparse.ArgumentDefaultsHelpFormatter)\n", - " parser.add_argument('--data-dir', default='file:///opt/data/criteo/parquet',\n", - " help='location of the transformed Criteo dataset in Parquet format')\n", - " parser.add_argument('--logs-dir', default='/opt/experiments/criteo', help='location of TensorFlow logs')\n", - " parser.add_argument('--dataloader', default=PETASTORM_DATALOADER,\n", - " choices=[PETASTORM_DATALOADER, NVTABULAR_DATALOADER],\n", - " help='dataloader to use')\n", - " parser.add_argument('--num-proc', type=int, default=1, help='number of worker processes for training')\n", - " parser.add_argument('--learning-rate', type=float, default=0.0001, help='initial learning rate')\n", - " parser.add_argument('--batch-size', type=int, default=64 * 1024, help='batch size')\n", - " parser.add_argument('--epochs', type=int, default=3, help='number of epochs to train')\n", - " parser.add_argument('--local-checkpoint-file', default='checkpoint', help='model checkpoint')\n", - " args = parser.parse_args(args=['--num-proc', '16', '--data-dir', 'file:///raid/spark-team/criteo/parquet', \n", - " '--dataloader', 'nvtabular', '--learning-rate', '0.001',\n", - " '--batch-size', '65535','--epochs', '1', '--logs-dir', 'tf_logs',\n", - " '--local-checkpoint-file', 'ckpt_file'])\n", - " \n", - "\n", - " dimensions = get_category_dimensions(spark, args.data_dir)\n", - "\n", - " train_df = spark.read.parquet(f'{args.data_dir}/train')\n", - " val_df = spark.read.parquet(f'{args.data_dir}/val')\n", - " test_df = spark.read.parquet(f'{args.data_dir}/test')\n", - " train_rows, val_rows, test_rows = train_df.count(), val_df.count(), test_df.count()\n", - " print('Training: %d' % train_rows)\n", - " print('Validation: %d' % val_rows)\n", - " print('Test: %d' % test_rows)\n", - "\n", - " train(dimensions, train_rows, val_rows, args)\n", - "\n", - " spark.stop()" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "a15573e3", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "21/09/06 09:03:44 WARN package: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n", - " \r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'c0': '7912888',\n", - " 'c1': '33822',\n", - " 'c10': '582468',\n", - " 'c11': '245827',\n", - " 'c12': '10',\n", - " 'c13': '2208',\n", - " 'c14': '10666',\n", - " 'c15': '103',\n", - " 'c16': '3',\n", - " 'c17': '967',\n", - " 'c18': '14',\n", - " 'c19': '8165895',\n", - " 'c2': '17138',\n", - " 'c20': '2675939',\n", - " 'c21': '7156452',\n", - " 'c22': '302515',\n", - " 'c23': '12021',\n", - " 'c24': '96',\n", - " 'c25': '34',\n", - " 'c3': '7338',\n", - " 'c4': '20045',\n", - " 'c5': '3',\n", - " 'c6': '7104',\n", - " 'c7': '1381',\n", - " 'c8': '62',\n", - " 'c9': '5554113'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Training: 4195197692\n", - "Validation: 89137318\n", - "Test: 89137319\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Checking whether extension tensorflow was built with MPI.\n", - "Extension tensorflow was built with MPI.\n", - "mpirun --allow-run-as-root --tag-output -np 16 -H dgx2h0194-a1adff968d508e8d1142986f3e2c42dc:16 -bind-to none -map-by slot -mca pml ob1 -mca btl ^openib --timestamp-output -mca btl_tcp_if_include enp134s0f0 -x NCCL_IB_GID_INDEX=3 -x NCCL_DEBUG=INFO -mca plm_rsh_agent \"/home/ngc-auth-ldap-allxu/miniconda3/bin/python -m horovod.spark.driver.mpirun_rsh gAWVcAEAAAAAAAB9lCiMAmxvlF2UjAkxMjcuMC4wLjGUTWIShpRhjAdlbnA1M3MwlF2UjAwxMC4xNDguMzAuNTmUTWIShpRhjAdlbnA1OHMwlF2UjAwxMC4xNDguOTQuNTmUTWIShpRhjAdlbnA4OHMwlF2UjAwxMC4xNDkuMzAuMzSUTWIShpRhjAdlbnA5M3MwlF2UjAwxMC4xNDkuOTQuNTeUTWIShpRhjAplbnAxMzRzMGYwlF2UjAsxMC4xNTAuMzAuMpRNYhKGlGGMCGVucDE4NHMwlF2UjA0xMC4xNDguMTU4LjU5lE1iEoaUYYwIZW5wMTg5czCUXZSMDTEwLjE0OC4yMjIuNTmUTWIShpRhjAhlbnAyMjVzMJRdlIwNMTAuMTQ5LjE1OC41N5RNYhKGlGGMCGVucDIzMHMwlF2UjA0xMC4xNDkuMjIyLjU3lE1iEoaUYYwHZG9ja2VyMJRdlIwKMTcyLjE3LjAuMZRNYhKGlGF1Lg== gAWVAwMAAAAAAACMI2hvcm92b2QucnVubmVyLmNvbW1vbi51dGlsLnNldHRpbmdzlIwIU2V0dGluZ3OUk5QpgZR9lCiMCG51bV9wcm9jlEsQjAd2ZXJib3NllEsCjAhzc2hfcG9ydJROjBFzc2hfaWRlbnRpdHlfZmlsZZROjA5leHRyYV9tcGlfYXJnc5SMOS1tY2EgYnRsX3RjcF9pZl9pbmNsdWRlIGVucDEzNHMwZjAgLXggTkNDTF9JQl9HSURfSU5ERVg9M5SMCHRjcF9mbGFnlE6MDGJpbmRpbmdfYXJnc5ROjANrZXmUTowNc3RhcnRfdGltZW91dJSMImhvcm92b2QucnVubmVyLmNvbW1vbi51dGlsLnRpbWVvdXSUjAdUaW1lb3V0lJOUKYGUfZQojAhfdGltZW91dJRNWAKMC190aW1lb3V0X2F0lEdB2E128/YMbIwIX21lc3NhZ2WUWA4BAABUaW1lZCBvdXQgd2FpdGluZyBmb3Ige2FjdGl2aXR5fS4gUGxlYXNlIGNoZWNrIHRoYXQgeW91IGhhdmUgZW5vdWdoIHJlc291cmNlcyB0byBydW4gYWxsIEhvcm92b2QgcHJvY2Vzc2VzLiBFYWNoIEhvcm92b2QgcHJvY2VzcyBydW5zIGluIGEgU3BhcmsgdGFzay4gWW91IG1heSBuZWVkIHRvIGluY3JlYXNlIHRoZSBzdGFydF90aW1lb3V0IHBhcmFtZXRlciB0byBhIGxhcmdlciB2YWx1ZSBpZiB5b3VyIFNwYXJrIHJlc291cmNlcyBhcmUgYWxsb2NhdGVkIG9uLWRlbWFuZC6UdWKMD291dHB1dF9maWxlbmFtZZROjA1ydW5fZnVuY19tb2RllIiMBG5pY3OUfZSMB2VsYXN0aWOUiYwccHJlZml4X291dHB1dF93aXRoX3RpbWVzdGFtcJSIjAVob3N0c5SMLWRneDJoMDE5NC1hMWFkZmY5NjhkNTA4ZThkMTE0Mjk4NmYzZTJjNDJkYzoxNpR1Yi4=\" /home/ngc-auth-ldap-allxu/miniconda3/bin/python -m horovod.spark.task.mpirun_exec_fn gAWVcAEAAAAAAAB9lCiMAmxvlF2UjAkxMjcuMC4wLjGUTWIShpRhjAdlbnA1M3MwlF2UjAwxMC4xNDguMzAuNTmUTWIShpRhjAdlbnA1OHMwlF2UjAwxMC4xNDguOTQuNTmUTWIShpRhjAdlbnA4OHMwlF2UjAwxMC4xNDkuMzAuMzSUTWIShpRhjAdlbnA5M3MwlF2UjAwxMC4xNDkuOTQuNTeUTWIShpRhjAplbnAxMzRzMGYwlF2UjAsxMC4xNTAuMzAuMpRNYhKGlGGMCGVucDE4NHMwlF2UjA0xMC4xNDguMTU4LjU5lE1iEoaUYYwIZW5wMTg5czCUXZSMDTEwLjE0OC4yMjIuNTmUTWIShpRhjAhlbnAyMjVzMJRdlIwNMTAuMTQ5LjE1OC41N5RNYhKGlGGMCGVucDIzMHMwlF2UjA0xMC4xNDkuMjIyLjU3lE1iEoaUYYwHZG9ja2VyMJRdlIwKMTcyLjE3LjAuMZRNYhKGlGF1Lg== gAWVkwkAAAAAAACMI2hvcm92b2QucnVubmVyLmNvbW1vbi51dGlsLnNldHRpbmdzlIwIU2V0dGluZ3OUk5QpgZR9lCiMCG51bV9wcm9jlEsQjAd2ZXJib3NllEsCjAhzc2hfcG9ydJROjBFzc2hfaWRlbnRpdHlfZmlsZZROjA5leHRyYV9tcGlfYXJnc5RYxgYAAC1tY2EgYnRsX3RjcF9pZl9pbmNsdWRlIGVucDEzNHMwZjAgLXggTkNDTF9JQl9HSURfSU5ERVg9MyAteCBOQ0NMX0RFQlVHPUlORk8gLW1jYSBwbG1fcnNoX2FnZW50ICIvaG9tZS9uZ2MtYXV0aC1sZGFwLWFsbHh1L21pbmljb25kYTMvYmluL3B5dGhvbiAtbSBob3Jvdm9kLnNwYXJrLmRyaXZlci5tcGlydW5fcnNoIGdBV1ZjQUVBQUFBQUFBQjlsQ2lNQW14dmxGMlVqQWt4TWpjdU1DNHdMakdVVFdJU2hwUmhqQWRsYm5BMU0zTXdsRjJVakF3eE1DNHhORGd1TXpBdU5UbVVUV0lTaHBSaGpBZGxibkExT0hNd2xGMlVqQXd4TUM0eE5EZ3VPVFF1TlRtVVRXSVNocFJoakFkbGJuQTRPSE13bEYyVWpBd3hNQzR4TkRrdU16QXVNelNVVFdJU2hwUmhqQWRsYm5BNU0zTXdsRjJVakF3eE1DNHhORGt1T1RRdU5UZVVUV0lTaHBSaGpBcGxibkF4TXpSek1HWXdsRjJVakFzeE1DNHhOVEF1TXpBdU1wUk5ZaEtHbEdHTUNHVnVjREU0TkhNd2xGMlVqQTB4TUM0eE5EZ3VNVFU0TGpVNWxFMWlFb2FVWVl3SVpXNXdNVGc1Y3pDVVhaU01EVEV3TGpFME9DNHlNakl1TlRtVVRXSVNocFJoakFobGJuQXlNalZ6TUpSZGxJd05NVEF1TVRRNUxqRTFPQzQxTjVSTlloS0dsR0dNQ0dWdWNESXpNSE13bEYyVWpBMHhNQzR4TkRrdU1qSXlMalUzbEUxaUVvYVVZWXdIWkc5amEyVnlNSlJkbEl3S01UY3lMakUzTGpBdU1aUk5ZaEtHbEdGMUxnPT0gZ0FXVkF3TUFBQUFBQUFDTUkyaHZjbTkyYjJRdWNuVnVibVZ5TG1OdmJXMXZiaTUxZEdsc0xuTmxkSFJwYm1kemxJd0lVMlYwZEdsdVozT1VrNVFwZ1pSOWxDaU1DRzUxYlY5d2NtOWpsRXNRakFkMlpYSmliM05sbEVzQ2pBaHpjMmhmY0c5eWRKUk9qQkZ6YzJoZmFXUmxiblJwZEhsZlptbHNaWlJPakE1bGVIUnlZVjl0Y0dsZllYSm5jNVNNT1MxdFkyRWdZblJzWDNSamNGOXBabDlwYm1Oc2RXUmxJR1Z1Y0RFek5ITXdaakFnTFhnZ1RrTkRURjlKUWw5SFNVUmZTVTVFUlZnOU01U01DSFJqY0Y5bWJHRm5sRTZNREdKcGJtUnBibWRmWVhKbmM1Uk9qQU5yWlhtVVRvd05jM1JoY25SZmRHbHRaVzkxZEpTTUltaHZjbTkyYjJRdWNuVnVibVZ5TG1OdmJXMXZiaTUxZEdsc0xuUnBiV1Z2ZFhTVWpBZFVhVzFsYjNWMGxKT1VLWUdVZlpRb2pBaGZkR2x0Wlc5MWRKUk5XQUtNQzE5MGFXMWxiM1YwWDJGMGxFZEIyRTEyOC9ZTWJJd0lYMjFsYzNOaFoyV1VXQTRCQUFCVWFXMWxaQ0J2ZFhRZ2QyRnBkR2x1WnlCbWIzSWdlMkZqZEdsMmFYUjVmUzRnVUd4bFlYTmxJR05vWldOcklIUm9ZWFFnZVc5MUlHaGhkbVVnWlc1dmRXZG9JSEpsYzI5MWNtTmxjeUIwYnlCeWRXNGdZV3hzSUVodmNtOTJiMlFnY0hKdlkyVnpjMlZ6TGlCRllXTm9JRWh2Y205MmIyUWdjSEp2WTJWemN5QnlkVzV6SUdsdUlHRWdVM0JoY21zZ2RHRnpheTRnV1c5MUlHMWhlU0J1WldWa0lIUnZJR2x1WTNKbFlYTmxJSFJvWlNCemRHRnlkRjkwYVcxbGIzVjBJSEJoY21GdFpYUmxjaUIwYnlCaElHeGhjbWRsY2lCMllXeDFaU0JwWmlCNWIzVnlJRk53WVhKcklISmxjMjkxY21ObGN5QmhjbVVnWVd4c2IyTmhkR1ZrSUc5dUxXUmxiV0Z1WkM2VWRXS01EMjkxZEhCMWRGOW1hV3hsYm1GdFpaUk9qQTF5ZFc1ZlpuVnVZMTl0YjJSbGxJaU1CRzVwWTNPVWZaU01CMlZzWVhOMGFXT1VpWXdjY0hKbFptbDRYMjkxZEhCMWRGOTNhWFJvWDNScGJXVnpkR0Z0Y0pTSWpBVm9iM04wYzVTTUxXUm5lREpvTURFNU5DMWhNV0ZrWm1ZNU5qaGtOVEE0WlRoa01URTBNams0Tm1ZelpUSmpOREprWXpveE5wUjFZaTQ9IpSMCHRjcF9mbGFnlE6MDGJpbmRpbmdfYXJnc5ROjANrZXmUTowNc3RhcnRfdGltZW91dJSMImhvcm92b2QucnVubmVyLmNvbW1vbi51dGlsLnRpbWVvdXSUjAdUaW1lb3V0lJOUKYGUfZQojAhfdGltZW91dJRNWAKMC190aW1lb3V0X2F0lEdB2E128/YMbIwIX21lc3NhZ2WUWA4BAABUaW1lZCBvdXQgd2FpdGluZyBmb3Ige2FjdGl2aXR5fS4gUGxlYXNlIGNoZWNrIHRoYXQgeW91IGhhdmUgZW5vdWdoIHJlc291cmNlcyB0byBydW4gYWxsIEhvcm92b2QgcHJvY2Vzc2VzLiBFYWNoIEhvcm92b2QgcHJvY2VzcyBydW5zIGluIGEgU3BhcmsgdGFzay4gWW91IG1heSBuZWVkIHRvIGluY3JlYXNlIHRoZSBzdGFydF90aW1lb3V0IHBhcmFtZXRlciB0byBhIGxhcmdlciB2YWx1ZSBpZiB5b3VyIFNwYXJrIHJlc291cmNlcyBhcmUgYWxsb2NhdGVkIG9uLWRlbWFuZC6UdWKMD291dHB1dF9maWxlbmFtZZROjA1ydW5fZnVuY19tb2RllIiMBG5pY3OUfZSMB2VsYXN0aWOUiYwccHJlZml4X291dHB1dF93aXRoX3RpbWVzdGFtcJSIjAVob3N0c5SMLWRneDJoMDE5NC1hMWFkZmY5NjhkNTA4ZThkMTE0Mjk4NmYzZTJjNDJkYzoxNpR1Yi4=\n", - "Mon Sep 6 09:03:56 2021[1,2]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,4]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,0]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,6]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,1]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,5]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,3]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,10]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,7]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,15]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,11]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,13]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,14]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,8]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:03:56 2021[1,9]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,12]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:16 2021[1,4]:2021-09-06 09:04:16.101560: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,4]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,2]:2021-09-06 09:04:16.101685: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,2]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,5]:2021-09-06 09:04:16.101662: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,5]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,13]:2021-09-06 09:04:16.101697: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,13]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,3]:2021-09-06 09:04:16.101749: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,3]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,1]:2021-09-06 09:04:16.102086: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,1]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,15]:2021-09-06 09:04:16.102170: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,15]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,8]:2021-09-06 09:04:16.102756: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,8]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,14]:2021-09-06 09:04:16.102705: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,14]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,7]:2021-09-06 09:04:16.103510: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,7]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,6]:2021-09-06 09:04:16.103663: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,6]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,12]:2021-09-06 09:04:16.103909: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,12]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,0]:2021-09-06 09:04:16.120233: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,0]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,10]:2021-09-06 09:04:16.120234: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,10]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,11]:2021-09-06 09:04:16.120458: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,11]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,9]:2021-09-06 09:04:16.121686: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,9]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:18 2021[1,5]:2021-09-06 09:04:18.050711: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:bc:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,2]:2021-09-06 09:04:18.111924: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:b7:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,1]:2021-09-06 09:04:18.192508: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:be:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,15]:2021-09-06 09:04:18.258547: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:e5:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,9]:2021-09-06 09:04:18.290947: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:59:00.0, compute capability: 7.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:18 2021[1,11]:2021-09-06 09:04:18.332073: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:3b:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,4]:2021-09-06 09:04:18.333252: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:e2:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,3]:2021-09-06 09:04:18.390624: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:e7:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,7]:2021-09-06 09:04:18.391931: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:b9:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,8]:2021-09-06 09:04:18.455155: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:34:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,13]:2021-09-06 09:04:18.458402: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:57:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,12]:2021-09-06 09:04:18.460130: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:39:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,10]:2021-09-06 09:04:18.545289: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:5e:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,14]:2021-09-06 09:04:18.545716: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:e0:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,0]:2021-09-06 09:04:18.548694: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:36:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,6]:2021-09-06 09:04:18.549196: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:5c:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,5]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 09:04:18 2021[1,5]: warnings.warn(\n", - "Mon Sep 6 09:04:18 2021[1,5]:2021-09-06 09:04:18.864738: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:04:18 2021[1,5]:2021-09-06 09:04:18.864766: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:18 2021[1,5]:2021-09-06 09:04:18.864823: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 09:04:18 2021[1,2]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 09:04:18 2021[1,2]: warnings.warn(\n", - "Mon Sep 6 09:04:18 2021[1,2]:2021-09-06 09:04:18.915255: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:04:18 2021[1,2]:2021-09-06 09:04:18.915277: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:18 2021[1,2]:2021-09-06 09:04:18.915322: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 09:04:18 2021[1,1]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 09:04:18 2021[1,1]: warnings.warn(\n", - "Mon Sep 6 09:04:18 2021[1,1]:2021-09-06 09:04:18.936224: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:04:18 2021[1,1]:2021-09-06 09:04:18.936247: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:18 2021[1,1]:2021-09-06 09:04:18.936299: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 09:04:18 2021[1,15]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 09:04:18 2021[1,15]: warnings.warn(\n", - "Mon Sep 6 09:04:18 2021[1,15]:2021-09-06 09:04:18.987091: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:04:18 2021[1,15]:2021-09-06 09:04:18.987119: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:18 2021[1,15]:2021-09-06 09:04:18.987164: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 09:04:19 2021[1,9]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 09:04:19 2021[1,9]: warnings.warn(\n", - "Mon Sep 6 09:04:19 2021[1,9]:2021-09-06 09:04:19.009288: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:04:19 2021[1,9]:2021-09-06 09:04:19.009310: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:19 2021[1,9]:2021-09-06 09:04:19.009354: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 09:04:19 2021[1,4]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 09:04:19 2021[1,4]: warnings.warn(\n", - "Mon Sep 6 09:04:19 2021[1,4]:2021-09-06 09:04:19.063519: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:04:19 2021[1,4]:2021-09-06 09:04:19.063549: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:19 2021[1,4]:2021-09-06 09:04:19.063599: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 09:04:19 2021[1,11]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 09:04:19 2021[1,11]: warnings.warn(\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,11]:2021-09-06 09:04:19.064564: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:04:19 2021[1,11]:2021-09-06 09:04:19.064584: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:19 2021[1,11]:2021-09-06 09:04:19.064626: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 09:04:19 2021[1,3]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 09:04:19 2021[1,3]: warnings.warn(\n", - "Mon Sep 6 09:04:19 2021[1,3]:2021-09-06 09:04:19.114647: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:04:19 2021[1,3]:2021-09-06 09:04:19.114675: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:19 2021[1,3]:2021-09-06 09:04:19.114721: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 09:04:19 2021[1,7]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 09:04:19 2021[1,7]: warnings.warn(\n", - "Mon Sep 6 09:04:19 2021[1,7]:2021-09-06 09:04:19.130694: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:04:19 2021[1,7]:2021-09-06 09:04:19.130715: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:19 2021[1,7]:2021-09-06 09:04:19.130764: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 09:04:19 2021[1,13]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 09:04:19 2021[1,13]: warnings.warn(\n", - "Mon Sep 6 09:04:19 2021[1,13]:2021-09-06 09:04:19.209150: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:04:19 2021[1,13]:2021-09-06 09:04:19.209178: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:19 2021[1,13]:2021-09-06 09:04:19.209230: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 09:04:19 2021[1,8]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 09:04:19 2021[1,8]: warnings.warn(\n", - "Mon Sep 6 09:04:19 2021[1,8]:2021-09-06 09:04:19.220618: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:04:19 2021[1,8]:2021-09-06 09:04:19.220639: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:19 2021[1,8]:2021-09-06 09:04:19.220688: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 09:04:19 2021[1,12]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 09:04:19 2021[1,12]: warnings.warn(\n", - "Mon Sep 6 09:04:19 2021[1,12]:2021-09-06 09:04:19.259405: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:04:19 2021[1,12]:2021-09-06 09:04:19.259428: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:19 2021[1,12]:2021-09-06 09:04:19.259476: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:Model: \"model\"\n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:Layer (type) Output Shape Param # Connected to \n", - "Mon Sep 6 09:04:19 2021[1,0]:==================================================================================================\n", - "Mon Sep 6 09:04:19 2021[1,0]:c0 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c1 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c2 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c3 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c4 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c6 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c7 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c8 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c9 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c10 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c11 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c12 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c13 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c14 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c15 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c17 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c18 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c19 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c20 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c21 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c22 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c23 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c24 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c25 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c0 (Embedding) (None, 1, 8) 63303112 ['c0[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c1 (Embedding) (None, 1, 8) 270584 ['c1[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c2 (Embedding) (None, 1, 8) 137112 ['c2[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c3 (Embedding) (None, 1, 8) 58712 ['c3[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c4 (Embedding) (None, 1, 8) 160368 ['c4[0][0]'] \n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c6 (Embedding) (None, 1, 8) 56840 ['c6[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c7 (Embedding) (None, 1, 8) 11056 ['c7[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]: warnings.warn(\n", - "Mon Sep 6 09:04:19 2021[1,0]:2021-09-06 09:04:19.311956: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c8 (Embedding) (None, 1, 8) 504 ['c8[0][0]'] \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:2021-09-06 09:04:19.311980: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:2021-09-06 09:04:19.312035: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c9 (Embedding) (None, 1, 8) 44432912 ['c9[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c10 (Embedding) (None, 1, 8) 4659752 ['c10[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c11 (Embedding) (None, 1, 8) 1966624 ['c11[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c12 (Embedding) (None, 1, 8) 88 ['c12[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c13 (Embedding) (None, 1, 8) 17672 ['c13[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c14 (Embedding) (None, 1, 8) 85336 ['c14[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c15 (Embedding) (None, 1, 8) 832 ['c15[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c17 (Embedding) (None, 1, 8) 7744 ['c17[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c18 (Embedding) (None, 1, 8) 120 ['c18[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c19 (Embedding) (None, 1, 8) 65327168 ['c19[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c20 (Embedding) (None, 1, 8) 21407520 ['c20[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c21 (Embedding) (None, 1, 8) 57251624 ['c21[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c22 (Embedding) (None, 1, 8) 2420128 ['c22[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c23 (Embedding) (None, 1, 8) 96176 ['c23[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c24 (Embedding) (None, 1, 8) 776 ['c24[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c25 (Embedding) (None, 1, 8) 280 ['c25[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embeddings_concat (Concatenate (None, 1, 192) 0 ['embedding_c0[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]:) 'embedding_c1[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c2[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c3[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c4[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c6[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c7[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c8[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c9[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c10[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c11[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c12[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c13[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c14[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c15[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c17[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c18[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c19[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c20[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c21[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c22[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c23[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c24[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c25[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c5 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c16 (InputLayer) [(None, 1)] 0 [] \n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embeddings_flatten (Flatten) (None, 192) 0 ['embeddings_concat[0][0]'] \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,10]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:one_hot_c5 (CategoryEncoding) (None, 4) 0 ['c5[0][0]'] \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,10]: warnings.warn(\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:one_hot_c16 (CategoryEncoding) (None, 4) 0 ['c16[0][0]'] \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,10]:2021-09-06 09:04:19.331721: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,10]:2021-09-06 09:04:19.331743: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:19 2021[1,10]:2021-09-06 09:04:19.331792: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:i0 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:i1 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:i2 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:i3 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:i4 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:i5 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:i6 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:i7 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:i8 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:i9 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:i10 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:i11 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,6]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:i12 (InputLayer) [(None, 1)] 0 [] \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,6]: warnings.warn(\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,6]:2021-09-06 09:04:19.341050: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:inputs_concat (Concatenate) (None, 213) 0 ['embeddings_flatten[0][0]', \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,6]:2021-09-06 09:04:19.341072: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]: 'one_hot_c5[0][0]', \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,6]:2021-09-06 09:04:19.341122: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]: 'one_hot_c16[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'i0[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'i1[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'i2[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'i3[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'i4[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'i5[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'i6[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'i7[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'i8[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'i9[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'i10[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'i11[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'i12[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:batch_normalization (BatchNorm (None, 213) 852 ['inputs_concat[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:alization) \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:dense (Dense) (None, 64) 13696 ['batch_normalization[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:batch_normalization_1 (BatchNo (None, 64) 256 ['dense[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:rmalization) \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:dense_1 (Dense) (None, 64) 4160 ['batch_normalization_1[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:batch_normalization_2 (BatchNo (None, 64) 256 ['dense_1[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:rmalization) \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:dense_2 (Dense) (None, 64) 4160 ['batch_normalization_2[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:batch_normalization_3 (BatchNo (None, 64) 256 ['dense_2[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:rmalization) \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:dense_3 (Dense) (None, 32) 2080 ['batch_normalization_3[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:output (Dense) (None, 1) 33 ['dense_3[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:==================================================================================================\n", - "Mon Sep 6 09:04:19 2021[1,0]:Total params: 261,698,789\n", - "Mon Sep 6 09:04:19 2021[1,0]:Trainable params: 261,697,979\n", - "Mon Sep 6 09:04:19 2021[1,0]:Non-trainable params: 810\n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,14]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 09:04:19 2021[1,14]: warnings.warn(\n", - "Mon Sep 6 09:04:19 2021[1,14]:2021-09-06 09:04:19.369461: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:04:19 2021[1,14]:2021-09-06 09:04:19.369483: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:19 2021[1,14]:2021-09-06 09:04:19.369533: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 09:04:19 2021[1,2]:2021-09-06 09:04:19.714971: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:19 2021[1,2]:2021-09-06 09:04:19.719251: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:19 2021[1,5]:2021-09-06 09:04:19.726330: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:19 2021[1,5]:2021-09-06 09:04:19.730619: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:19 2021[1,1]:2021-09-06 09:04:19.849382: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:19 2021[1,1]:2021-09-06 09:04:19.852162: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:19 2021[1,9]:2021-09-06 09:04:19.976160: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:19 2021[1,9]:2021-09-06 09:04:19.979006: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:20 2021[1,15]:2021-09-06 09:04:20.045149: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:20 2021[1,15]:2021-09-06 09:04:20.047988: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:20 2021[1,11]:2021-09-06 09:04:20.289338: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:20 2021[1,11]:2021-09-06 09:04:20.292348: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:20 2021[1,4]:2021-09-06 09:04:20.312786: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:20 2021[1,4]:2021-09-06 09:04:20.315716: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:20 2021[1,3]:2021-09-06 09:04:20.391964: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:20 2021[1,3]:2021-09-06 09:04:20.394729: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:20 2021[1,12]:2021-09-06 09:04:20.510335: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:20 2021[1,12]:2021-09-06 09:04:20.514644: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:20 2021[1,8]:2021-09-06 09:04:20.573990: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:20 2021[1,7]:2021-09-06 09:04:20.577094: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:20 2021[1,8]:2021-09-06 09:04:20.583448: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:20 2021[1,7]:2021-09-06 09:04:20.591334: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:20 2021[1,13]:2021-09-06 09:04:20.631241: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:20 2021[1,10]:2021-09-06 09:04:20.639145: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:20 2021[1,13]:2021-09-06 09:04:20.639919: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:20 2021[1,10]:2021-09-06 09:04:20.641938: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:20 2021[1,0]:2021-09-06 09:04:20.642622: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:20 2021[1,0]:2021-09-06 09:04:20.645310: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:20 2021[1,14]:2021-09-06 09:04:20.672134: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:20 2021[1,14]:2021-09-06 09:04:20.676577: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:20 2021[1,6]:2021-09-06 09:04:20.851373: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:20 2021[1,6]:2021-09-06 09:04:20.855704: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:45 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:45 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,0]:NCCL version 2.10.3+cuda11.0\n", - "Mon Sep 6 09:04:46 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:46 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Trees [0] 1/-1/-1->13->9 [1] 1/-1/-1->13->9 [2] 1/-1/-1->13->9 [3] 1/-1/-1->13->9 [4] 1/-1/-1->13->9 [5] 1/-1/-1->13->9 [6] 1/-1/-1->13->9 [7] 1/-1/-1->13->9 [8] 1/-1/-1->13->9 [9] 1/-1/-1->13->9 [10] 1/-1/-1->13->9 [11] 1/-1/-1->13->9\n", - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Setting affinity for GPU 4 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 09:04:57 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Trees [0] 6/-1/-1->12->11 [1] 6/-1/-1->12->11 [2] 6/-1/-1->12->11 [3] 6/-1/-1->12->11 [4] 6/-1/-1->12->11 [5] 6/-1/-1->12->11 [6] 6/-1/-1->12->11 [7] 6/-1/-1->12->11 [8] 6/-1/-1->12->11 [9] 6/-1/-1->12->11 [10] 6/-1/-1->12->11 [11] 6/-1/-1->12->11\n", - "Mon Sep 6 09:04:57 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Setting affinity for GPU 2 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Trees [0] 4/-1/-1->15->3 [1] 4/-1/-1->15->3 [2] 4/-1/-1->15->3 [3] 4/-1/-1->15->3 [4] 4/-1/-1->15->3 [5] 4/-1/-1->15->3 [6] 4/-1/-1->15->3 [7] 4/-1/-1->15->3 [8] 4/-1/-1->15->3 [9] 4/-1/-1->15->3 [10] 4/-1/-1->15->3 [11] 4/-1/-1->15->3\n", - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Setting affinity for GPU 14 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Trees [0] -1/-1/-1->14->4 [1] -1/-1/-1->14->4 [2] -1/-1/-1->14->4 [3] -1/-1/-1->14->4 [4] -1/-1/-1->14->4 [5] -1/-1/-1->14->4 [6] -1/-1/-1->14->4 [7] -1/-1/-1->14->4 [8] -1/-1/-1->14->4 [9] -1/-1/-1->14->4 [10] -1/-1/-1->14->4 [11] -1/-1/-1->14->4\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Setting affinity for GPU 12 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 00/12 : 0 8 11 12 6 10 9 13 1 5 2 7 3 15 4 14\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 01/12 : 0 8 11 12 6 10 9 13 1 5 2 7 3 15 4 14\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 02/12 : 0 8 11 12 6 10 9 13 1 5 2 7 3 15 4 14\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 03/12 : 0 8 11 12 6 10 9 13 1 5 2 7 3 15 4 14\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 04/12 : 0 8 11 12 6 10 9 13 1 5 2 7 3 15 4 14\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 05/12 : 0 8 11 12 6 10 9 13 1 5 2 7 3 15 4 14\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 06/12 : 0 8 11 12 6 10 9 13 1 5 2 7 3 15 4 14\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 07/12 : 0 8 11 12 6 10 9 13 1 5 2 7 3 15 4 14\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 08/12 : 0 8 11 12 6 10 9 13 1 5 2 7 3 15 4 14\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Trees [0] 5/-1/-1->1->13 [1] 5/-1/-1->1->13 [2] 5/-1/-1->1->13 [3] 5/-1/-1->1->13 [4] 5/-1/-1->1->13 [5] 5/-1/-1->1->13 [6] 5/-1/-1->1->13 [7] 5/-1/-1->1->13 [8] 5/-1/-1->1->13 [9] 5/-1/-1->1->13 [10] 5/-1/-1->1->13 [11] 5/-1/-1->1->13\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Setting affinity for GPU 11 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 09/12 : 0 8 11 12 6 10 9 13 1 5 2 7 3 15 4 14\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 10/12 : 0 8 11 12 6 10 9 13 1 5 2 7 3 15 4 14\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 11/12 : 0 8 11 12 6 10 9 13 1 5 2 7 3 15 4 14\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Trees [0] 8/-1/-1->0->-1 [1] 8/-1/-1->0->-1 [2] 8/-1/-1->0->-1 [3] 8/-1/-1->0->-1 [4] 8/-1/-1->0->-1 [5] 8/-1/-1->0->-1 [6] 8/-1/-1->0->-1 [7] 8/-1/-1->0->-1 [8] 8/-1/-1->0->-1 [9] 8/-1/-1->0->-1 [10] 8/-1/-1->0->-1 [11] 8/-1/-1->0->-1\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Trees [0] 12/-1/-1->11->8 [1] 12/-1/-1->11->8 [2] 12/-1/-1->11->8 [3] 12/-1/-1->11->8 [4] 12/-1/-1->11->8 [5] 12/-1/-1->11->8 [6] 12/-1/-1->11->8 [7] 12/-1/-1->11->8 [8] 12/-1/-1->11->8 [9] 12/-1/-1->11->8 [10] 12/-1/-1->11->8 [11] 12/-1/-1->11->8\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Setting affinity for GPU 3 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Trees [0] 7/-1/-1->2->5 [1] 7/-1/-1->2->5 [2] 7/-1/-1->2->5 [3] 7/-1/-1->2->5 [4] 7/-1/-1->2->5 [5] 7/-1/-1->2->5 [6] 7/-1/-1->2->5 [7] 7/-1/-1->2->5 [8] 7/-1/-1->2->5 [9] 7/-1/-1->2->5 [10] 7/-1/-1->2->5 [11] 7/-1/-1->2->5\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Setting affinity for GPU 8 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 09:04:57 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Trees [0] 2/-1/-1->5->1 [1] 2/-1/-1->5->1 [2] 2/-1/-1->5->1 [3] 2/-1/-1->5->1 [4] 2/-1/-1->5->1 [5] 2/-1/-1->5->1 [6] 2/-1/-1->5->1 [7] 2/-1/-1->5->1 [8] 2/-1/-1->5->1 [9] 2/-1/-1->5->1 [10] 2/-1/-1->5->1 [11] 2/-1/-1->5->1\n", - "Mon Sep 6 09:04:57 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Setting affinity for GPU 10 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 09:04:57 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Trees [0] 3/-1/-1->7->2 [1] 3/-1/-1->7->2 [2] 3/-1/-1->7->2 [3] 3/-1/-1->7->2 [4] 3/-1/-1->7->2 [5] 3/-1/-1->7->2 [6] 3/-1/-1->7->2 [7] 3/-1/-1->7->2 [8] 3/-1/-1->7->2 [9] 3/-1/-1->7->2 [10] 3/-1/-1->7->2 [11] 3/-1/-1->7->2\n", - "Mon Sep 6 09:04:57 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Setting affinity for GPU 9 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Trees [0] 15/-1/-1->3->7 [1] 15/-1/-1->3->7 [2] 15/-1/-1->3->7 [3] 15/-1/-1->3->7 [4] 15/-1/-1->3->7 [5] 15/-1/-1->3->7 [6] 15/-1/-1->3->7 [7] 15/-1/-1->3->7 [8] 15/-1/-1->3->7 [9] 15/-1/-1->3->7 [10] 15/-1/-1->3->7 [11] 15/-1/-1->3->7\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Setting affinity for GPU 15 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Setting affinity for GPU 1 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 09:04:57 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Trees [0] 14/-1/-1->4->15 [1] 14/-1/-1->4->15 [2] 14/-1/-1->4->15 [3] 14/-1/-1->4->15 [4] 14/-1/-1->4->15 [5] 14/-1/-1->4->15 [6] 14/-1/-1->4->15 [7] 14/-1/-1->4->15 [8] 14/-1/-1->4->15 [9] 14/-1/-1->4->15 [10] 14/-1/-1->4->15 [11] 14/-1/-1->4->15\n", - "Mon Sep 6 09:04:57 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Setting affinity for GPU 13 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Trees [0] 10/-1/-1->6->12 [1] 10/-1/-1->6->12 [2] 10/-1/-1->6->12 [3] 10/-1/-1->6->12 [4] 10/-1/-1->6->12 [5] 10/-1/-1->6->12 [6] 10/-1/-1->6->12 [7] 10/-1/-1->6->12 [8] 10/-1/-1->6->12 [9] 10/-1/-1->6->12 [10] 10/-1/-1->6->12 [11] 10/-1/-1->6->12\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Setting affinity for GPU 6 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Trees [0] 11/-1/-1->8->0 [1] 11/-1/-1->8->0 [2] 11/-1/-1->8->0 [3] 11/-1/-1->8->0 [4] 11/-1/-1->8->0 [5] 11/-1/-1->8->0 [6] 11/-1/-1->8->0 [7] 11/-1/-1->8->0 [8] 11/-1/-1->8->0 [9] 11/-1/-1->8->0 [10] 11/-1/-1->8->0 [11] 11/-1/-1->8->0\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Setting affinity for GPU 0 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Trees [0] 13/-1/-1->9->10 [1] 13/-1/-1->9->10 [2] 13/-1/-1->9->10 [3] 13/-1/-1->9->10 [4] 13/-1/-1->9->10 [5] 13/-1/-1->9->10 [6] 13/-1/-1->9->10 [7] 13/-1/-1->9->10 [8] 13/-1/-1->9->10 [9] 13/-1/-1->9->10 [10] 13/-1/-1->9->10 [11] 13/-1/-1->9->10\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Setting affinity for GPU 5 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 09:04:57 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Trees [0] 9/-1/-1->10->6 [1] 9/-1/-1->10->6 [2] 9/-1/-1->10->6 [3] 9/-1/-1->10->6 [4] 9/-1/-1->10->6 [5] 9/-1/-1->10->6 [6] 9/-1/-1->10->6 [7] 9/-1/-1->10->6 [8] 9/-1/-1->10->6 [9] 9/-1/-1->10->6 [10] 9/-1/-1->10->6 [11] 9/-1/-1->10->6\n", - "Mon Sep 6 09:04:57 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Setting affinity for GPU 7 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 00 : 15[e5000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 00 : 6[5c000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 00 : 8[34000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 00 : 9[59000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 00 : 11[3b000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 00 : 14[e0000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 00 : 2[b7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 01 : 15[e5000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 01 : 6[5c000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 01 : 9[59000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 01 : 8[34000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 01 : 11[3b000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 01 : 14[e0000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 01 : 2[b7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 02 : 15[e5000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 02 : 6[5c000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 02 : 8[34000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 02 : 9[59000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 02 : 11[3b000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 02 : 14[e0000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 02 : 2[b7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 03 : 15[e5000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 03 : 6[5c000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 03 : 9[59000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 03 : 8[34000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 03 : 11[3b000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 03 : 14[e0000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 03 : 2[b7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 04 : 15[e5000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 04 : 6[5c000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 04 : 8[34000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 04 : 9[59000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 04 : 11[3b000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 04 : 14[e0000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 04 : 2[b7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 05 : 15[e5000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 05 : 6[5c000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 05 : 9[59000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 05 : 8[34000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 05 : 11[3b000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 05 : 14[e0000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 05 : 2[b7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 06 : 15[e5000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 06 : 6[5c000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 06 : 9[59000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 06 : 8[34000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 06 : 11[3b000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 06 : 14[e0000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 06 : 2[b7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 07 : 15[e5000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 07 : 6[5c000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 07 : 9[59000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 07 : 8[34000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 07 : 11[3b000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 07 : 14[e0000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 07 : 2[b7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 08 : 15[e5000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 08 : 6[5c000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 08 : 8[34000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 08 : 9[59000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 08 : 11[3b000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 08 : 14[e0000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 08 : 2[b7000] -> 7[b9000] via P2P/IPC\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 09 : 15[e5000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 09 : 6[5c000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 09 : 9[59000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 09 : 8[34000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 09 : 11[3b000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 09 : 14[e0000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 09 : 2[b7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 10 : 15[e5000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 10 : 6[5c000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 10 : 9[59000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 10 : 8[34000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 10 : 11[3b000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 10 : 14[e0000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 10 : 2[b7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 11 : 15[e5000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 11 : 6[5c000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 11 : 9[59000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 11 : 8[34000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 11 : 11[3b000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 11 : 14[e0000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 11 : 2[b7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 00 : 13[57000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 00 : 3[e7000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 00 : 1[be000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 01 : 13[57000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 01 : 3[e7000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 01 : 1[be000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 02 : 13[57000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 02 : 3[e7000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 02 : 1[be000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 03 : 13[57000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 03 : 3[e7000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 03 : 1[be000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 04 : 13[57000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 04 : 3[e7000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 05 : 13[57000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 04 : 1[be000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 05 : 3[e7000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 05 : 1[be000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 06 : 13[57000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 06 : 3[e7000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 07 : 13[57000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 06 : 1[be000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 07 : 3[e7000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 07 : 1[be000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 08 : 13[57000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 08 : 3[e7000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 08 : 1[be000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 09 : 13[57000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 09 : 1[be000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 09 : 3[e7000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 10 : 13[57000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 10 : 3[e7000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 10 : 1[be000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 11 : 13[57000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 11 : 1[be000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 11 : 3[e7000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 00 : 12[39000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 01 : 12[39000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 02 : 12[39000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 03 : 12[39000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 00 : 10[5e000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 04 : 12[39000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 01 : 10[5e000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 05 : 12[39000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 02 : 10[5e000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 06 : 12[39000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 03 : 10[5e000] -> 9[59000] via P2P/IPC\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 07 : 12[39000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 04 : 10[5e000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 08 : 12[39000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 05 : 10[5e000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 09 : 12[39000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 06 : 10[5e000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 00 : 7[b9000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 10 : 12[39000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 07 : 10[5e000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 01 : 7[b9000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 11 : 12[39000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 08 : 10[5e000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 02 : 7[b9000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 09 : 10[5e000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 03 : 7[b9000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 10 : 10[5e000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 04 : 7[b9000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 11 : 10[5e000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 05 : 7[b9000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 06 : 7[b9000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 00 : 0[36000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 07 : 7[b9000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 01 : 0[36000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 08 : 7[b9000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 02 : 0[36000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 09 : 7[b9000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 03 : 0[36000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 10 : 7[b9000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 04 : 0[36000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 11 : 7[b9000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 05 : 0[36000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 06 : 0[36000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 00 : 4[e2000] -> 14[e0000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 07 : 0[36000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 01 : 4[e2000] -> 14[e0000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 08 : 0[36000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 02 : 4[e2000] -> 14[e0000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 09 : 0[36000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 03 : 4[e2000] -> 14[e0000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 10 : 0[36000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 04 : 4[e2000] -> 14[e0000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 11 : 0[36000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 05 : 4[e2000] -> 14[e0000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 06 : 4[e2000] -> 14[e0000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 07 : 4[e2000] -> 14[e0000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 08 : 4[e2000] -> 14[e0000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 09 : 4[e2000] -> 14[e0000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 10 : 4[e2000] -> 14[e0000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 11 : 4[e2000] -> 14[e0000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 00 : 5[bc000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 01 : 5[bc000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 02 : 5[bc000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 03 : 5[bc000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 00 : 11[3b000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 04 : 5[bc000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 01 : 11[3b000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 05 : 5[bc000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 02 : 11[3b000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 06 : 5[bc000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 03 : 11[3b000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 07 : 5[bc000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 04 : 11[3b000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 08 : 5[bc000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 05 : 11[3b000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 09 : 5[bc000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 06 : 11[3b000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 00 : 6[5c000] -> 12[39000] via P2P/IPC\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 10 : 5[bc000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 07 : 11[3b000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 01 : 6[5c000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 11 : 5[bc000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 08 : 11[3b000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 02 : 6[5c000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 09 : 11[3b000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 03 : 6[5c000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 10 : 11[3b000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 04 : 6[5c000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 11 : 11[3b000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 05 : 6[5c000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 06 : 6[5c000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 07 : 6[5c000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 08 : 6[5c000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 09 : 6[5c000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 10 : 6[5c000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 11 : 6[5c000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 00 : 1[be000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 00 : 15[e5000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 00 : 12[39000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 01 : 1[be000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 01 : 15[e5000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 01 : 12[39000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 02 : 1[be000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 02 : 15[e5000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 02 : 12[39000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 00 : 13[57000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 03 : 1[be000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 03 : 15[e5000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 03 : 12[39000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 01 : 13[57000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 04 : 1[be000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 04 : 15[e5000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 04 : 12[39000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 02 : 13[57000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 05 : 1[be000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 05 : 15[e5000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 05 : 12[39000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 06 : 1[be000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 03 : 13[57000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 06 : 15[e5000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 06 : 12[39000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 04 : 13[57000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 07 : 1[be000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 07 : 15[e5000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 07 : 12[39000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 05 : 13[57000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 08 : 1[be000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 08 : 15[e5000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 08 : 12[39000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 06 : 13[57000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 09 : 1[be000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 09 : 15[e5000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 00 : 14[e0000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 09 : 12[39000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 07 : 13[57000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 10 : 1[be000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 10 : 15[e5000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 01 : 14[e0000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 10 : 12[39000] -> 11[3b000] via P2P/IPC\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:58 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 08 : 13[57000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 11 : 1[be000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 11 : 15[e5000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 02 : 14[e0000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 11 : 12[39000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 09 : 13[57000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 03 : 14[e0000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 10 : 13[57000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 00 : 9[59000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 04 : 14[e0000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 01 : 9[59000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 11 : 13[57000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 05 : 14[e0000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 02 : 9[59000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 06 : 14[e0000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 03 : 9[59000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 00 : 2[b7000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 07 : 14[e0000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 04 : 9[59000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 01 : 2[b7000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 08 : 14[e0000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 05 : 9[59000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 02 : 2[b7000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 09 : 14[e0000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 06 : 9[59000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 03 : 2[b7000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 10 : 14[e0000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 07 : 9[59000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 04 : 2[b7000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 11 : 14[e0000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 08 : 9[59000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 05 : 2[b7000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 09 : 9[59000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 06 : 2[b7000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 00 : 8[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 10 : 9[59000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 07 : 2[b7000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 01 : 8[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 11 : 9[59000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 08 : 2[b7000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 02 : 8[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 09 : 2[b7000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 03 : 8[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 10 : 2[b7000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 04 : 8[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 11 : 2[b7000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 05 : 8[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:58 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 06 : 8[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 00 : 4[e2000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 07 : 8[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 01 : 4[e2000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 08 : 8[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 02 : 4[e2000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 09 : 8[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 03 : 4[e2000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 10 : 8[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 00 : 10[5e000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 04 : 4[e2000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 01 : 10[5e000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 11 : 8[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 05 : 4[e2000] -> 15[e5000] via P2P/IPC\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 02 : 10[5e000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 06 : 4[e2000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 03 : 10[5e000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 00 : 5[bc000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 07 : 4[e2000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 04 : 10[5e000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 01 : 5[bc000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 08 : 4[e2000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 05 : 10[5e000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 00 : 3[e7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 02 : 5[bc000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 09 : 4[e2000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 06 : 10[5e000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 01 : 3[e7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 03 : 5[bc000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 10 : 4[e2000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 07 : 10[5e000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 04 : 5[bc000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 02 : 3[e7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 11 : 4[e2000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 08 : 10[5e000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 05 : 5[bc000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 03 : 3[e7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 09 : 10[5e000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 06 : 5[bc000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 04 : 3[e7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 10 : 10[5e000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 07 : 5[bc000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 05 : 3[e7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 11 : 10[5e000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 08 : 5[bc000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 06 : 3[e7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 09 : 5[bc000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 07 : 3[e7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 10 : 5[bc000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 08 : 3[e7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 11 : 5[bc000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 09 : 3[e7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 10 : 3[e7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 11 : 3[e7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:59 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 00 : 7[b9000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:59 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 01 : 7[b9000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 02 : 7[b9000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:59 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 03 : 7[b9000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:59 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 04 : 7[b9000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:59 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 05 : 7[b9000] -> 2[b7000] via P2P/IPC\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:59 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 06 : 7[b9000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:59 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 07 : 7[b9000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 08 : 7[b9000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:59 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 09 : 7[b9000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 10 : 7[b9000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 11 : 7[b9000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:59 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO comm 0x7f159c62d200 rank 4 nranks 16 cudaDev 0 busId e2000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO comm 0x7fec6462ce60 rank 7 nranks 16 cudaDev 0 busId b9000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO comm 0x7fbe3862ca40 rank 15 nranks 16 cudaDev 0 busId e5000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO comm 0x7f00e462e120 rank 12 nranks 16 cudaDev 0 busId 39000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO comm 0x7f9c0062da40 rank 2 nranks 16 cudaDev 0 busId b7000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO comm 0x7f3e2462d9e0 rank 8 nranks 16 cudaDev 0 busId 34000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO comm 0x7f50ac62cfb0 rank 10 nranks 16 cudaDev 0 busId 5e000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO comm 0x7f47c062cad0 rank 11 nranks 16 cudaDev 0 busId 3b000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO comm 0x7f7c2c62d050 rank 5 nranks 16 cudaDev 0 busId bc000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO comm 0x7f10a862d190 rank 6 nranks 16 cudaDev 0 busId 5c000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO comm 0x7f635462cf70 rank 1 nranks 16 cudaDev 0 busId be000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO comm 0x7fa9b065f3d0 rank 0 nranks 16 cudaDev 0 busId 36000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO comm 0x7f330c62cf00 rank 9 nranks 16 cudaDev 0 busId 59000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO comm 0x7f786062ca30 rank 3 nranks 16 cudaDev 0 busId e7000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO comm 0x7ff78062ce20 rank 13 nranks 16 cudaDev 0 busId 57000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO comm 0x7fa09462d170 rank 14 nranks 16 cudaDev 0 busId e0000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Launch mode Parallel\n", - "Mon Sep 6 09:04:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:05:08 2021[1,3]:2021-09-06 09:05:08.906524: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,3]:2021-09-06 09:05:08.906568: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,9]:2021-09-06 09:05:08.907713: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,9]:2021-09-06 09:05:08.907738: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,11]:2021-09-06 09:05:08.915421: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,11]:2021-09-06 09:05:08.915453: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,1]:2021-09-06 09:05:08.920370: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,1]:2021-09-06 09:05:08.920406: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,15]:2021-09-06 09:05:08.931629: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,15]:2021-09-06 09:05:08.931668: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,0]:2021-09-06 09:05:08.935354: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,0]:2021-09-06 09:05:08.935382: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,4]:2021-09-06 09:05:08.941341: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,4]:2021-09-06 09:05:08.941368: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,6]:2021-09-06 09:05:08.945179: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,6]:2021-09-06 09:05:08.945222: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,12]:2021-09-06 09:05:08.953442: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,12]:2021-09-06 09:05:08.953478: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,5]:2021-09-06 09:05:08.955782: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,5]:2021-09-06 09:05:08.955817: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,13]:2021-09-06 09:05:08.955878: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,13]:2021-09-06 09:05:08.955902: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,2]:2021-09-06 09:05:08.964502: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,2]:2021-09-06 09:05:08.964533: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,10]:2021-09-06 09:05:08.966637: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,10]:2021-09-06 09:05:08.966668: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,7]:2021-09-06 09:05:08.975908: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,7]:2021-09-06 09:05:08.975936: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,8]:2021-09-06 09:05:08.977191: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,8]:2021-09-06 09:05:08.977212: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,14]:2021-09-06 09:05:08.977981: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,14]:2021-09-06 09:05:08.978007: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:10 2021[1,15]:2021-09-06 09:05:10.305228: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,2]:2021-09-06 09:05:10.305433: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,4]:2021-09-06 09:05:10.305459: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,5]:2021-09-06 09:05:10.305543: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,15]:2021-09-06 09:05:10.306514: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,2]:2021-09-06 09:05:10.306618: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,5]:2021-09-06 09:05:10.306708: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,4]:2021-09-06 09:05:10.307508: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,13]:2021-09-06 09:05:10.309214: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,6]:2021-09-06 09:05:10.309215: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,7]:2021-09-06 09:05:10.309278: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,9]:2021-09-06 09:05:10.309264: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,10]:2021-09-06 09:05:10.309370: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,13]:2021-09-06 09:05:10.310410: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,7]:2021-09-06 09:05:10.310457: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,9]:2021-09-06 09:05:10.310481: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,10]:2021-09-06 09:05:10.310655: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,6]:2021-09-06 09:05:10.311490: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,3]:2021-09-06 09:05:10.316781: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,3]:2021-09-06 09:05:10.319146: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 1/4000 [..............................]Mon Sep 6 09:04:59 2021[1,0]: - ETA: 31:27:06 - loss: 0.9112 - auc: 0.4892Mon Sep 6 09:05:10 2021[1,0]:\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:05:10 2021[1,11]:2021-09-06 09:05:10.342218: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,8]:2021-09-06 09:05:10.342267: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,1]:2021-09-06 09:05:10.342284: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,14]:2021-09-06 09:05:10.342547: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,0]:2021-09-06 09:05:10.342667: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,11]:2021-09-06 09:05:10.343662: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,1]:2021-09-06 09:05:10.343704: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,0]:2021-09-06 09:05:10.344116: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,8]:2021-09-06 09:05:10.344609: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,12]:2021-09-06 09:05:10.345052: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,14]:2021-09-06 09:05:10.345700: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,12]:2021-09-06 09:05:10.346509: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,2]:2021-09-06 09:05:10.358042: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2649 callback api events and 2583 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,13]:2021-09-06 09:05:10.358854: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2649 callback api events and 2583 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,7]:2021-09-06 09:05:10.359228: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2648 callback api events and 2582 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,9]:2021-09-06 09:05:10.360356: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2648 callback api events and 2582 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,4]:2021-09-06 09:05:10.364826: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2648 callback api events and 2582 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,15]:2021-09-06 09:05:10.367892: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2648 callback api events and 2582 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,10]:2021-09-06 09:05:10.368670: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2652 callback api events and 2586 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,6]:2021-09-06 09:05:10.369775: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2649 callback api events and 2583 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,5]:2021-09-06 09:05:10.373559: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2651 callback api events and 2585 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,3]:2021-09-06 09:05:10.383506: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2648 callback api events and 2582 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,0]:2021-09-06 09:05:10.392125: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2648 callback api events and 2582 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,11]:2021-09-06 09:05:10.392648: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2648 callback api events and 2582 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,12]:2021-09-06 09:05:10.393439: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2648 callback api events and 2582 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,1]:2021-09-06 09:05:10.393559: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2648 callback api events and 2582 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,2]:2021-09-06 09:05:10.400431: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,13]:2021-09-06 09:05:10.402509: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,7]:2021-09-06 09:05:10.402749: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,8]:2021-09-06 09:05:10.405715: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2648 callback api events and 2582 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,9]:2021-09-06 09:05:10.410689: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,6]:2021-09-06 09:05:10.415497: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,15]:2021-09-06 09:05:10.415594: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,5]:2021-09-06 09:05:10.415980: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,4]:2021-09-06 09:05:10.419823: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,10]:2021-09-06 09:05:10.424162: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,3]:2021-09-06 09:05:10.426899: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,0]:2021-09-06 09:05:10.436674: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,12]:2021-09-06 09:05:10.437051: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,14]:2021-09-06 09:05:10.437544: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2648 callback api events and 2582 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,1]:2021-09-06 09:05:10.437594: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,11]:2021-09-06 09:05:10.440492: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,2]:2021-09-06 09:05:10.452744: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,2]:\n", - "Mon Sep 6 09:05:10 2021[1,8]:2021-09-06 09:05:10.453531: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,13]:2021-09-06 09:05:10.454222: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,13]:\n", - "Mon Sep 6 09:05:10 2021[1,7]:2021-09-06 09:05:10.454482: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,7]:\n", - "Mon Sep 6 09:05:10 2021[1,5]:2021-09-06 09:05:10.467085: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:05:10 2021[1,5]:\n", - "Mon Sep 6 09:05:10 2021[1,9]:2021-09-06 09:05:10.468653: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,9]:\n", - "Mon Sep 6 09:05:10 2021[1,6]:2021-09-06 09:05:10.469858: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,6]:\n", - "Mon Sep 6 09:05:10 2021[1,15]:2021-09-06 09:05:10.471366: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,15]:\n", - "Mon Sep 6 09:05:10 2021[1,4]:2021-09-06 09:05:10.477411: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,4]:\n", - "Mon Sep 6 09:05:10 2021[1,14]:2021-09-06 09:05:10.478317: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,10]:2021-09-06 09:05:10.479776: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,10]:\n", - "Mon Sep 6 09:05:10 2021[1,2]:2021-09-06 09:05:10.480068: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,13]:2021-09-06 09:05:10.481862: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,7]:2021-09-06 09:05:10.482391: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,3]:2021-09-06 09:05:10.483605: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,3]:\n", - "Mon Sep 6 09:05:10 2021[1,12]:2021-09-06 09:05:10.485892: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,12]:\n", - "Mon Sep 6 09:05:10 2021[1,1]:2021-09-06 09:05:10.488111: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,1]:\n", - "Mon Sep 6 09:05:10 2021[1,0]:2021-09-06 09:05:10.488549: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,0]:\n", - "Mon Sep 6 09:05:10 2021[1,11]:2021-09-06 09:05:10.492599: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,11]:\n", - "Mon Sep 6 09:05:10 2021[1,5]:2021-09-06 09:05:10.495745: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,9]:2021-09-06 09:05:10.497173: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,6]:2021-09-06 09:05:10.498271: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,15]:2021-09-06 09:05:10.502046: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,8]:2021-09-06 09:05:10.505250: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,8]:\n", - "Mon Sep 6 09:05:10 2021[1,4]:2021-09-06 09:05:10.505717: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,10]:2021-09-06 09:05:10.510133: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,3]:2021-09-06 09:05:10.511769: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,12]:2021-09-06 09:05:10.513653: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,1]:2021-09-06 09:05:10.516357: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,0]:2021-09-06 09:05:10.516749: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,11]:2021-09-06 09:05:10.520974: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,14]:2021-09-06 09:05:10.533211: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,14]:\n", - "Mon Sep 6 09:05:10 2021[1,8]:2021-09-06 09:05:10.533268: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,2]:2021-09-06 09:05:10.543716: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,2]:\n", - "Mon Sep 6 09:05:10 2021[1,13]:2021-09-06 09:05:10.544599: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,13]:\n", - "Mon Sep 6 09:05:10 2021[1,7]:2021-09-06 09:05:10.545527: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,7]:\n", - "Mon Sep 6 09:05:10 2021[1,2]:2021-09-06 09:05:10.551058: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,13]:2021-09-06 09:05:10.551684: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,7]:2021-09-06 09:05:10.552556: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,2]:2021-09-06 09:05:10.554892: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,2]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:05:10 2021[1,2]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,2]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,2]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,2]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,2]:\n", - "Mon Sep 6 09:05:10 2021[1,13]:2021-09-06 09:05:10.555408: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,13]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,13]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,13]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,13]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,13]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,13]:\n", - "Mon Sep 6 09:05:10 2021[1,7]:2021-09-06 09:05:10.556035: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,7]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,7]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,7]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,7]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,7]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,7]:\n", - "Mon Sep 6 09:05:10 2021[1,9]:2021-09-06 09:05:10.559516: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,9]:\n", - "Mon Sep 6 09:05:10 2021[1,5]:2021-09-06 09:05:10.560753: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,5]:\n", - "Mon Sep 6 09:05:10 2021[1,6]:2021-09-06 09:05:10.562556: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,6]:\n", - "Mon Sep 6 09:05:10 2021[1,9]:2021-09-06 09:05:10.566678: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,5]:2021-09-06 09:05:10.568086: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,6]:2021-09-06 09:05:10.569781: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,9]:2021-09-06 09:05:10.570665: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,9]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,9]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,9]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,9]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,9]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,9]:\n", - "Mon Sep 6 09:05:10 2021[1,14]:2021-09-06 09:05:10.572285: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,5]:2021-09-06 09:05:10.572728: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,5]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,5]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,5]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,5]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,5]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,5]:\n", - "Mon Sep 6 09:05:10 2021[1,6]:2021-09-06 09:05:10.573924: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,6]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,6]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,6]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,6]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,6]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,6]:\n", - "Mon Sep 6 09:05:10 2021[1,12]:2021-09-06 09:05:10.578011: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,12]:\n", - "Mon Sep 6 09:05:10 2021[1,3]:2021-09-06 09:05:10.580067: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,3]:\n", - "Mon Sep 6 09:05:10 2021[1,4]:2021-09-06 09:05:10.581250: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,4]:\n", - "Mon Sep 6 09:05:10 2021[1,0]:2021-09-06 09:05:10.581588: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,0]:\n", - "Mon Sep 6 09:05:10 2021[1,12]:2021-09-06 09:05:10.585279: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:05:10 2021[1,11]:2021-09-06 09:05:10.585794: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,11]:\n", - "Mon Sep 6 09:05:10 2021[1,15]:2021-09-06 09:05:10.586581: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,15]:\n", - "Mon Sep 6 09:05:10 2021[1,3]:2021-09-06 09:05:10.587215: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,12]:2021-09-06 09:05:10.589198: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,12]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,12]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,12]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,12]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,12]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,12]:\n", - "Mon Sep 6 09:05:10 2021[1,0]:2021-09-06 09:05:10.589550: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,4]:2021-09-06 09:05:10.590763: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,3]:2021-09-06 09:05:10.591167: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,3]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,3]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,3]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,3]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,3]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,3]:\n", - "Mon Sep 6 09:05:10 2021[1,11]:2021-09-06 09:05:10.593091: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,0]:2021-09-06 09:05:10.594288: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,0]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,0]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,0]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,0]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,0]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,0]:\n", - "Mon Sep 6 09:05:10 2021[1,15]:2021-09-06 09:05:10.596368: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,4]:2021-09-06 09:05:10.596671: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,4]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,4]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,4]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,4]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,4]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,4]:\n", - "Mon Sep 6 09:05:10 2021[1,11]:2021-09-06 09:05:10.597264: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,11]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,11]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,11]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,11]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,11]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,11]:\n", - "Mon Sep 6 09:05:10 2021[1,10]:2021-09-06 09:05:10.598865: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,10]:\n", - "Mon Sep 6 09:05:10 2021[1,1]:2021-09-06 09:05:10.601237: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,1]:\n", - "Mon Sep 6 09:05:10 2021[1,15]:2021-09-06 09:05:10.601247: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,15]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,15]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,15]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,15]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,15]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,15]:\n", - "Mon Sep 6 09:05:10 2021[1,8]:2021-09-06 09:05:10.601392: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:05:10 2021[1,8]:\n", - "Mon Sep 6 09:05:10 2021[1,8]:2021-09-06 09:05:10.609086: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,10]:2021-09-06 09:05:10.609124: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,1]:2021-09-06 09:05:10.609444: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,8]:2021-09-06 09:05:10.614234: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,8]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,8]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,8]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,8]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,8]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,8]:\n", - "Mon Sep 6 09:05:10 2021[1,10]:2021-09-06 09:05:10.614705: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,10]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,10]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,10]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,10]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,10]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,10]:\n", - "Mon Sep 6 09:05:10 2021[1,1]:2021-09-06 09:05:10.617210: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,1]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,1]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,1]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,1]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,1]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,1]:\n", - "Mon Sep 6 09:05:10 2021[1,14]:2021-09-06 09:05:10.677166: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,14]:\n", - "Mon Sep 6 09:05:10 2021[1,14]:2021-09-06 09:05:10.684214: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,14]:2021-09-06 09:05:10.687649: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,14]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,14]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,14]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,14]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,14]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,14]:\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 5/4000 [..............................]Mon Sep 6 09:05:11 2021[1,0]: - ETA: 3:15:09 - loss: 0.8028 - auc: 0.4982Mon Sep 6 09:05:11 2021[1,0]Mon Sep 6 09:05:10 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:05:11 2021[1,12]:WARNING:tensorflow:Callback method `on_train_batch_begin` is slow compared to the batch time (batch time: 0.1455s vs `on_train_batch_begin` time: 0.1987s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,12]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1455s vs `on_train_batch_end` time: 1.6635s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,3]:WARNING:tensorflow:Callback method `on_train_batch_begin` is slow compared to the batch time (batch time: 0.1708s vs `on_train_batch_begin` time: 0.2090s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,3]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1708s vs `on_train_batch_end` time: 1.6582s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,10]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1720s vs `on_train_batch_end` time: 1.6715s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,7]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1905s vs `on_train_batch_end` time: 1.6623s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,8]:WARNING:tensorflow:Callback method `on_train_batch_begin` is slow compared to the batch time (batch time: 0.1486s vs `on_train_batch_begin` time: 0.1849s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,13]:WARNING:tensorflow:Callback method `on_train_batch_begin` is slow compared to the batch time (batch time: 0.1773s vs `on_train_batch_begin` time: 0.2086s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,13]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1773s vs `on_train_batch_end` time: 1.6576s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,5]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1958s vs `on_train_batch_end` time: 1.6611s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,1]:WARNING:tensorflow:Callback method `on_train_batch_begin` is slow compared to the batch time (batch time: 0.1316s vs `on_train_batch_begin` time: 0.2118s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,1]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1316s vs `on_train_batch_end` time: 1.6669s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,8]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1486s vs `on_train_batch_end` time: 1.6714s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,4]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1880s vs `on_train_batch_end` time: 1.6633s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,6]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1824s vs `on_train_batch_end` time: 1.6600s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,2]:WARNING:tensorflow:Callback method `on_train_batch_begin` is slow compared to the batch time (batch time: 0.1603s vs `on_train_batch_begin` time: 0.2171s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,11]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1957s vs `on_train_batch_end` time: 1.6587s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,2]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1603s vs `on_train_batch_end` time: 1.6624s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,15]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1938s vs `on_train_batch_end` time: 1.6640s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,9]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1979s vs `on_train_batch_end` time: 1.6555s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,14]:WARNING:tensorflow:Callback method `on_train_batch_begin` is slow compared to the batch time (batch time: 0.1465s vs `on_train_batch_begin` time: 0.1744s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,14]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1465s vs `on_train_batch_end` time: 1.6863s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,0]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1925s vs `on_train_batch_end` time: 1.6647s). Check your callbacks.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 212/4000 [>.............................]Mon Sep 6 09:05:52 2021[1,0]: - ETA: 15:58 - loss: 0.1657 - auc: 0.6865Mon Sep 6 09:05:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 513/4000 [==>...........................]Mon Sep 6 09:06:52 2021[1,0]: - ETA: 12:52 - loss: 0.1432 - auc: 0.7375Mon Sep 6 09:06:52 2021[1,0] (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 799/4000 [====>.........................]Mon Sep 6 09:07:50 2021[1,0]: - ETA: 11:25 - loss: 0.1371 - auc: 0.7529Mon Sep 6 09:07:52 2021[1,0] (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1102/4000 [=======>......................]Mon Sep 6 09:08:52 2021[1,0]: - ETA: 10:14 - loss: 0.1339 - auc: 0.7614Mon Sep 6 09:08:52 2021[1,0] (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1398/4000 [=========>....................]Mon Sep 6 09:09:49 2021[1,0]: - ETA: 9:01 - loss: 0.1321 - auc: 0.7666Mon Sep 6 09:09:50 2021[1,0]3Mon Sep 6 09:09:47 2021[1,0]>\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1702/4000 [===========>..................]Mon Sep 6 09:10:52 2021[1,0]: - ETA: 7:57 - loss: 0.1308 - auc: 0.7706Mon Sep 6 09:10:52 2021[1,0]5Mon Sep 6 09:10:52 2021[1,0]Mon Sep 6 09:10:08 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:11:50 2021[1,0]:1998/4000 [=============>................] - ETA: 6:52 - loss: 0.1299 - auc: 0.7735Mon Sep 6 09:11:50 2021[1,0]1Mon Sep 6 09:11:42 2021[1,0]Mon Sep 6 09:10:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2300/4000 [================>.............]Mon Sep 6 09:12:52 2021[1,0]: - ETA: 5:50 - loss: 0.1292 - auc: 0.7758Mon Sep 6 09:12:52 2021[1,0]8Mon Sep 6 09:12:52 2021[1,0]\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2599/4000 [==================>...........]Mon Sep 6 09:13:50 2021[1,0]: - ETA: 4:46 - loss: 0.1286 - auc: 0.7776Mon Sep 6 09:13:53 2021[1,0]6Mon Sep 6 09:13:50 2021[1,0]\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2910/4000 [====================>.........]Mon Sep 6 09:14:52 2021[1,0]: - ETA: 3:42 - loss: 0.1282 - auc: 0.7792Mon Sep 6 09:14:53 2021[1,0]2Mon Sep 6 09:14:52 2021[1,0]\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3199/4000 [======================>.......]Mon Sep 6 09:15:49 2021[1,0]: - ETA: 2:42 - loss: 0.1278 - auc: 0.7805Mon Sep 6 09:15:51 2021[1,0]5Mon Sep 6 09:15:48 2021[1,0]\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3512/4000 [=========================>....]Mon Sep 6 09:16:52 2021[1,0]: - ETA: 1:39 - loss: 0.1274 - auc: 0.7816Mon Sep 6 09:16:53 2021[1,0]6Mon Sep 6 09:16:52 2021[1,0]\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3799/4000 [===========================>..]Mon Sep 6 09:17:50 2021[1,0]: - ETA: 40s - loss: 0.1272 - auc: 0.7826Mon Sep 6 09:17:52 2021[1,0]\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "4000/4000 [==============================]Mon Sep 6 09:18:36 2021[1,0]: - 845s 204ms/step - loss: 0.1270 - auc: 0.7832 - val_loss: 0.1273 - val_auc: 0.7933 6 09:18:22 2021[1,0]\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Best Loss: 0.127242\n" - ] - } - ], - "source": [ - "main()" - ] - }, - { - "cell_type": "markdown", - "id": "5a1c01ad", - "metadata": {}, - "source": [ - "## Use Petastorm\n", - "\n", - "Here we set `--dataloader` to `petastorm` to force Petastorm run." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "fee60d10", - "metadata": {}, - "outputs": [], - "source": [ - "def main_petastorm():\n", - " parser = argparse.ArgumentParser(description='Criteo Spark Keras Training Example',\n", - " formatter_class=argparse.ArgumentDefaultsHelpFormatter)\n", - " parser.add_argument('--data-dir', default='file:///opt/data/criteo/parquet',\n", - " help='location of the transformed Criteo dataset in Parquet format')\n", - " parser.add_argument('--logs-dir', default='/opt/experiments/criteo', help='location of TensorFlow logs')\n", - " parser.add_argument('--dataloader', default=PETASTORM_DATALOADER,\n", - " choices=[PETASTORM_DATALOADER, NVTABULAR_DATALOADER],\n", - " help='dataloader to use')\n", - " parser.add_argument('--num-proc', type=int, default=1, help='number of worker processes for training')\n", - " parser.add_argument('--learning-rate', type=float, default=0.0001, help='initial learning rate')\n", - " parser.add_argument('--batch-size', type=int, default=64 * 1024, help='batch size')\n", - " parser.add_argument('--epochs', type=int, default=3, help='number of epochs to train')\n", - " parser.add_argument('--local-checkpoint-file', default='checkpoint', help='model checkpoint')\n", - " args = parser.parse_args(args=['--num-proc', '16', '--data-dir', 'file:///raid/spark-team/criteo/parquet', \n", - " '--dataloader', 'petastorm', '--learning-rate', '0.001',\n", - " '--batch-size', '65535','--epochs', '1', '--logs-dir', 'tf_logs',\n", - " '--local-checkpoint-file', 'ckpt_file'])\n", - " \n", - "\n", - " dimensions = get_category_dimensions(spark, args.data_dir)\n", - "\n", - " train_df = spark.read.parquet(f'{args.data_dir}/train')\n", - " val_df = spark.read.parquet(f'{args.data_dir}/val')\n", - " test_df = spark.read.parquet(f'{args.data_dir}/test')\n", - " train_rows, val_rows, test_rows = train_df.count(), val_df.count(), test_df.count()\n", - " print('Training: %d' % train_rows)\n", - " print('Validation: %d' % val_rows)\n", - " print('Test: %d' % test_rows)\n", - "\n", - " train(dimensions, train_rows, val_rows, args)\n", - "\n", - " spark.stop()" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "1cb35049", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "21/09/06 11:44:44 WARN package: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n", - " \r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'c0': '7912888',\n", - " 'c1': '33822',\n", - " 'c10': '582468',\n", - " 'c11': '245827',\n", - " 'c12': '10',\n", - " 'c13': '2208',\n", - " 'c14': '10666',\n", - " 'c15': '103',\n", - " 'c16': '3',\n", - " 'c17': '967',\n", - " 'c18': '14',\n", - " 'c19': '8165895',\n", - " 'c2': '17138',\n", - " 'c20': '2675939',\n", - " 'c21': '7156452',\n", - " 'c22': '302515',\n", - " 'c23': '12021',\n", - " 'c24': '96',\n", - " 'c25': '34',\n", - " 'c3': '7338',\n", - " 'c4': '20045',\n", - " 'c5': '3',\n", - " 'c6': '7104',\n", - " 'c7': '1381',\n", - " 'c8': '62',\n", - " 'c9': '5554113'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Training: 4195197692\n", - "Validation: 89137318\n", - "Test: 89137319\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Checking whether extension tensorflow was built with MPI.\n", - "Extension tensorflow was built with MPI.\n", - "mpirun --allow-run-as-root --tag-output -np 16 -H dgx2h0194-a1adff968d508e8d1142986f3e2c42dc:16 -bind-to none -map-by slot -mca pml ob1 -mca btl ^openib --timestamp-output -mca btl_tcp_if_include enp134s0f0 -x NCCL_IB_GID_INDEX=3 -x NCCL_DEBUG=INFO -mca plm_rsh_agent \"/home/ngc-auth-ldap-allxu/miniconda3/bin/python -m horovod.spark.driver.mpirun_rsh gAWVcAEAAAAAAAB9lCiMAmxvlF2UjAkxMjcuMC4wLjGUTU5+hpRhjAdlbnA1M3MwlF2UjAwxMC4xNDguMzAuNTmUTU5+hpRhjAdlbnA1OHMwlF2UjAwxMC4xNDguOTQuNTmUTU5+hpRhjAdlbnA4OHMwlF2UjAwxMC4xNDkuMzAuMzSUTU5+hpRhjAdlbnA5M3MwlF2UjAwxMC4xNDkuOTQuNTeUTU5+hpRhjAplbnAxMzRzMGYwlF2UjAsxMC4xNTAuMzAuMpRNTn6GlGGMCGVucDE4NHMwlF2UjA0xMC4xNDguMTU4LjU5lE1OfoaUYYwIZW5wMTg5czCUXZSMDTEwLjE0OC4yMjIuNTmUTU5+hpRhjAhlbnAyMjVzMJRdlIwNMTAuMTQ5LjE1OC41N5RNTn6GlGGMCGVucDIzMHMwlF2UjA0xMC4xNDkuMjIyLjU3lE1OfoaUYYwHZG9ja2VyMJRdlIwKMTcyLjE3LjAuMZRNTn6GlGF1Lg== gAWVAwMAAAAAAACMI2hvcm92b2QucnVubmVyLmNvbW1vbi51dGlsLnNldHRpbmdzlIwIU2V0dGluZ3OUk5QpgZR9lCiMCG51bV9wcm9jlEsQjAd2ZXJib3NllEsCjAhzc2hfcG9ydJROjBFzc2hfaWRlbnRpdHlfZmlsZZROjA5leHRyYV9tcGlfYXJnc5SMOS1tY2EgYnRsX3RjcF9pZl9pbmNsdWRlIGVucDEzNHMwZjAgLXggTkNDTF9JQl9HSURfSU5ERVg9M5SMCHRjcF9mbGFnlE6MDGJpbmRpbmdfYXJnc5ROjANrZXmUTowNc3RhcnRfdGltZW91dJSMImhvcm92b2QucnVubmVyLmNvbW1vbi51dGlsLnRpbWVvdXSUjAdUaW1lb3V0lJOUKYGUfZQojAhfdGltZW91dJRNWAKMC190aW1lb3V0X2F0lEdB2E2AYtTXh4wIX21lc3NhZ2WUWA4BAABUaW1lZCBvdXQgd2FpdGluZyBmb3Ige2FjdGl2aXR5fS4gUGxlYXNlIGNoZWNrIHRoYXQgeW91IGhhdmUgZW5vdWdoIHJlc291cmNlcyB0byBydW4gYWxsIEhvcm92b2QgcHJvY2Vzc2VzLiBFYWNoIEhvcm92b2QgcHJvY2VzcyBydW5zIGluIGEgU3BhcmsgdGFzay4gWW91IG1heSBuZWVkIHRvIGluY3JlYXNlIHRoZSBzdGFydF90aW1lb3V0IHBhcmFtZXRlciB0byBhIGxhcmdlciB2YWx1ZSBpZiB5b3VyIFNwYXJrIHJlc291cmNlcyBhcmUgYWxsb2NhdGVkIG9uLWRlbWFuZC6UdWKMD291dHB1dF9maWxlbmFtZZROjA1ydW5fZnVuY19tb2RllIiMBG5pY3OUfZSMB2VsYXN0aWOUiYwccHJlZml4X291dHB1dF93aXRoX3RpbWVzdGFtcJSIjAVob3N0c5SMLWRneDJoMDE5NC1hMWFkZmY5NjhkNTA4ZThkMTE0Mjk4NmYzZTJjNDJkYzoxNpR1Yi4=\" /home/ngc-auth-ldap-allxu/miniconda3/bin/python -m horovod.spark.task.mpirun_exec_fn gAWVcAEAAAAAAAB9lCiMAmxvlF2UjAkxMjcuMC4wLjGUTU5+hpRhjAdlbnA1M3MwlF2UjAwxMC4xNDguMzAuNTmUTU5+hpRhjAdlbnA1OHMwlF2UjAwxMC4xNDguOTQuNTmUTU5+hpRhjAdlbnA4OHMwlF2UjAwxMC4xNDkuMzAuMzSUTU5+hpRhjAdlbnA5M3MwlF2UjAwxMC4xNDkuOTQuNTeUTU5+hpRhjAplbnAxMzRzMGYwlF2UjAsxMC4xNTAuMzAuMpRNTn6GlGGMCGVucDE4NHMwlF2UjA0xMC4xNDguMTU4LjU5lE1OfoaUYYwIZW5wMTg5czCUXZSMDTEwLjE0OC4yMjIuNTmUTU5+hpRhjAhlbnAyMjVzMJRdlIwNMTAuMTQ5LjE1OC41N5RNTn6GlGGMCGVucDIzMHMwlF2UjA0xMC4xNDkuMjIyLjU3lE1OfoaUYYwHZG9ja2VyMJRdlIwKMTcyLjE3LjAuMZRNTn6GlGF1Lg== gAWVkwkAAAAAAACMI2hvcm92b2QucnVubmVyLmNvbW1vbi51dGlsLnNldHRpbmdzlIwIU2V0dGluZ3OUk5QpgZR9lCiMCG51bV9wcm9jlEsQjAd2ZXJib3NllEsCjAhzc2hfcG9ydJROjBFzc2hfaWRlbnRpdHlfZmlsZZROjA5leHRyYV9tcGlfYXJnc5RYxgYAAC1tY2EgYnRsX3RjcF9pZl9pbmNsdWRlIGVucDEzNHMwZjAgLXggTkNDTF9JQl9HSURfSU5ERVg9MyAteCBOQ0NMX0RFQlVHPUlORk8gLW1jYSBwbG1fcnNoX2FnZW50ICIvaG9tZS9uZ2MtYXV0aC1sZGFwLWFsbHh1L21pbmljb25kYTMvYmluL3B5dGhvbiAtbSBob3Jvdm9kLnNwYXJrLmRyaXZlci5tcGlydW5fcnNoIGdBV1ZjQUVBQUFBQUFBQjlsQ2lNQW14dmxGMlVqQWt4TWpjdU1DNHdMakdVVFU1K2hwUmhqQWRsYm5BMU0zTXdsRjJVakF3eE1DNHhORGd1TXpBdU5UbVVUVTUraHBSaGpBZGxibkExT0hNd2xGMlVqQXd4TUM0eE5EZ3VPVFF1TlRtVVRVNStocFJoakFkbGJuQTRPSE13bEYyVWpBd3hNQzR4TkRrdU16QXVNelNVVFU1K2hwUmhqQWRsYm5BNU0zTXdsRjJVakF3eE1DNHhORGt1T1RRdU5UZVVUVTUraHBSaGpBcGxibkF4TXpSek1HWXdsRjJVakFzeE1DNHhOVEF1TXpBdU1wUk5UbjZHbEdHTUNHVnVjREU0TkhNd2xGMlVqQTB4TUM0eE5EZ3VNVFU0TGpVNWxFMU9mb2FVWVl3SVpXNXdNVGc1Y3pDVVhaU01EVEV3TGpFME9DNHlNakl1TlRtVVRVNStocFJoakFobGJuQXlNalZ6TUpSZGxJd05NVEF1TVRRNUxqRTFPQzQxTjVSTlRuNkdsR0dNQ0dWdWNESXpNSE13bEYyVWpBMHhNQzR4TkRrdU1qSXlMalUzbEUxT2ZvYVVZWXdIWkc5amEyVnlNSlJkbEl3S01UY3lMakUzTGpBdU1aUk5UbjZHbEdGMUxnPT0gZ0FXVkF3TUFBQUFBQUFDTUkyaHZjbTkyYjJRdWNuVnVibVZ5TG1OdmJXMXZiaTUxZEdsc0xuTmxkSFJwYm1kemxJd0lVMlYwZEdsdVozT1VrNVFwZ1pSOWxDaU1DRzUxYlY5d2NtOWpsRXNRakFkMlpYSmliM05sbEVzQ2pBaHpjMmhmY0c5eWRKUk9qQkZ6YzJoZmFXUmxiblJwZEhsZlptbHNaWlJPakE1bGVIUnlZVjl0Y0dsZllYSm5jNVNNT1MxdFkyRWdZblJzWDNSamNGOXBabDlwYm1Oc2RXUmxJR1Z1Y0RFek5ITXdaakFnTFhnZ1RrTkRURjlKUWw5SFNVUmZTVTVFUlZnOU01U01DSFJqY0Y5bWJHRm5sRTZNREdKcGJtUnBibWRmWVhKbmM1Uk9qQU5yWlhtVVRvd05jM1JoY25SZmRHbHRaVzkxZEpTTUltaHZjbTkyYjJRdWNuVnVibVZ5TG1OdmJXMXZiaTUxZEdsc0xuUnBiV1Z2ZFhTVWpBZFVhVzFsYjNWMGxKT1VLWUdVZlpRb2pBaGZkR2x0Wlc5MWRKUk5XQUtNQzE5MGFXMWxiM1YwWDJGMGxFZEIyRTJBWXRUWGg0d0lYMjFsYzNOaFoyV1VXQTRCQUFCVWFXMWxaQ0J2ZFhRZ2QyRnBkR2x1WnlCbWIzSWdlMkZqZEdsMmFYUjVmUzRnVUd4bFlYTmxJR05vWldOcklIUm9ZWFFnZVc5MUlHaGhkbVVnWlc1dmRXZG9JSEpsYzI5MWNtTmxjeUIwYnlCeWRXNGdZV3hzSUVodmNtOTJiMlFnY0hKdlkyVnpjMlZ6TGlCRllXTm9JRWh2Y205MmIyUWdjSEp2WTJWemN5QnlkVzV6SUdsdUlHRWdVM0JoY21zZ2RHRnpheTRnV1c5MUlHMWhlU0J1WldWa0lIUnZJR2x1WTNKbFlYTmxJSFJvWlNCemRHRnlkRjkwYVcxbGIzVjBJSEJoY21GdFpYUmxjaUIwYnlCaElHeGhjbWRsY2lCMllXeDFaU0JwWmlCNWIzVnlJRk53WVhKcklISmxjMjkxY21ObGN5QmhjbVVnWVd4c2IyTmhkR1ZrSUc5dUxXUmxiV0Z1WkM2VWRXS01EMjkxZEhCMWRGOW1hV3hsYm1GdFpaUk9qQTF5ZFc1ZlpuVnVZMTl0YjJSbGxJaU1CRzVwWTNPVWZaU01CMlZzWVhOMGFXT1VpWXdjY0hKbFptbDRYMjkxZEhCMWRGOTNhWFJvWDNScGJXVnpkR0Z0Y0pTSWpBVm9iM04wYzVTTUxXUm5lREpvTURFNU5DMWhNV0ZrWm1ZNU5qaGtOVEE0WlRoa01URTBNams0Tm1ZelpUSmpOREprWXpveE5wUjFZaTQ9IpSMCHRjcF9mbGFnlE6MDGJpbmRpbmdfYXJnc5ROjANrZXmUTowNc3RhcnRfdGltZW91dJSMImhvcm92b2QucnVubmVyLmNvbW1vbi51dGlsLnRpbWVvdXSUjAdUaW1lb3V0lJOUKYGUfZQojAhfdGltZW91dJRNWAKMC190aW1lb3V0X2F0lEdB2E2AYtTXh4wIX21lc3NhZ2WUWA4BAABUaW1lZCBvdXQgd2FpdGluZyBmb3Ige2FjdGl2aXR5fS4gUGxlYXNlIGNoZWNrIHRoYXQgeW91IGhhdmUgZW5vdWdoIHJlc291cmNlcyB0byBydW4gYWxsIEhvcm92b2QgcHJvY2Vzc2VzLiBFYWNoIEhvcm92b2QgcHJvY2VzcyBydW5zIGluIGEgU3BhcmsgdGFzay4gWW91IG1heSBuZWVkIHRvIGluY3JlYXNlIHRoZSBzdGFydF90aW1lb3V0IHBhcmFtZXRlciB0byBhIGxhcmdlciB2YWx1ZSBpZiB5b3VyIFNwYXJrIHJlc291cmNlcyBhcmUgYWxsb2NhdGVkIG9uLWRlbWFuZC6UdWKMD291dHB1dF9maWxlbmFtZZROjA1ydW5fZnVuY19tb2RllIiMBG5pY3OUfZSMB2VsYXN0aWOUiYwccHJlZml4X291dHB1dF93aXRoX3RpbWVzdGFtcJSIjAVob3N0c5SMLWRneDJoMDE5NC1hMWFkZmY5NjhkNTA4ZThkMTE0Mjk4NmYzZTJjNDJkYzoxNpR1Yi4=\n", - "Mon Sep 6 11:44:58 2021[1,5]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,4]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,1]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,0]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,7]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,2]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,3]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,8]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,10]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,11]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,14]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,9]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,6]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,15]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:44:58 2021[1,12]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,13]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:19 2021[1,14]:2021-09-06 11:45:19.644980: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:19 2021[1,14]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:19 2021[1,5]:2021-09-06 11:45:19.658879: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:19 2021[1,5]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:19 2021[1,2]:2021-09-06 11:45:19.677330: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:19 2021[1,2]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:19 2021[1,15]:2021-09-06 11:45:19.692565: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:19 2021[1,15]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:19 2021[1,13]:2021-09-06 11:45:19.710165: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:19 2021[1,13]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:19 2021[1,11]:2021-09-06 11:45:19.730864: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:19 2021[1,11]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:19 2021[1,10]:2021-09-06 11:45:19.755246: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:19 2021[1,10]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:19 2021[1,7]:2021-09-06 11:45:19.777504: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:19 2021[1,7]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:19 2021[1,6]:2021-09-06 11:45:19.803092: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:19 2021[1,6]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:19 2021[1,3]:2021-09-06 11:45:19.823747: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:19 2021[1,3]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:19 2021[1,8]:2021-09-06 11:45:19.845900: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:19 2021[1,8]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:19 2021[1,4]:2021-09-06 11:45:19.868706: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:19 2021[1,4]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:19 2021[1,0]:2021-09-06 11:45:19.955887: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:19 2021[1,0]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:20 2021[1,9]:2021-09-06 11:45:20.026533: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:20 2021[1,9]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:20 2021[1,1]:2021-09-06 11:45:20.142189: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:20 2021[1,1]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:20 2021[1,12]:2021-09-06 11:45:20.356066: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:20 2021[1,12]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:22 2021[1,5]:2021-09-06 11:45:22.866987: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:59:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:22 2021[1,2]:2021-09-06 11:45:22.934896: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:e5:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:22 2021[1,14]:2021-09-06 11:45:22.935986: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:3b:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:23 2021[1,11]:2021-09-06 11:45:23.037281: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:5e:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:23 2021[1,13]:2021-09-06 11:45:23.046456: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:34:00.0, compute capability: 7.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:23 2021[1,7]:2021-09-06 11:45:23.107462: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:5c:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:23 2021[1,6]:2021-09-06 11:45:23.108097: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:b7:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:23 2021[1,10]:2021-09-06 11:45:23.112989: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:e0:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:23 2021[1,3]:2021-09-06 11:45:23.203766: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:e2:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:23 2021[1,8]:2021-09-06 11:45:23.211519: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:be:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:23 2021[1,15]:2021-09-06 11:45:23.270928: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:b9:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:23 2021[1,4]:2021-09-06 11:45:23.279986: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:bc:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:23 2021[1,1]:2021-09-06 11:45:23.341641: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:39:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:23 2021[1,0]:2021-09-06 11:45:23.346429: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:36:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:23 2021[1,12]:2021-09-06 11:45:23.435210: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:57:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:23 2021[1,9]:2021-09-06 11:45:23.477277: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:e7:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:23 2021[1,5]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:23 2021[1,5]: warnings.warn(\n", - "Mon Sep 6 11:45:23 2021[1,5]:2021-09-06 11:45:23.647929: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:23 2021[1,5]:2021-09-06 11:45:23.647952: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:23 2021[1,5]:2021-09-06 11:45:23.647993: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 11:45:23 2021[1,2]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:23 2021[1,2]: warnings.warn(\n", - "Mon Sep 6 11:45:23 2021[1,2]:2021-09-06 11:45:23.744042: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:23 2021[1,2]:2021-09-06 11:45:23.744065: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:23 2021[1,2]:2021-09-06 11:45:23.744117: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 11:45:23 2021[1,14]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:23 2021[1,14]: warnings.warn(\n", - "Mon Sep 6 11:45:23 2021[1,14]:2021-09-06 11:45:23.759698: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:23 2021[1,14]:2021-09-06 11:45:23.759719: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:23 2021[1,14]:2021-09-06 11:45:23.759764: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 11:45:23 2021[1,13]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:23 2021[1,13]: warnings.warn(\n", - "Mon Sep 6 11:45:23 2021[1,13]:2021-09-06 11:45:23.858335: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:23 2021[1,13]:2021-09-06 11:45:23.858356: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:23 2021[1,11]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:23 2021[1,11]: warnings.warn(\n", - "Mon Sep 6 11:45:23 2021[1,13]:2021-09-06 11:45:23.858399: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 11:45:23 2021[1,11]:2021-09-06 11:45:23.858710: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:23 2021[1,11]:2021-09-06 11:45:23.858730: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:23 2021[1,11]:2021-09-06 11:45:23.858772: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 11:45:23 2021[1,7]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:23 2021[1,7]: warnings.warn(\n", - "Mon Sep 6 11:45:23 2021[1,7]:2021-09-06 11:45:23.940736: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:23 2021[1,7]:2021-09-06 11:45:23.940759: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:23 2021[1,7]:2021-09-06 11:45:23.940801: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 11:45:23 2021[1,10]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:23 2021[1,10]: warnings.warn(\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:23 2021[1,10]:2021-09-06 11:45:23.941176: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:23 2021[1,10]:2021-09-06 11:45:23.941195: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:23 2021[1,10]:2021-09-06 11:45:23.941240: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 11:45:23 2021[1,5]:2021-09-06 11:45:23.950215: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:23 2021[1,5]:2021-09-06 11:45:23.952931: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:23 2021[1,6]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:23 2021[1,6]: warnings.warn(\n", - "Mon Sep 6 11:45:23 2021[1,6]:2021-09-06 11:45:23.984430: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:23 2021[1,6]:2021-09-06 11:45:23.984453: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:23 2021[1,6]:2021-09-06 11:45:23.984507: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 11:45:24 2021[1,8]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:24 2021[1,8]: warnings.warn(\n", - "Mon Sep 6 11:45:24 2021[1,8]:2021-09-06 11:45:24.013682: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:24 2021[1,8]:2021-09-06 11:45:24.013706: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:24 2021[1,8]:2021-09-06 11:45:24.013758: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 11:45:24 2021[1,3]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:24 2021[1,3]: warnings.warn(\n", - "Mon Sep 6 11:45:24 2021[1,3]:2021-09-06 11:45:24.046403: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:24 2021[1,3]:2021-09-06 11:45:24.046428: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:24 2021[1,3]:2021-09-06 11:45:24.046476: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 11:45:24 2021[1,4]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:24 2021[1,4]: warnings.warn(\n", - "Mon Sep 6 11:45:24 2021[1,4]:2021-09-06 11:45:24.067336: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:24 2021[1,4]:2021-09-06 11:45:24.067359: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:24 2021[1,4]:2021-09-06 11:45:24.067412: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 11:45:24 2021[1,15]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:24 2021[1,15]: warnings.warn(\n", - "Mon Sep 6 11:45:24 2021[1,15]:2021-09-06 11:45:24.118355: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:24 2021[1,15]:2021-09-06 11:45:24.118379: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:24 2021[1,15]:2021-09-06 11:45:24.118447: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]:Model: \"model\"\n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:Layer (type) Output Shape Param # Connected to \n", - "Mon Sep 6 11:45:24 2021[1,0]:==================================================================================================\n", - "Mon Sep 6 11:45:24 2021[1,0]:c0 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c1 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c2 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c3 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c4 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c6 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c7 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c8 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c9 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c10 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c11 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c12 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c13 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c14 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c15 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c17 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c18 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c19 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c20 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c21 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c22 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c23 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c24 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,1]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:24 2021[1,1]: warnings.warn(\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]:c25 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,1]:2021-09-06 11:45:24.144886: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c0 (Embedding) (None, 1, 8) 63303112 ['c0[0][0]'] \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,1]:2021-09-06 11:45:24.144909: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:24 2021[1,1]:2021-09-06 11:45:24.144956: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c1 (Embedding) (None, 1, 8) 270584 ['c1[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c2 (Embedding) (None, 1, 8) 137112 ['c2[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c3 (Embedding) (None, 1, 8) 58712 ['c3[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c4 (Embedding) (None, 1, 8) 160368 ['c4[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c6 (Embedding) (None, 1, 8) 56840 ['c6[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c7 (Embedding) (None, 1, 8) 11056 ['c7[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c8 (Embedding) (None, 1, 8) 504 ['c8[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c9 (Embedding) (None, 1, 8) 44432912 ['c9[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c10 (Embedding) (None, 1, 8) 4659752 ['c10[0][0]'] \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]: warnings.warn(\n", - "Mon Sep 6 11:45:24 2021[1,0]:2021-09-06 11:45:24.151145: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c11 (Embedding) (None, 1, 8) 1966624 ['c11[0][0]'] \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]:2021-09-06 11:45:24.151167: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]:2021-09-06 11:45:24.151215: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c12 (Embedding) (None, 1, 8) 88 ['c12[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c13 (Embedding) (None, 1, 8) 17672 ['c13[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c14 (Embedding) (None, 1, 8) 85336 ['c14[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c15 (Embedding) (None, 1, 8) 832 ['c15[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c17 (Embedding) (None, 1, 8) 7744 ['c17[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c18 (Embedding) (None, 1, 8) 120 ['c18[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c19 (Embedding) (None, 1, 8) 65327168 ['c19[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c20 (Embedding) (None, 1, 8) 21407520 ['c20[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c21 (Embedding) (None, 1, 8) 57251624 ['c21[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c22 (Embedding) (None, 1, 8) 2420128 ['c22[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c23 (Embedding) (None, 1, 8) 96176 ['c23[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c24 (Embedding) (None, 1, 8) 776 ['c24[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c25 (Embedding) (None, 1, 8) 280 ['c25[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embeddings_concat (Concatenate (None, 1, 192) 0 ['embedding_c0[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]:) 'embedding_c1[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c2[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c3[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c4[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c6[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c7[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c8[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c9[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c10[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c11[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c12[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c13[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c14[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c15[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c17[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c18[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c19[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c20[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c21[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c22[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c23[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c24[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c25[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c5 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c16 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embeddings_flatten (Flatten) (None, 192) 0 ['embeddings_concat[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:one_hot_c5 (CategoryEncoding) (None, 4) 0 ['c5[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:one_hot_c16 (CategoryEncoding) (None, 4) 0 ['c16[0][0]'] \n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:i0 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:i1 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:i2 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:i3 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:i4 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:i5 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:i6 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:i7 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:i8 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:i9 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:i10 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:i11 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:i12 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:inputs_concat (Concatenate) (None, 213) 0 ['embeddings_flatten[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'one_hot_c5[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'one_hot_c16[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'i0[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'i1[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'i2[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'i3[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'i4[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'i5[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'i6[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'i7[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'i8[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'i9[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'i10[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'i11[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'i12[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:batch_normalization (BatchNorm (None, 213) 852 ['inputs_concat[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:alization) \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:dense (Dense) (None, 64) 13696 ['batch_normalization[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:batch_normalization_1 (BatchNo (None, 64) 256 ['dense[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:rmalization) \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:dense_1 (Dense) (None, 64) 4160 ['batch_normalization_1[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:batch_normalization_2 (BatchNo (None, 64) 256 ['dense_1[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:rmalization) \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:dense_2 (Dense) (None, 64) 4160 ['batch_normalization_2[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:batch_normalization_3 (BatchNo (None, 64) 256 ['dense_2[0][0]'] \n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]:rmalization) \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:dense_3 (Dense) (None, 32) 2080 ['batch_normalization_3[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:output (Dense) (None, 1) 33 ['dense_3[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:==================================================================================================\n", - "Mon Sep 6 11:45:24 2021[1,0]:Total params: 261,698,789\n", - "Mon Sep 6 11:45:24 2021[1,0]:Trainable params: 261,697,979\n", - "Mon Sep 6 11:45:24 2021[1,0]:Non-trainable params: 810\n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,12]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:24 2021[1,12]: warnings.warn(\n", - "Mon Sep 6 11:45:24 2021[1,12]:2021-09-06 11:45:24.193332: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:24 2021[1,12]:2021-09-06 11:45:24.193353: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:24 2021[1,12]:2021-09-06 11:45:24.193402: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 11:45:24 2021[1,14]:2021-09-06 11:45:24.238850: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:24 2021[1,14]:2021-09-06 11:45:24.241542: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:24 2021[1,9]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:24 2021[1,9]: warnings.warn(\n", - "Mon Sep 6 11:45:24 2021[1,9]:2021-09-06 11:45:24.250555: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:24 2021[1,9]:2021-09-06 11:45:24.250576: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:24 2021[1,9]:2021-09-06 11:45:24.250629: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 11:45:24 2021[1,11]:2021-09-06 11:45:24.257864: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:24 2021[1,11]:2021-09-06 11:45:24.260597: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:24 2021[1,2]:2021-09-06 11:45:24.280993: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:24 2021[1,13]:2021-09-06 11:45:24.287541: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:24 2021[1,2]:2021-09-06 11:45:24.287675: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:24 2021[1,13]:2021-09-06 11:45:24.290185: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:24 2021[1,10]:2021-09-06 11:45:24.509055: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:24 2021[1,10]:2021-09-06 11:45:24.511856: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:24 2021[1,7]:2021-09-06 11:45:24.559661: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:24 2021[1,7]:2021-09-06 11:45:24.562511: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:24 2021[1,6]:2021-09-06 11:45:24.633007: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:24 2021[1,6]:2021-09-06 11:45:24.635583: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:24 2021[1,8]:2021-09-06 11:45:24.856408: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:24 2021[1,3]:2021-09-06 11:45:24.859705: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:24 2021[1,8]:2021-09-06 11:45:24.861849: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:24 2021[1,3]:2021-09-06 11:45:24.864132: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:24 2021[1,0]:2021-09-06 11:45:24.865753: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:24 2021[1,0]:2021-09-06 11:45:24.870050: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:24 2021[1,9]:2021-09-06 11:45:24.920789: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:24 2021[1,9]:2021-09-06 11:45:24.923768: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:24 2021[1,12]:2021-09-06 11:45:24.927953: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:24 2021[1,12]:2021-09-06 11:45:24.930836: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:24 2021[1,1]:2021-09-06 11:45:24.944178: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:24 2021[1,1]:2021-09-06 11:45:24.951561: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:25 2021[1,15]:2021-09-06 11:45:25.019351: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:25 2021[1,15]:2021-09-06 11:45:25.023739: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:25 2021[1,4]:2021-09-06 11:45:25.132725: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:25 2021[1,4]:2021-09-06 11:45:25.139297: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:25 2021[1,5]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fcebaccea60> and will run it as-is.\n", - "Mon Sep 6 11:45:25 2021[1,5]:Cause: could not parse the source code of . at 0x7fcebaccea60>: no matching AST found\n", - "Mon Sep 6 11:45:25 2021[1,5]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:26 2021[1,5]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fceb96944c0> and will run it as-is.\n", - "Mon Sep 6 11:45:26 2021[1,5]:Cause: could not parse the source code of . at 0x7fceb96944c0>: no matching AST found\n", - "Mon Sep 6 11:45:26 2021[1,5]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:26 2021[1,14]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f7cc813dee0> and will run it as-is.\n", - "Mon Sep 6 11:45:26 2021[1,14]:Cause: could not parse the source code of . at 0x7f7cc813dee0>: no matching AST found\n", - "Mon Sep 6 11:45:26 2021[1,14]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:26 2021[1,2]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f540c075d30> and will run it as-is.\n", - "Mon Sep 6 11:45:26 2021[1,2]:Cause: could not parse the source code of . at 0x7f540c075d30>: no matching AST found\n", - "Mon Sep 6 11:45:26 2021[1,2]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:26 2021[1,11]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f276c530670> and will run it as-is.\n", - "Mon Sep 6 11:45:26 2021[1,11]:Cause: could not parse the source code of . at 0x7f276c530670>: no matching AST found\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:26 2021[1,11]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:26 2021[1,13]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f3cc12349d0> and will run it as-is.\n", - "Mon Sep 6 11:45:26 2021[1,13]:Cause: could not parse the source code of . at 0x7f3cc12349d0>: no matching AST found\n", - "Mon Sep 6 11:45:26 2021[1,13]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:26 2021[1,14]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f7bf4174040> and will run it as-is.\n", - "Mon Sep 6 11:45:26 2021[1,14]:Cause: could not parse the source code of . at 0x7f7bf4174040>: no matching AST found\n", - "Mon Sep 6 11:45:26 2021[1,14]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:26 2021[1,2]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f53c4044160> and will run it as-is.\n", - "Mon Sep 6 11:45:26 2021[1,2]:Cause: could not parse the source code of . at 0x7f53c4044160>: no matching AST found\n", - "Mon Sep 6 11:45:26 2021[1,2]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:27 2021[1,11]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f27cc054280> and will run it as-is.\n", - "Mon Sep 6 11:45:27 2021[1,11]:Cause: could not parse the source code of . at 0x7f27cc054280>: no matching AST found\n", - "Mon Sep 6 11:45:27 2021[1,11]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:27 2021[1,13]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f3c04103280> and will run it as-is.\n", - "Mon Sep 6 11:45:27 2021[1,13]:Cause: could not parse the source code of . at 0x7f3c04103280>: no matching AST found\n", - "Mon Sep 6 11:45:27 2021[1,13]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:27 2021[1,7]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fdbaa897550> and will run it as-is.\n", - "Mon Sep 6 11:45:27 2021[1,7]:Cause: could not parse the source code of . at 0x7fdbaa897550>: no matching AST found\n", - "Mon Sep 6 11:45:27 2021[1,7]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:27 2021[1,6]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f24600dbaf0> and will run it as-is.\n", - "Mon Sep 6 11:45:27 2021[1,6]:Cause: could not parse the source code of . at 0x7f24600dbaf0>: no matching AST found\n", - "Mon Sep 6 11:45:27 2021[1,6]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:27 2021[1,10]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f175809ac10> and will run it as-is.\n", - "Mon Sep 6 11:45:27 2021[1,10]:Cause: could not parse the source code of . at 0x7f175809ac10>: no matching AST found\n", - "Mon Sep 6 11:45:27 2021[1,10]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:27 2021[1,6]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f2460074280> and will run it as-is.\n", - "Mon Sep 6 11:45:27 2021[1,6]:Cause: could not parse the source code of . at 0x7f2460074280>: no matching AST found\n", - "Mon Sep 6 11:45:27 2021[1,6]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:27 2021[1,7]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fdc342ec280> and will run it as-is.\n", - "Mon Sep 6 11:45:27 2021[1,7]:Cause: could not parse the source code of . at 0x7fdc342ec280>: no matching AST found\n", - "Mon Sep 6 11:45:27 2021[1,7]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:27 2021[1,10]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f17a81bf5e0> and will run it as-is.\n", - "Mon Sep 6 11:45:27 2021[1,10]:Cause: could not parse the source code of . at 0x7f17a81bf5e0>: no matching AST found\n", - "Mon Sep 6 11:45:27 2021[1,10]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,0]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f1ae4675ee0> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,0]:Cause: could not parse the source code of . at 0x7f1ae4675ee0>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,0]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,3]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f32a406d550> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,3]:Cause: could not parse the source code of . at 0x7f32a406d550>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,3]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,1]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fd8c81b6b80> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,1]:Cause: could not parse the source code of . at 0x7fd8c81b6b80>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,1]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,8]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fd84e254dc0> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,8]:Cause: could not parse the source code of . at 0x7fd84e254dc0>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,8]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,15]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fe3a71ee1f0> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,15]:Cause: could not parse the source code of . at 0x7fe3a71ee1f0>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,15]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,9]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fb74018b550> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,9]:Cause: could not parse the source code of . at 0x7fb74018b550>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,9]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,0]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f1bd00ae280> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,0]:Cause: could not parse the source code of . at 0x7f1bd00ae280>: no matching AST found\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:28 2021[1,0]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,12]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fce00127ca0> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,12]:Cause: could not parse the source code of . at 0x7fce00127ca0>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,12]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,8]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fd7b01be430> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,8]:Cause: could not parse the source code of . at 0x7fd7b01be430>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,8]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,4]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f2a840e49d0> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,4]:Cause: could not parse the source code of . at 0x7f2a840e49d0>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,4]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,3]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f3254018280> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,3]:Cause: could not parse the source code of . at 0x7f3254018280>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,3]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,1]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fd91404c280> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,1]:Cause: could not parse the source code of . at 0x7fd91404c280>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,1]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,12]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fcdac026280> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,12]:Cause: could not parse the source code of . at 0x7fcdac026280>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,12]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,15]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fe2d40f7280> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,15]:Cause: could not parse the source code of . at 0x7fe2d40f7280>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,15]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,4]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f2a64056160> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,4]:Cause: could not parse the source code of . at 0x7f2a64056160>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,4]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,9]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fb740149040> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,9]:Cause: could not parse the source code of . at 0x7fb740149040>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,9]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:36 2021[1,14]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:36 2021[1,14]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:36 2021[1,2]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:36 2021[1,2]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:36 2021[1,11]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:36 2021[1,11]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:36 2021[1,5]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:36 2021[1,5]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:36 2021[1,6]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:36 2021[1,6]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:36 2021[1,10]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:36 2021[1,10]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:36 2021[1,7]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:36 2021[1,7]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:36 2021[1,13]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:36 2021[1,13]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:37 2021[1,8]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:37 2021[1,8]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:37 2021[1,4]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:37 2021[1,4]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:37 2021[1,0]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:37 2021[1,0]: column_as_pandas = column.data.chunks[0].to_pandas()\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:37 2021[1,3]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:37 2021[1,3]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:37 2021[1,1]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:37 2021[1,1]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:37 2021[1,12]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:37 2021[1,12]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:37 2021[1,15]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:37 2021[1,15]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:37 2021[1,9]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:37 2021[1,9]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:46 2021[1,14]:2021-09-06 11:45:46.163581: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 633106 of 655350\n", - "Mon Sep 6 11:45:46 2021[1,2]:2021-09-06 11:45:46.173808: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 625961 of 655350\n", - "Mon Sep 6 11:45:46 2021[1,11]:2021-09-06 11:45:46.470637: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 575620 of 655350\n", - "Mon Sep 6 11:45:46 2021[1,5]:2021-09-06 11:45:46.483315: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 602125 of 655350\n", - "Mon Sep 6 11:45:46 2021[1,14]:2021-09-06 11:45:46.515473: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:46 2021[1,6]:2021-09-06 11:45:46.620458: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 612890 of 655350\n", - "Mon Sep 6 11:45:46 2021[1,2]:2021-09-06 11:45:46.655265: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:46 2021[1,10]:2021-09-06 11:45:46.681795: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 601332 of 655350\n", - "Mon Sep 6 11:45:46 2021[1,7]:2021-09-06 11:45:46.708487: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 572490 of 655350\n", - "Mon Sep 6 11:45:46 2021[1,13]:2021-09-06 11:45:46.848852: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 584226 of 655350\n", - "Mon Sep 6 11:45:47 2021[1,8]:2021-09-06 11:45:47.028900: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 631682 of 655350\n", - "Mon Sep 6 11:45:47 2021[1,4]:2021-09-06 11:45:47.123354: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 622824 of 655350\n", - "Mon Sep 6 11:45:47 2021[1,0]:2021-09-06 11:45:47.160067: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 624462 of 655350\n", - "Mon Sep 6 11:45:47 2021[1,3]:2021-09-06 11:45:47.306279: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 559646 of 655350\n", - "Mon Sep 6 11:45:47 2021[1,6]:2021-09-06 11:45:47.317935: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:47 2021[1,1]:2021-09-06 11:45:47.328390: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 602957 of 655350\n", - "Mon Sep 6 11:45:47 2021[1,12]:2021-09-06 11:45:47.401105: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 609749 of 655350\n", - "Mon Sep 6 11:45:47 2021[1,8]:2021-09-06 11:45:47.406648: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:47 2021[1,5]:2021-09-06 11:45:47.410203: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:47 2021[1,15]:2021-09-06 11:45:47.490260: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 606865 of 655350\n", - "Mon Sep 6 11:45:47 2021[1,10]:2021-09-06 11:45:47.576105: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:47 2021[1,9]:2021-09-06 11:45:47.631347: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 610214 of 655350\n", - "Mon Sep 6 11:45:47 2021[1,4]:2021-09-06 11:45:47.638664: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:47 2021[1,0]:2021-09-06 11:45:47.648328: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:47 2021[1,11]:2021-09-06 11:45:47.874307: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:48 2021[1,12]:2021-09-06 11:45:48.136479: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:48 2021[1,7]:2021-09-06 11:45:48.178890: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:48 2021[1,1]:2021-09-06 11:45:48.243785: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:48 2021[1,15]:2021-09-06 11:45:48.283844: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:48 2021[1,13]:2021-09-06 11:45:48.332091: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:48 2021[1,9]:2021-09-06 11:45:48.383569: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:49 2021[1,3]:2021-09-06 11:45:49.210125: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:53 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,0]:NCCL version 2.10.3+cuda11.0\n", - "Mon Sep 6 11:45:53 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Using network IB\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:53 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Trees [0] 1/-1/-1->13->0 [1] 1/-1/-1->13->0 [2] 1/-1/-1->13->0 [3] 1/-1/-1->13->0 [4] 1/-1/-1->13->0 [5] 1/-1/-1->13->0 [6] 1/-1/-1->13->0 [7] 1/-1/-1->13->0 [8] 1/-1/-1->13->0 [9] 1/-1/-1->13->0 [10] 1/-1/-1->13->0 [11] 1/-1/-1->13->0\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Setting affinity for GPU 0 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 00/12 : 0 13 1 14 5 12 7 11 2 9 3 10 4 8 6 15\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 01/12 : 0 13 1 14 5 12 7 11 2 9 3 10 4 8 6 15\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 02/12 : 0 13 1 14 5 12 7 11 2 9 3 10 4 8 6 15\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 03/12 : 0 13 1 14 5 12 7 11 2 9 3 10 4 8 6 15\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 04/12 : 0 13 1 14 5 12 7 11 2 9 3 10 4 8 6 15\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 05/12 : 0 13 1 14 5 12 7 11 2 9 3 10 4 8 6 15\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 06/12 : 0 13 1 14 5 12 7 11 2 9 3 10 4 8 6 15\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 07/12 : 0 13 1 14 5 12 7 11 2 9 3 10 4 8 6 15\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Trees [0] -1/-1/-1->15->6 [1] -1/-1/-1->15->6 [2] -1/-1/-1->15->6 [3] -1/-1/-1->15->6 [4] -1/-1/-1->15->6 [5] -1/-1/-1->15->6 [6] -1/-1/-1->15->6 [7] -1/-1/-1->15->6 [8] -1/-1/-1->15->6 [9] -1/-1/-1->15->6 [10] -1/-1/-1->15->6 [11] -1/-1/-1->15->6\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Setting affinity for GPU 9 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Trees [0] 9/-1/-1->2->11 [1] 9/-1/-1->2->11 [2] 9/-1/-1->2->11 [3] 9/-1/-1->2->11 [4] 9/-1/-1->2->11 [5] 9/-1/-1->2->11 [6] 9/-1/-1->2->11 [7] 9/-1/-1->2->11 [8] 9/-1/-1->2->11 [9] 9/-1/-1->2->11 [10] 9/-1/-1->2->11 [11] 9/-1/-1->2->11\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Setting affinity for GPU 14 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Trees [0] 14/-1/-1->1->13 [1] 14/-1/-1->1->13 [2] 14/-1/-1->1->13 [3] 14/-1/-1->1->13 [4] 14/-1/-1->1->13 [5] 14/-1/-1->1->13 [6] 14/-1/-1->1->13 [7] 14/-1/-1->1->13 [8] 14/-1/-1->1->13 [9] 14/-1/-1->1->13 [10] 14/-1/-1->1->13 [11] 14/-1/-1->1->13\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Setting affinity for GPU 2 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Trees [0] 10/-1/-1->3->9 [1] 10/-1/-1->3->9 [2] 10/-1/-1->3->9 [3] 10/-1/-1->3->9 [4] 10/-1/-1->3->9 [5] 10/-1/-1->3->9 [6] 10/-1/-1->3->9 [7] 10/-1/-1->3->9 [8] 10/-1/-1->3->9 [9] 10/-1/-1->3->9 [10] 10/-1/-1->3->9 [11] 10/-1/-1->3->9\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Setting affinity for GPU 13 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Trees [0] 5/-1/-1->14->1 [1] 5/-1/-1->14->1 [2] 5/-1/-1->14->1 [3] 5/-1/-1->14->1 [4] 5/-1/-1->14->1 [5] 5/-1/-1->14->1 [6] 5/-1/-1->14->1 [7] 5/-1/-1->14->1 [8] 5/-1/-1->14->1 [9] 5/-1/-1->14->1 [10] 5/-1/-1->14->1 [11] 5/-1/-1->14->1\n", - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Setting affinity for GPU 3 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 08/12 : 0 13 1 14 5 12 7 11 2 9 3 10 4 8 6 15\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 09/12 : 0 13 1 14 5 12 7 11 2 9 3 10 4 8 6 15\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 10/12 : 0 13 1 14 5 12 7 11 2 9 3 10 4 8 6 15\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 11/12 : 0 13 1 14 5 12 7 11 2 9 3 10 4 8 6 15\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Trees [0] 13/-1/-1->0->-1 [1] 13/-1/-1->0->-1 [2] 13/-1/-1->0->-1 [3] 13/-1/-1->0->-1 [4] 13/-1/-1->0->-1 [5] 13/-1/-1->0->-1 [6] 13/-1/-1->0->-1 [7] 13/-1/-1->0->-1 [8] 13/-1/-1->0->-1 [9] 13/-1/-1->0->-1 [10] 13/-1/-1->0->-1 [11] 13/-1/-1->0->-1\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Setting affinity for GPU 1 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Trees [0] 15/-1/-1->6->8 [1] 15/-1/-1->6->8 [2] 15/-1/-1->6->8 [3] 15/-1/-1->6->8 [4] 15/-1/-1->6->8 [5] 15/-1/-1->6->8 [6] 15/-1/-1->6->8 [7] 15/-1/-1->6->8 [8] 15/-1/-1->6->8 [9] 15/-1/-1->6->8 [10] 15/-1/-1->6->8 [11] 15/-1/-1->6->8\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Setting affinity for GPU 8 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Trees [0] 8/-1/-1->4->10 [1] 8/-1/-1->4->10 [2] 8/-1/-1->4->10 [3] 8/-1/-1->4->10 [4] 8/-1/-1->4->10 [5] 8/-1/-1->4->10 [6] 8/-1/-1->4->10 [7] 8/-1/-1->4->10 [8] 8/-1/-1->4->10 [9] 8/-1/-1->4->10 [10] 8/-1/-1->4->10 [11] 8/-1/-1->4->10\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Setting affinity for GPU 10 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Trees [0] 6/-1/-1->8->4 [1] 6/-1/-1->8->4 [2] 6/-1/-1->8->4 [3] 6/-1/-1->8->4 [4] 6/-1/-1->8->4 [5] 6/-1/-1->8->4 [6] 6/-1/-1->8->4 [7] 6/-1/-1->8->4 [8] 6/-1/-1->8->4 [9] 6/-1/-1->8->4 [10] 6/-1/-1->8->4 [11] 6/-1/-1->8->4\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Setting affinity for GPU 11 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Trees [0] 12/-1/-1->5->14 [1] 12/-1/-1->5->14 [2] 12/-1/-1->5->14 [3] 12/-1/-1->5->14 [4] 12/-1/-1->5->14 [5] 12/-1/-1->5->14 [6] 12/-1/-1->5->14 [7] 12/-1/-1->5->14 [8] 12/-1/-1->5->14 [9] 12/-1/-1->5->14 [10] 12/-1/-1->5->14 [11] 12/-1/-1->5->14\n", - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Setting affinity for GPU 5 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Trees [0] 11/-1/-1->7->12 [1] 11/-1/-1->7->12 [2] 11/-1/-1->7->12 [3] 11/-1/-1->7->12 [4] 11/-1/-1->7->12 [5] 11/-1/-1->7->12 [6] 11/-1/-1->7->12 [7] 11/-1/-1->7->12 [8] 11/-1/-1->7->12 [9] 11/-1/-1->7->12 [10] 11/-1/-1->7->12 [11] 11/-1/-1->7->12\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Setting affinity for GPU 6 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Trees [0] 4/-1/-1->10->3 [1] 4/-1/-1->10->3 [2] 4/-1/-1->10->3 [3] 4/-1/-1->10->3 [4] 4/-1/-1->10->3 [5] 4/-1/-1->10->3 [6] 4/-1/-1->10->3 [7] 4/-1/-1->10->3 [8] 4/-1/-1->10->3 [9] 4/-1/-1->10->3 [10] 4/-1/-1->10->3 [11] 4/-1/-1->10->3\n", - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Setting affinity for GPU 12 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Trees [0] 3/-1/-1->9->2 [1] 3/-1/-1->9->2 [2] 3/-1/-1->9->2 [3] 3/-1/-1->9->2 [4] 3/-1/-1->9->2 [5] 3/-1/-1->9->2 [6] 3/-1/-1->9->2 [7] 3/-1/-1->9->2 [8] 3/-1/-1->9->2 [9] 3/-1/-1->9->2 [10] 3/-1/-1->9->2 [11] 3/-1/-1->9->2\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Setting affinity for GPU 15 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Trees [0] 2/-1/-1->11->7 [1] 2/-1/-1->11->7 [2] 2/-1/-1->11->7 [3] 2/-1/-1->11->7 [4] 2/-1/-1->11->7 [5] 2/-1/-1->11->7 [6] 2/-1/-1->11->7 [7] 2/-1/-1->11->7 [8] 2/-1/-1->11->7 [9] 2/-1/-1->11->7 [10] 2/-1/-1->11->7 [11] 2/-1/-1->11->7\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Setting affinity for GPU 7 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Trees [0] 7/-1/-1->12->5 [1] 7/-1/-1->12->5 [2] 7/-1/-1->12->5 [3] 7/-1/-1->12->5 [4] 7/-1/-1->12->5 [5] 7/-1/-1->12->5 [6] 7/-1/-1->12->5 [7] 7/-1/-1->12->5 [8] 7/-1/-1->12->5 [9] 7/-1/-1->12->5 [10] 7/-1/-1->12->5 [11] 7/-1/-1->12->5\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Setting affinity for GPU 4 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 00 : 13[34000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 00 : 14[3b000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 00 : 7[5c000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 00 : 15[b9000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 00 : 4[bc000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 00 : 3[e2000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 01 : 13[34000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 00 : 6[b7000] -> 15[b9000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 01 : 14[3b000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 01 : 7[5c000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 01 : 15[b9000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 01 : 4[bc000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 01 : 3[e2000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 01 : 6[b7000] -> 15[b9000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 02 : 13[34000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 02 : 14[3b000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 02 : 7[5c000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 02 : 15[b9000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 02 : 4[bc000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 02 : 3[e2000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 03 : 13[34000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 02 : 6[b7000] -> 15[b9000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 03 : 14[3b000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 03 : 7[5c000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 03 : 15[b9000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 03 : 4[bc000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 03 : 3[e2000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 03 : 6[b7000] -> 15[b9000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 04 : 13[34000] -> 1[39000] via P2P/IPC\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 04 : 14[3b000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 04 : 7[5c000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 04 : 15[b9000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 04 : 4[bc000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 04 : 3[e2000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 04 : 6[b7000] -> 15[b9000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 05 : 13[34000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 05 : 14[3b000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 05 : 7[5c000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 05 : 15[b9000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 05 : 4[bc000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 05 : 3[e2000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 06 : 13[34000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 05 : 6[b7000] -> 15[b9000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 06 : 14[3b000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 06 : 7[5c000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 06 : 15[b9000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 06 : 4[bc000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 06 : 3[e2000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 06 : 6[b7000] -> 15[b9000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 07 : 13[34000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 07 : 14[3b000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 07 : 7[5c000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 07 : 15[b9000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 07 : 4[bc000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 07 : 3[e2000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 08 : 13[34000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 07 : 6[b7000] -> 15[b9000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 08 : 14[3b000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 08 : 7[5c000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 08 : 15[b9000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 08 : 4[bc000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 08 : 3[e2000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 08 : 6[b7000] -> 15[b9000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 09 : 13[34000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 09 : 14[3b000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 09 : 7[5c000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 09 : 15[b9000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 09 : 4[bc000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 09 : 3[e2000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 10 : 13[34000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 09 : 6[b7000] -> 15[b9000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 10 : 14[3b000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 10 : 7[5c000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 10 : 15[b9000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 10 : 4[bc000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 10 : 3[e2000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 10 : 6[b7000] -> 15[b9000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 11 : 13[34000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 11 : 14[3b000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 11 : 7[5c000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 11 : 15[b9000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 11 : 4[bc000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 11 : 3[e2000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 11 : 6[b7000] -> 15[b9000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 00 : 5[59000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 00 : 2[e5000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 01 : 5[59000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 01 : 2[e5000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 02 : 5[59000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 02 : 2[e5000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 03 : 5[59000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 04 : 5[59000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 03 : 2[e5000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 05 : 5[59000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 04 : 2[e5000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 06 : 5[59000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 05 : 2[e5000] -> 9[e7000] via P2P/IPC\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 07 : 5[59000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 06 : 2[e5000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 08 : 5[59000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 07 : 2[e5000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 09 : 5[59000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 08 : 2[e5000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 10 : 5[59000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 09 : 2[e5000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 11 : 5[59000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 10 : 2[e5000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 11 : 2[e5000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 00 : 1[39000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 01 : 1[39000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 02 : 1[39000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 03 : 1[39000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 04 : 1[39000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 05 : 1[39000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 06 : 1[39000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 07 : 1[39000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 08 : 1[39000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 09 : 1[39000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 00 : 11[5e000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 10 : 1[39000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 01 : 11[5e000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 11 : 1[39000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 02 : 11[5e000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 03 : 11[5e000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 04 : 11[5e000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 05 : 11[5e000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 06 : 11[5e000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 00 : 0[36000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 07 : 11[5e000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 01 : 0[36000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 08 : 11[5e000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 02 : 0[36000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 09 : 11[5e000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 03 : 0[36000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 00 : 8[be000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 10 : 11[5e000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 04 : 0[36000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 01 : 8[be000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 11 : 11[5e000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 05 : 0[36000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 02 : 8[be000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 06 : 0[36000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 03 : 8[be000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 07 : 0[36000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 04 : 8[be000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 08 : 0[36000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 05 : 8[be000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 09 : 0[36000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 06 : 8[be000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 10 : 0[36000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 07 : 8[be000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 11 : 0[36000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 08 : 8[be000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 09 : 8[be000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 00 : 10[e0000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 10 : 8[be000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 01 : 10[e0000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 11 : 8[be000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 02 : 10[e0000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 03 : 10[e0000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 04 : 10[e0000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 05 : 10[e0000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 06 : 10[e0000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 07 : 10[e0000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 08 : 10[e0000] -> 4[bc000] via P2P/IPC\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 09 : 10[e0000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 00 : 12[57000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 10 : 10[e0000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 01 : 12[57000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 11 : 10[e0000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 02 : 12[57000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 03 : 12[57000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 00 : 9[e7000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 04 : 12[57000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 01 : 9[e7000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 05 : 12[57000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 02 : 9[e7000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 06 : 12[57000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 03 : 9[e7000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 00 : 15[b9000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 07 : 12[57000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 04 : 9[e7000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 01 : 15[b9000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 08 : 12[57000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 05 : 9[e7000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 02 : 15[b9000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 06 : 9[e7000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 09 : 12[57000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 03 : 15[b9000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 10 : 12[57000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 07 : 9[e7000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 04 : 15[b9000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 11 : 12[57000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 08 : 9[e7000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 05 : 15[b9000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 09 : 9[e7000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 06 : 15[b9000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 10 : 9[e7000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 07 : 15[b9000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 11 : 9[e7000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 08 : 15[b9000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 09 : 15[b9000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 10 : 15[b9000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 11 : 15[b9000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 00 : 13[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 01 : 13[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 02 : 13[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 03 : 13[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 04 : 13[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 05 : 13[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 06 : 13[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 07 : 13[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 08 : 13[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 09 : 13[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 10 : 13[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 11 : 13[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 00 : 4[bc000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 01 : 4[bc000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 02 : 4[bc000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 03 : 4[bc000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 00 : 14[3b000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 04 : 4[bc000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 01 : 14[3b000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 05 : 4[bc000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 02 : 14[3b000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 00 : 5[59000] -> 14[3b000] via P2P/IPC\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:59 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 06 : 4[bc000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 03 : 14[3b000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 01 : 5[59000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 07 : 4[bc000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 04 : 14[3b000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 02 : 5[59000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 08 : 4[bc000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 05 : 14[3b000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 03 : 5[59000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 09 : 4[bc000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 06 : 14[3b000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 00 : 7[5c000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 04 : 5[59000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 10 : 4[bc000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 01 : 7[5c000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 07 : 14[3b000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 05 : 5[59000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 11 : 4[bc000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 08 : 14[3b000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 02 : 7[5c000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 06 : 5[59000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 03 : 7[5c000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 09 : 14[3b000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 07 : 5[59000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 10 : 14[3b000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 04 : 7[5c000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 08 : 5[59000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 11 : 14[3b000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 05 : 7[5c000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 09 : 5[59000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 06 : 7[5c000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 10 : 5[59000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 07 : 7[5c000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 11 : 5[59000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 08 : 7[5c000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 09 : 7[5c000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 00 : 3[e2000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 10 : 7[5c000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 00 : 10[e0000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 01 : 3[e2000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 11 : 7[5c000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 02 : 3[e2000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 01 : 10[e0000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 02 : 10[e0000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 03 : 3[e2000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 03 : 10[e0000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 04 : 3[e2000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 05 : 3[e2000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 04 : 10[e0000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 06 : 3[e2000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 05 : 10[e0000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 00 : 1[39000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 07 : 3[e2000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 06 : 10[e0000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 01 : 1[39000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 08 : 3[e2000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 07 : 10[e0000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 02 : 1[39000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 09 : 3[e2000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 08 : 10[e0000] -> 3[e2000] via P2P/IPC\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 00 : 6[b7000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 03 : 1[39000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 10 : 3[e2000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 09 : 10[e0000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 01 : 6[b7000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 04 : 1[39000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 11 : 3[e2000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 10 : 10[e0000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 02 : 6[b7000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 05 : 1[39000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 11 : 10[e0000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 03 : 6[b7000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 06 : 1[39000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 00 : 12[57000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 04 : 6[b7000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 07 : 1[39000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 05 : 6[b7000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 01 : 12[57000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 08 : 1[39000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 02 : 12[57000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 06 : 6[b7000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 09 : 1[39000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 03 : 12[57000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 07 : 6[b7000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 10 : 1[39000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 08 : 6[b7000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 04 : 12[57000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 11 : 1[39000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 09 : 6[b7000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 05 : 12[57000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 00 : 2[e5000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 00 : 9[e7000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 10 : 6[b7000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 01 : 2[e5000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 06 : 12[57000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 01 : 9[e7000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 07 : 12[57000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 02 : 2[e5000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 11 : 6[b7000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 02 : 9[e7000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 03 : 2[e5000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 08 : 12[57000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 03 : 9[e7000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 09 : 12[57000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 04 : 2[e5000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 10 : 12[57000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 05 : 2[e5000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 04 : 9[e7000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 11 : 12[57000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 06 : 2[e5000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 05 : 9[e7000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 07 : 2[e5000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 06 : 9[e7000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 08 : 2[e5000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 07 : 9[e7000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 09 : 2[e5000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 08 : 9[e7000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 10 : 2[e5000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 09 : 9[e7000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 00 : 8[be000] -> 4[bc000] via P2P/IPC\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 11 : 2[e5000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 10 : 9[e7000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 01 : 8[be000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 11 : 9[e7000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 02 : 8[be000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 03 : 8[be000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 04 : 8[be000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 05 : 8[be000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 06 : 8[be000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 07 : 8[be000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 08 : 8[be000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 00 : 11[5e000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 09 : 8[be000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 01 : 11[5e000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 10 : 8[be000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 02 : 11[5e000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 11 : 8[be000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 03 : 11[5e000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 04 : 11[5e000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 05 : 11[5e000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 06 : 11[5e000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 07 : 11[5e000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 08 : 11[5e000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 09 : 11[5e000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 10 : 11[5e000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 11 : 11[5e000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO comm 0x7fdcb862ad60 rank 7 nranks 16 cudaDev 0 busId 5c000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO comm 0x7f2914629ae0 rank 11 nranks 16 cudaDev 0 busId 5e000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO comm 0x7fe41462a050 rank 15 nranks 16 cudaDev 0 busId b9000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO comm 0x7f3cf4629f50 rank 13 nranks 16 cudaDev 0 busId 34000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO comm 0x7fcea062aa60 rank 12 nranks 16 cudaDev 0 busId 57000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO comm 0x7f2bb862ae60 rank 4 nranks 16 cudaDev 0 busId bc000 - Init COMPLETE\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO comm 0x7fda3462aef0 rank 1 nranks 16 cudaDev 0 busId 39000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO comm 0x7f257462aab0 rank 6 nranks 16 cudaDev 0 busId b7000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO comm 0x7f54e462ac10 rank 2 nranks 16 cudaDev 0 busId e5000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO comm 0x7fd8c862b120 rank 8 nranks 16 cudaDev 0 busId be000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO comm 0x7f33b062a600 rank 3 nranks 16 cudaDev 0 busId e2000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO comm 0x7fcf1462aa00 rank 5 nranks 16 cudaDev 0 busId 59000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO comm 0x7fb7d062a3c0 rank 9 nranks 16 cudaDev 0 busId e7000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO comm 0x7f1c8068aa60 rank 0 nranks 16 cudaDev 0 busId 36000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO comm 0x7f7db062a080 rank 14 nranks 16 cudaDev 0 busId 3b000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO comm 0x7f18ac629ef0 rank 10 nranks 16 cudaDev 0 busId e0000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Launch mode Parallel\n", - "Mon Sep 6 11:45:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:46:11 2021[1,11]:2021-09-06 11:46:11.401104: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,11]:2021-09-06 11:46:11.401164: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,3]:2021-09-06 11:46:11.440290: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,3]:2021-09-06 11:46:11.440344: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,9]:2021-09-06 11:46:11.442416: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,9]:2021-09-06 11:46:11.442453: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,5]:2021-09-06 11:46:11.445379: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,5]:2021-09-06 11:46:11.445409: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,8]:2021-09-06 11:46:11.461500: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,8]:2021-09-06 11:46:11.461545: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,0]:2021-09-06 11:46:11.467532: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,0]:2021-09-06 11:46:11.467566: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,14]:2021-09-06 11:46:11.467699: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,14]:2021-09-06 11:46:11.467725: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,7]:2021-09-06 11:46:11.467927: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,7]:2021-09-06 11:46:11.467954: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,15]:2021-09-06 11:46:11.478742: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,15]:2021-09-06 11:46:11.478775: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,2]:2021-09-06 11:46:11.480342: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,2]:2021-09-06 11:46:11.480375: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,4]:2021-09-06 11:46:11.493077: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,4]:2021-09-06 11:46:11.493108: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,12]:2021-09-06 11:46:11.514149: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,12]:2021-09-06 11:46:11.514199: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,1]:2021-09-06 11:46:11.517690: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,1]:2021-09-06 11:46:11.517726: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,13]:2021-09-06 11:46:11.526581: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,13]:2021-09-06 11:46:11.526613: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,6]:2021-09-06 11:46:11.537704: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,6]:2021-09-06 11:46:11.537739: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,10]:2021-09-06 11:46:11.555526: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,10]:2021-09-06 11:46:11.555565: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:12 2021[1,13]:2021-09-06 11:46:12.486392: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,14]:2021-09-06 11:46:12.486420: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,2]:2021-09-06 11:46:12.486804: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,6]:2021-09-06 11:46:12.486894: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,11]:2021-09-06 11:46:12.487354: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,6]:2021-09-06 11:46:12.489094: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,13]:2021-09-06 11:46:12.489218: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,2]:2021-09-06 11:46:12.489982: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,11]:2021-09-06 11:46:12.490307: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,14]:2021-09-06 11:46:12.490535: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:59 2021[1,0]: 1/4000 [..............................]Mon Sep 6 11:45:59 2021[1,0]: - ETA: 34:42:44 - loss: 1.3348 - auc: 0.5421Mon Sep 6 11:46:12 2021[1,0]:\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\bMon Sep 6 11:46:12 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:46:12 2021[1,5]:2021-09-06 11:46:12.494098: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,9]:2021-09-06 11:46:12.494182: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,12]:2021-09-06 11:46:12.494154: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,15]:2021-09-06 11:46:12.494339: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,8]:2021-09-06 11:46:12.494413: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,3]:2021-09-06 11:46:12.494461: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,1]:2021-09-06 11:46:12.494477: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,4]:2021-09-06 11:46:12.494922: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,10]:2021-09-06 11:46:12.495422: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,12]:2021-09-06 11:46:12.496543: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,9]:2021-09-06 11:46:12.496573: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,0]:2021-09-06 11:46:12.496713: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,5]:2021-09-06 11:46:12.496875: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,7]:2021-09-06 11:46:12.497310: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,8]:2021-09-06 11:46:12.497408: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,1]:2021-09-06 11:46:12.497466: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,15]:2021-09-06 11:46:12.497573: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,10]:2021-09-06 11:46:12.497718: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,4]:2021-09-06 11:46:12.497957: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,3]:2021-09-06 11:46:12.498785: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,0]:2021-09-06 11:46:12.499468: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,7]:2021-09-06 11:46:12.499616: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,10]:2021-09-06 11:46:12.689220: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2615 callback api events and 2562 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,7]:2021-09-06 11:46:12.738674: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2612 callback api events and 2559 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,10]:2021-09-06 11:46:12.739311: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,2]:2021-09-06 11:46:12.772937: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2614 callback api events and 2561 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,13]:2021-09-06 11:46:12.775817: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2615 callback api events and 2562 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,0]:2021-09-06 11:46:12.788372: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2614 callback api events and 2561 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,11]:2021-09-06 11:46:12.793984: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2615 callback api events and 2562 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,14]:2021-09-06 11:46:12.797338: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2615 callback api events and 2562 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,12]:2021-09-06 11:46:12.799541: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2614 callback api events and 2561 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,3]:2021-09-06 11:46:12.804509: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2615 callback api events and 2562 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,9]:2021-09-06 11:46:12.809215: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2614 callback api events and 2561 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,6]:2021-09-06 11:46:12.817765: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2615 callback api events and 2562 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,7]:2021-09-06 11:46:12.825298: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,4]:2021-09-06 11:46:12.830887: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2612 callback api events and 2559 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,15]:2021-09-06 11:46:12.831932: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2612 callback api events and 2559 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,5]:2021-09-06 11:46:12.836523: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2615 callback api events and 2562 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,2]:2021-09-06 11:46:12.838553: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,13]:2021-09-06 11:46:12.839676: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,1]:2021-09-06 11:46:12.840641: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2615 callback api events and 2562 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,8]:2021-09-06 11:46:12.847102: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2615 callback api events and 2562 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,0]:2021-09-06 11:46:12.851656: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,12]:2021-09-06 11:46:12.862278: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,11]:2021-09-06 11:46:12.862736: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,14]:2021-09-06 11:46:12.865552: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,9]:2021-09-06 11:46:12.874796: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,3]:2021-09-06 11:46:12.883307: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:46:12 2021[1,4]:2021-09-06 11:46:12.908955: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,5]:2021-09-06 11:46:12.909348: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,15]:2021-09-06 11:46:12.911988: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,6]:2021-09-06 11:46:12.913338: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,8]:2021-09-06 11:46:12.919742: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,1]:2021-09-06 11:46:12.921926: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,10]:2021-09-06 11:46:12.971783: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:12 2021[1,10]:\n", - "Mon Sep 6 11:46:13 2021[1,10]:2021-09-06 11:46:13.126279: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,7]:2021-09-06 11:46:13.220381: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,7]:\n", - "Mon Sep 6 11:46:13 2021[1,10]:2021-09-06 11:46:13.236756: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,10]:\n", - "Mon Sep 6 11:46:13 2021[1,10]:2021-09-06 11:46:13.244220: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,0]:2021-09-06 11:46:13.247347: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,0]:\n", - "Mon Sep 6 11:46:13 2021[1,10]:2021-09-06 11:46:13.248759: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,10]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:13 2021[1,10]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:13 2021[1,10]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:13 2021[1,10]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:13 2021[1,10]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:13 2021[1,10]:\n", - "Mon Sep 6 11:46:13 2021[1,12]:2021-09-06 11:46:13.282450: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,12]:\n", - "Mon Sep 6 11:46:13 2021[1,9]:2021-09-06 11:46:13.305546: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,9]:\n", - "Mon Sep 6 11:46:13 2021[1,13]:2021-09-06 11:46:13.314358: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,13]:\n", - "Mon Sep 6 11:46:13 2021[1,14]:2021-09-06 11:46:13.347053: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,14]:\n", - "Mon Sep 6 11:46:13 2021[1,11]:2021-09-06 11:46:13.389071: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,11]:\n", - "Mon Sep 6 11:46:13 2021[1,8]:2021-09-06 11:46:13.417449: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,8]:\n", - "Mon Sep 6 11:46:13 2021[1,15]:2021-09-06 11:46:13.440662: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,15]:\n", - "Mon Sep 6 11:46:13 2021[1,7]:2021-09-06 11:46:13.445472: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,5]:2021-09-06 11:46:13.464452: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,5]:\n", - "Mon Sep 6 11:46:13 2021[1,4]:2021-09-06 11:46:13.505475: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,4]:\n", - "Mon Sep 6 11:46:13 2021[1,1]:2021-09-06 11:46:13.538948: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,1]:\n", - "Mon Sep 6 11:46:13 2021[1,6]:2021-09-06 11:46:13.539917: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,6]:\n", - "Mon Sep 6 11:46:13 2021[1,7]:2021-09-06 11:46:13.575977: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,7]:\n", - "Mon Sep 6 11:46:13 2021[1,7]:2021-09-06 11:46:13.583159: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,7]:2021-09-06 11:46:13.588595: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,7]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:13 2021[1,7]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:13 2021[1,7]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:13 2021[1,7]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:13 2021[1,7]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:13 2021[1,7]:\n", - "Mon Sep 6 11:46:13 2021[1,0]:2021-09-06 11:46:13.596438: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,12]:2021-09-06 11:46:13.663240: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,9]:2021-09-06 11:46:13.669575: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:46:13 2021[1,3]:2021-09-06 11:46:13.705164: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,3]:\n", - "Mon Sep 6 11:46:13 2021[1,2]:2021-09-06 11:46:13.727985: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,2]:\n", - "Mon Sep 6 11:46:13 2021[1,14]:2021-09-06 11:46:13.770514: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,0]:2021-09-06 11:46:13.770742: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,0]:\n", - "Mon Sep 6 11:46:13 2021[1,0]:2021-09-06 11:46:13.777869: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,0]:2021-09-06 11:46:13.784817: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,0]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:13 2021[1,0]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:13 2021[1,0]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:13 2021[1,0]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:13 2021[1,0]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:13 2021[1,0]:\n", - "Mon Sep 6 11:46:13 2021[1,13]:2021-09-06 11:46:13.806429: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,12]:2021-09-06 11:46:13.836961: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,12]:\n", - "Mon Sep 6 11:46:13 2021[1,12]:2021-09-06 11:46:13.844073: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,12]:2021-09-06 11:46:13.850941: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,12]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:13 2021[1,12]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:13 2021[1,12]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:13 2021[1,12]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:13 2021[1,12]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:13 2021[1,12]:\n", - "Mon Sep 6 11:46:13 2021[1,9]:2021-09-06 11:46:13.852005: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,9]:\n", - "Mon Sep 6 11:46:13 2021[1,8]:2021-09-06 11:46:13.858089: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,9]:2021-09-06 11:46:13.858866: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,9]:2021-09-06 11:46:13.865756: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,9]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:13 2021[1,9]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:13 2021[1,9]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:13 2021[1,9]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:13 2021[1,9]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:13 2021[1,9]:\n", - "Mon Sep 6 11:46:13 2021[1,1]:2021-09-06 11:46:13.919252: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,5]:2021-09-06 11:46:13.921579: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,15]:2021-09-06 11:46:13.935380: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,11]:2021-09-06 11:46:13.953863: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,14]:2021-09-06 11:46:13.969342: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,14]:\n", - "Mon Sep 6 11:46:13 2021[1,14]:2021-09-06 11:46:13.976218: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,14]:2021-09-06 11:46:13.983131: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,14]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:13 2021[1,14]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:13 2021[1,14]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:13 2021[1,14]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:13 2021[1,14]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:13 2021[1,14]:\n", - "Mon Sep 6 11:46:14 2021[1,13]:2021-09-06 11:46:14.025457: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,13]:\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:46:14 2021[1,4]:2021-09-06 11:46:14.031628: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,13]:2021-09-06 11:46:14.035052: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,13]:2021-09-06 11:46:14.044101: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,13]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:14 2021[1,13]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:14 2021[1,13]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:14 2021[1,13]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,13]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,13]:\n", - "Mon Sep 6 11:46:14 2021[1,6]:2021-09-06 11:46:14.051427: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,8]:2021-09-06 11:46:14.066023: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,8]:\n", - "Mon Sep 6 11:46:14 2021[1,8]:2021-09-06 11:46:14.073202: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,8]:2021-09-06 11:46:14.080723: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,8]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:14 2021[1,8]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:14 2021[1,8]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:14 2021[1,8]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,8]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,8]:\n", - "Mon Sep 6 11:46:14 2021[1,1]:2021-09-06 11:46:14.110787: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,1]:\n", - "Mon Sep 6 11:46:14 2021[1,1]:2021-09-06 11:46:14.118317: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,1]:2021-09-06 11:46:14.125946: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,1]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:14 2021[1,1]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:14 2021[1,1]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:14 2021[1,1]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,1]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,1]:\n", - "Mon Sep 6 11:46:14 2021[1,5]:2021-09-06 11:46:14.131987: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,5]:\n", - "Mon Sep 6 11:46:14 2021[1,5]:2021-09-06 11:46:14.138893: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,3]:2021-09-06 11:46:14.143929: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,5]:2021-09-06 11:46:14.146526: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,5]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:14 2021[1,5]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:14 2021[1,5]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:14 2021[1,5]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,5]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,5]:\n", - "Mon Sep 6 11:46:14 2021[1,15]:2021-09-06 11:46:14.170109: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,15]:\n", - "Mon Sep 6 11:46:14 2021[1,11]:2021-09-06 11:46:14.176378: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,11]:\n", - "Mon Sep 6 11:46:14 2021[1,15]:2021-09-06 11:46:14.177822: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,11]:2021-09-06 11:46:14.184029: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,15]:2021-09-06 11:46:14.186611: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,15]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:14 2021[1,15]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:14 2021[1,15]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:14 2021[1,15]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,15]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:46:14 2021[1,15]:\n", - "Mon Sep 6 11:46:14 2021[1,11]:2021-09-06 11:46:14.192490: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,11]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:14 2021[1,11]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:14 2021[1,11]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:14 2021[1,11]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,11]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,11]:\n", - "Mon Sep 6 11:46:14 2021[1,6]:2021-09-06 11:46:14.235507: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,6]:\n", - "Mon Sep 6 11:46:14 2021[1,6]:2021-09-06 11:46:14.242545: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,2]:2021-09-06 11:46:14.242914: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,6]:2021-09-06 11:46:14.249229: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,6]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:14 2021[1,6]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:14 2021[1,6]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:14 2021[1,6]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,6]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,6]:\n", - "Mon Sep 6 11:46:14 2021[1,4]:2021-09-06 11:46:14.293566: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,4]:\n", - "Mon Sep 6 11:46:14 2021[1,4]:2021-09-06 11:46:14.301188: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,4]:2021-09-06 11:46:14.310530: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,4]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:14 2021[1,4]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:14 2021[1,4]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:14 2021[1,4]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,4]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,4]:\n", - "Mon Sep 6 11:46:14 2021[1,3]:2021-09-06 11:46:14.330532: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,3]:\n", - "Mon Sep 6 11:46:14 2021[1,3]:2021-09-06 11:46:14.339903: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,3]:2021-09-06 11:46:14.347180: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,3]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:14 2021[1,3]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:14 2021[1,3]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:14 2021[1,3]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,3]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,3]:\n", - "Mon Sep 6 11:46:14 2021[1,2]:2021-09-06 11:46:14.471034: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,2]:\n", - "Mon Sep 6 11:46:14 2021[1,2]:2021-09-06 11:46:14.477906: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,2]:2021-09-06 11:46:14.491195: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,2]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:14 2021[1,2]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:14 2021[1,2]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:14 2021[1,2]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,2]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,2]:\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:46:16 2021[1,0]: 5/4000 [..............................]Mon Sep 6 11:46:16 2021[1,0]: - ETA: 4:44:11 - loss: 1.1971 - auc: 0.5447Mon Sep 6 11:46:21 2021[1,0]:>:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:46:21 2021[1,5]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.5142s vs `on_train_batch_end` time: 2.2162s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,15]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.4938s vs `on_train_batch_end` time: 2.2278s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,9]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.5230s vs `on_train_batch_end` time: 2.1673s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,12]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.5441s vs `on_train_batch_end` time: 2.1769s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,8]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.5099s vs `on_train_batch_end` time: 2.2067s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,6]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.4587s vs `on_train_batch_end` time: 2.2470s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,10]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.6162s vs `on_train_batch_end` time: 2.0815s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,4]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.4682s vs `on_train_batch_end` time: 2.2509s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,14]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.5344s vs `on_train_batch_end` time: 2.1920s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,13]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.5144s vs `on_train_batch_end` time: 2.2132s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,1]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.4933s vs `on_train_batch_end` time: 2.2237s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,3]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.4516s vs `on_train_batch_end` time: 2.2473s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,7]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.6179s vs `on_train_batch_end` time: 2.1218s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,11]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.5059s vs `on_train_batch_end` time: 2.2149s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,2]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.4030s vs `on_train_batch_end` time: 2.2793s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,0]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.5182s vs `on_train_batch_end` time: 2.1593s). Check your callbacks.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:46:45 2021[1,0]: 11/4000 [..............................]Mon Sep 6 11:46:45 2021[1,0]: - ETA: 5:01:18 - loss: 0.9724 - auc: 0.5378Mon Sep 6 11:46:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:47:41 2021[1,0]: 21/4000 [..............................]Mon Sep 6 11:47:41 2021[1,0]: - ETA: 5:37:25 - loss: 0.7117 - auc: 0.5280Mon Sep 6 11:47:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:48:44 2021[1,0]: 31/4000 [..............................]Mon Sep 6 11:48:44 2021[1,0]: - ETA: 6:02:07 - loss: 0.5560 - auc: 0.5265Mon Sep 6 11:48:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:49:44 2021[1,0]: 41/4000 [..............................]Mon Sep 6 11:49:44 2021[1,0]: - ETA: 6:10:48 - loss: 0.4611 - auc: 0.5275Mon Sep 6 11:49:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:50:42 2021[1,0]: 50/4000 [..............................]Mon Sep 6 11:50:42 2021[1,0]: - ETA: 6:20:05 - loss: 0.4050 - auc: 0.5309Mon Sep 6 11:50:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:51:41 2021[1,0]: 59/4000 [..............................]Mon Sep 6 11:51:41 2021[1,0]: - ETA: 6:26:39 - loss: 0.3650 - auc: 0.5373Mon Sep 6 11:51:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:52:44 2021[1,0]: 68/4000 [..............................]Mon Sep 6 11:52:44 2021[1,0]: - ETA: 6:35:30 - loss: 0.3352 - auc: 0.5462Mon Sep 6 11:52:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:53:40 2021[1,0]: 76/4000 [..............................]Mon Sep 6 11:53:40 2021[1,0]: - ETA: 6:41:37 - loss: 0.3143 - auc: 0.5561Mon Sep 6 11:53:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:54:38 2021[1,0]: 84/4000 [..............................]Mon Sep 6 11:54:38 2021[1,0]: - ETA: 6:48:12 - loss: 0.2972 - auc: 0.5662Mon Sep 6 11:54:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:55:44 2021[1,0]: 93/4000 [..............................]Mon Sep 6 11:55:44 2021[1,0]: - ETA: 6:53:56 - loss: 0.2815 - auc: 0.5771Mon Sep 6 11:55:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:56:37 2021[1,0]: 100/4000 [..............................]Mon Sep 6 11:56:37 2021[1,0]: - ETA: 6:58:55 - loss: 0.2710 - auc: 0.5854Mon Sep 6 11:56:45 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:57:39 2021[1,0]: 108/4000 [..............................]Mon Sep 6 11:57:39 2021[1,0]: - ETA: 7:04:18 - loss: 0.2609 - auc: 0.5939Mon Sep 6 11:57:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:58:42 2021[1,0]: 116/4000 [..............................]Mon Sep 6 11:58:42 2021[1,0]: - ETA: 7:09:03 - loss: 0.2521 - auc: 0.6023Mon Sep 6 11:58:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:59:37 2021[1,0]: 123/4000 [..............................]Mon Sep 6 11:59:37 2021[1,0]: - ETA: 7:13:04 - loss: 0.2452 - auc: 0.6090Mon Sep 6 11:59:45 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:00:42 2021[1,0]: 131/4000 [..............................]Mon Sep 6 12:00:42 2021[1,0]: - ETA: 7:17:40 - loss: 0.2382 - auc: 0.6162Mon Sep 6 12:00:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:01:39 2021[1,0]: 138/4000 [>.............................]Mon Sep 6 12:01:39 2021[1,0]: - ETA: 7:21:24 - loss: 0.2328 - auc: 0.6220Mon Sep 6 12:01:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:02:37 2021[1,0]: 145/4000 [>.............................]Mon Sep 6 12:02:37 2021[1,0]: - ETA: 7:25:19 - loss: 0.2279 - auc: 0.6274Mon Sep 6 12:02:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:03:37 2021[1,0]: 152/4000 [>.............................]Mon Sep 6 12:03:37 2021[1,0]: - ETA: 7:29:15 - loss: 0.2233 - auc: 0.6327Mon Sep 6 12:03:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:04:36 2021[1,0]: 159/4000 [>.............................]Mon Sep 6 12:04:36 2021[1,0]: - ETA: 7:32:18 - loss: 0.2192 - auc: 0.6377Mon Sep 6 12:04:44 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:05:43 2021[1,0]: 167/4000 [>.............................]Mon Sep 6 12:05:43 2021[1,0]: - ETA: 7:35:26 - loss: 0.2149 - auc: 0.6427Mon Sep 6 12:05:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:06:42 2021[1,0]: 174/4000 [>.............................]Mon Sep 6 12:06:42 2021[1,0]: - ETA: 7:38:11 - loss: 0.2115 - auc: 0.6469Mon Sep 6 12:06:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:07:42 2021[1,0]: 181/4000 [>.............................]Mon Sep 6 12:07:42 2021[1,0]: - ETA: 7:40:34 - loss: 0.2084 - auc: 0.6507Mon Sep 6 12:07:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:08:41 2021[1,0]: 188/4000 [>.............................]Mon Sep 6 12:08:41 2021[1,0]: - ETA: 7:42:41 - loss: 0.2055 - auc: 0.6545Mon Sep 6 12:08:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:09:40 2021[1,0]: 195/4000 [>.............................] - ETA: 7:44:22 - loss: 0.2027 - auc: 0.6581Mon Sep 6 12:09:49 2021[1,0]:Mon Sep 6 12:09:32 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:10:40 2021[1,0]: 202/4000 [>.............................]Mon Sep 6 12:10:40 2021[1,0]: - ETA: 7:46:12 - loss: 0.2002 - auc: 0.6614Mon Sep 6 12:10:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:11:40 2021[1,0]: 209/4000 [>.............................]Mon Sep 6 12:11:40 2021[1,0]: - ETA: 7:47:50 - loss: 0.1978 - auc: 0.6646Mon Sep 6 12:11:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:12:39 2021[1,0]: 216/4000 [>.............................]Mon Sep 6 12:12:39 2021[1,0]: - ETA: 7:49:20 - loss: 0.1956 - auc: 0.6675Mon Sep 6 12:12:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:13:40 2021[1,0]: 223/4000 [>.............................]Mon Sep 6 12:13:40 2021[1,0]: - ETA: 7:50:49 - loss: 0.1935 - auc: 0.6702Mon Sep 6 12:13:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:14:38 2021[1,0]: 230/4000 [>.............................]Mon Sep 6 12:14:38 2021[1,0]: - ETA: 7:51:37 - loss: 0.1916 - auc: 0.6730Mon Sep 6 12:14:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:15:38 2021[1,0]: 237/4000 [>.............................]Mon Sep 6 12:15:38 2021[1,0]: - ETA: 7:52:41 - loss: 0.1897 - auc: 0.6754Mon Sep 6 12:15:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:16:38 2021[1,0]: 244/4000 [>.............................]Mon Sep 6 12:16:38 2021[1,0]: - ETA: 7:53:46 - loss: 0.1880 - auc: 0.6779Mon Sep 6 12:16:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:17:38 2021[1,0]: 251/4000 [>.............................]Mon Sep 6 12:17:38 2021[1,0]: - ETA: 7:54:34 - loss: 0.1864 - auc: 0.6802Mon Sep 6 12:17:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:18:37 2021[1,0]: 258/4000 [>.............................]Mon Sep 6 12:18:37 2021[1,0]: - ETA: 7:55:03 - loss: 0.1848 - auc: 0.6824Mon Sep 6 12:18:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:19:37 2021[1,0]: 265/4000 [>.............................]Mon Sep 6 12:19:37 2021[1,0]: - ETA: 7:55:43 - loss: 0.1832 - auc: 0.6846Mon Sep 6 12:19:45 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 272/4000 [=>............................]Mon Sep 6 12:20:37 2021[1,0]: - ETA: 7:56:22 - loss: 0.1818 - auc: 0.6867Mon Sep 6 12:20:46 2021[1,0]:Mon Sep 6 12:20:20 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 279/4000 [=>............................]Mon Sep 6 12:21:38 2021[1,0]: - ETA: 7:57:08 - loss: 0.1804 - auc: 0.6887Mon Sep 6 12:21:47 2021[1,0]:Mon Sep 6 12:21:29 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:22:38 2021[1,0]: 286/4000 [=>............................]Mon Sep 6 12:22:38 2021[1,0]: - ETA: 7:57:26 - loss: 0.1791 - auc: 0.6906Mon Sep 6 12:22:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:23:38 2021[1,0]: 293/4000 [=>............................]Mon Sep 6 12:23:38 2021[1,0]: - ETA: 7:57:59 - loss: 0.1779 - auc: 0.6924Mon Sep 6 12:23:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:24:40 2021[1,0]: 300/4000 [=>............................]Mon Sep 6 12:24:40 2021[1,0]: - ETA: 7:58:35 - loss: 0.1767 - auc: 0.6941Mon Sep 6 12:24:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:25:40 2021[1,0]: 307/4000 [=>............................]Mon Sep 6 12:25:40 2021[1,0]: - ETA: 7:58:54 - loss: 0.1756 - auc: 0.6957Mon Sep 6 12:25:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:26:40 2021[1,0]: 314/4000 [=>............................]Mon Sep 6 12:26:40 2021[1,0]: - ETA: 7:59:03 - loss: 0.1745 - auc: 0.6973Mon Sep 6 12:26:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:27:40 2021[1,0]: 321/4000 [=>............................]Mon Sep 6 12:27:40 2021[1,0]: - ETA: 7:59:04 - loss: 0.1735 - auc: 0.6988Mon Sep 6 12:27:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:28:40 2021[1,0]: 328/4000 [=>............................]Mon Sep 6 12:28:40 2021[1,0]: - ETA: 7:59:18 - loss: 0.1726 - auc: 0.7003Mon Sep 6 12:28:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:29:42 2021[1,0]: 335/4000 [=>............................]Mon Sep 6 12:29:42 2021[1,0]: - ETA: 7:59:35 - loss: 0.1716 - auc: 0.7017Mon Sep 6 12:29:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:30:42 2021[1,0]: 342/4000 [=>............................]Mon Sep 6 12:30:42 2021[1,0]: - ETA: 7:59:39 - loss: 0.1707 - auc: 0.7031Mon Sep 6 12:30:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:31:42 2021[1,0]: 349/4000 [=>............................]Mon Sep 6 12:31:42 2021[1,0]: - ETA: 7:59:34 - loss: 0.1698 - auc: 0.7044Mon Sep 6 12:31:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:32:43 2021[1,0]: 356/4000 [=>............................]Mon Sep 6 12:32:43 2021[1,0]: - ETA: 7:59:36 - loss: 0.1689 - auc: 0.7058Mon Sep 6 12:32:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:33:36 2021[1,0]: 362/4000 [=>............................]Mon Sep 6 12:33:36 2021[1,0]: - ETA: 7:59:43 - loss: 0.1683 - auc: 0.7069Mon Sep 6 12:33:44 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:34:37 2021[1,0]: 369/4000 [=>............................]Mon Sep 6 12:34:37 2021[1,0]: - ETA: 7:59:48 - loss: 0.1674 - auc: 0.7082Mon Sep 6 12:34:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:35:38 2021[1,0]: 376/4000 [=>............................]Mon Sep 6 12:35:38 2021[1,0]: - ETA: 7:59:40 - loss: 0.1667 - auc: 0.7093Mon Sep 6 12:35:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:36:37 2021[1,0]: 383/4000 [=>............................]Mon Sep 6 12:36:37 2021[1,0]: - ETA: 7:59:26 - loss: 0.1660 - auc: 0.7106Mon Sep 6 12:36:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:37:39 2021[1,0]: 390/4000 [=>............................]Mon Sep 6 12:37:39 2021[1,0]: - ETA: 7:59:27 - loss: 0.1653 - auc: 0.7117Mon Sep 6 12:37:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:38:40 2021[1,0]: 397/4000 [=>............................]Mon Sep 6 12:38:40 2021[1,0]: - ETA: 7:59:20 - loss: 0.1646 - auc: 0.7127Mon Sep 6 12:38:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:39:42 2021[1,0]: 404/4000 [==>...........................]Mon Sep 6 12:39:42 2021[1,0]: - ETA: 7:59:13 - loss: 0.1640 - auc: 0.7137Mon Sep 6 12:39:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:40:42 2021[1,0]: 411/4000 [==>...........................]Mon Sep 6 12:40:42 2021[1,0]: - ETA: 7:58:54 - loss: 0.1633 - auc: 0.7148Mon Sep 6 12:40:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:41:44 2021[1,0]: 418/4000 [==>...........................]Mon Sep 6 12:41:44 2021[1,0]: - ETA: 7:58:49 - loss: 0.1627 - auc: 0.7157Mon Sep 6 12:41:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:42:36 2021[1,0]: 424/4000 [==>...........................]Mon Sep 6 12:42:36 2021[1,0]: - ETA: 7:58:35 - loss: 0.1622 - auc: 0.7165Mon Sep 6 12:42:45 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:43:36 2021[1,0]: 431/4000 [==>...........................]Mon Sep 6 12:43:36 2021[1,0]: - ETA: 7:58:09 - loss: 0.1617 - auc: 0.7174Mon Sep 6 12:43:45 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:44:39 2021[1,0]: 438/4000 [==>...........................]Mon Sep 6 12:44:39 2021[1,0]: - ETA: 7:58:06 - loss: 0.1611 - auc: 0.7183Mon Sep 6 12:44:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:45:40 2021[1,0]: 445/4000 [==>...........................]Mon Sep 6 12:45:40 2021[1,0]: - ETA: 7:57:50 - loss: 0.1606 - auc: 0.7192Mon Sep 6 12:45:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:46:42 2021[1,0]: 452/4000 [==>...........................]Mon Sep 6 12:46:42 2021[1,0]: - ETA: 7:57:34 - loss: 0.1601 - auc: 0.7201Mon Sep 6 12:46:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:47:44 2021[1,0]: 459/4000 [==>...........................]Mon Sep 6 12:47:44 2021[1,0]: - ETA: 7:57:19 - loss: 0.1596 - auc: 0.7209Mon Sep 6 12:47:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:48:36 2021[1,0]: 465/4000 [==>...........................]Mon Sep 6 12:48:36 2021[1,0]: - ETA: 7:56:58 - loss: 0.1591 - auc: 0.7215Mon Sep 6 12:48:44 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:49:37 2021[1,0]: 472/4000 [==>...........................]Mon Sep 6 12:49:37 2021[1,0]: - ETA: 7:56:33 - loss: 0.1587 - auc: 0.7223Mon Sep 6 12:49:45 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:50:36 2021[1,0]: 479/4000 [==>...........................]Mon Sep 6 12:50:36 2021[1,0]: - ETA: 7:55:58 - loss: 0.1582 - auc: 0.7231Mon Sep 6 12:50:45 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:51:37 2021[1,0]: 486/4000 [==>...........................]Mon Sep 6 12:51:37 2021[1,0]: - ETA: 7:55:26 - loss: 0.1577 - auc: 0.7239Mon Sep 6 12:51:45 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:52:38 2021[1,0]: 493/4000 [==>...........................]Mon Sep 6 12:52:38 2021[1,0]: - ETA: 7:55:02 - loss: 0.1573 - auc: 0.7246Mon Sep 6 12:52:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:53:40 2021[1,0]: 500/4000 [==>...........................]Mon Sep 6 12:53:40 2021[1,0]: - ETA: 7:54:37 - loss: 0.1568 - auc: 0.7253Mon Sep 6 12:53:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:54:40 2021[1,0]: 507/4000 [==>...........................]Mon Sep 6 12:54:40 2021[1,0]: - ETA: 7:54:04 - loss: 0.1564 - auc: 0.7261Mon Sep 6 12:54:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:55:40 2021[1,0]: 514/4000 [==>...........................]Mon Sep 6 12:55:40 2021[1,0]: - ETA: 7:53:27 - loss: 0.1560 - auc: 0.7268Mon Sep 6 12:55:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:56:40 2021[1,0]: 521/4000 [==>...........................]Mon Sep 6 12:56:40 2021[1,0]: - ETA: 7:52:50 - loss: 0.1556 - auc: 0.7274Mon Sep 6 12:56:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:57:39 2021[1,0]: 528/4000 [==>...........................]Mon Sep 6 12:57:39 2021[1,0]: - ETA: 7:52:08 - loss: 0.1553 - auc: 0.7281Mon Sep 6 12:57:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:58:40 2021[1,0]: 535/4000 [===>..........................]Mon Sep 6 12:58:40 2021[1,0]: - ETA: 7:51:35 - loss: 0.1549 - auc: 0.7287Mon Sep 6 12:58:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:59:41 2021[1,0]: 542/4000 [===>..........................]Mon Sep 6 12:59:41 2021[1,0]: - ETA: 7:51:05 - loss: 0.1545 - auc: 0.7293Mon Sep 6 12:59:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:00:42 2021[1,0]: 549/4000 [===>..........................]Mon Sep 6 13:00:42 2021[1,0]: - ETA: 7:50:26 - loss: 0.1542 - auc: 0.7299Mon Sep 6 13:00:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 556/4000 [===>..........................]Mon Sep 6 13:01:41 2021[1,0]: - ETA: 7:49:40 - loss: 0.1538 - auc: 0.7305Mon Sep 6 13:01:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:02:41 2021[1,0]: 563/4000 [===>..........................]Mon Sep 6 13:02:41 2021[1,0]: - ETA: 7:49:01 - loss: 0.1535 - auc: 0.7311Mon Sep 6 13:02:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:03:40 2021[1,0]: 570/4000 [===>..........................]Mon Sep 6 13:03:40 2021[1,0]: - ETA: 7:48:17 - loss: 0.1531 - auc: 0.7316Mon Sep 6 13:03:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:04:38 2021[1,0]: 577/4000 [===>..........................]Mon Sep 6 13:04:38 2021[1,0]: - ETA: 7:47:19 - loss: 0.1528 - auc: 0.7322Mon Sep 6 13:04:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:05:42 2021[1,0]: 585/4000 [===>..........................]Mon Sep 6 13:05:42 2021[1,0]: - ETA: 7:46:07 - loss: 0.1524 - auc: 0.7328Mon Sep 6 13:05:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:06:43 2021[1,0]: 592/4000 [===>..........................]Mon Sep 6 13:06:43 2021[1,0]: - ETA: 7:45:28 - loss: 0.1521 - auc: 0.7333Mon Sep 6 13:06:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:07:43 2021[1,0]: 599/4000 [===>..........................]Mon Sep 6 13:07:43 2021[1,0]: - ETA: 7:44:49 - loss: 0.1518 - auc: 0.7338Mon Sep 6 13:07:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:08:44 2021[1,0]: 606/4000 [===>..........................]Mon Sep 6 13:08:44 2021[1,0]: - ETA: 7:44:09 - loss: 0.1515 - auc: 0.7343Mon Sep 6 13:08:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:09:43 2021[1,0]: 613/4000 [===>..........................]Mon Sep 6 13:09:43 2021[1,0]: - ETA: 7:43:22 - loss: 0.1513 - auc: 0.7348Mon Sep 6 13:09:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:10:41 2021[1,0]: 620/4000 [===>..........................]Mon Sep 6 13:10:41 2021[1,0]: - ETA: 7:42:25 - loss: 0.1510 - auc: 0.7353Mon Sep 6 13:10:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:11:38 2021[1,0]: 627/4000 [===>..........................]Mon Sep 6 13:11:38 2021[1,0]: - ETA: 7:41:25 - loss: 0.1507 - auc: 0.7358Mon Sep 6 13:11:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:12:44 2021[1,0]: 635/4000 [===>..........................]Mon Sep 6 13:12:44 2021[1,0]: - ETA: 7:40:24 - loss: 0.1504 - auc: 0.7363Mon Sep 6 13:12:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:13:44 2021[1,0]: 642/4000 [===>..........................]Mon Sep 6 13:13:44 2021[1,0]: - ETA: 7:39:39 - loss: 0.1502 - auc: 0.7367Mon Sep 6 13:13:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:14:45 2021[1,0]: 649/4000 [===>..........................]Mon Sep 6 13:14:45 2021[1,0]: - ETA: 7:38:58 - loss: 0.1499 - auc: 0.7372Mon Sep 6 13:14:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 656/4000 [===>..........................]Mon Sep 6 13:15:45 2021[1,0]: - ETA: 7:38:14 - loss: 0.1497 - auc: 0.7377Mon Sep 6 13:15:54 2021[1,0]:Mon Sep 6 13:15:28 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:16:44 2021[1,0]: 663/4000 [===>..........................]Mon Sep 6 13:16:44 2021[1,0]: - ETA: 7:37:26 - loss: 0.1494 - auc: 0.7381Mon Sep 6 13:16:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:17:36 2021[1,0]: 669/4000 [====>.........................]Mon Sep 6 13:17:36 2021[1,0]: - ETA: 7:36:48 - loss: 0.1492 - auc: 0.7385Mon Sep 6 13:17:45 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 676/4000 [====>.........................]Mon Sep 6 13:18:38 2021[1,0]: - ETA: 7:36:12 - loss: 0.1490 - auc: 0.7389Mon Sep 6 13:18:47 2021[1,0]:Mon Sep 6 13:18:38 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 684/4000 [====>.........................]Mon Sep 6 13:19:45 2021[1,0]: - ETA: 7:35:13 - loss: 0.1487 - auc: 0.7394Mon Sep 6 13:19:54 2021[1,0]:Mon Sep 6 13:19:37 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:20:45 2021[1,0]: 691/4000 [====>.........................]Mon Sep 6 13:20:45 2021[1,0]: - ETA: 7:34:27 - loss: 0.1485 - auc: 0.7398Mon Sep 6 13:20:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:21:43 2021[1,0]: 698/4000 [====>.........................]Mon Sep 6 13:21:43 2021[1,0]: - ETA: 7:33:32 - loss: 0.1483 - auc: 0.7402Mon Sep 6 13:21:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:22:41 2021[1,0]: 705/4000 [====>.........................]Mon Sep 6 13:22:41 2021[1,0]: - ETA: 7:32:35 - loss: 0.1481 - auc: 0.7406Mon Sep 6 13:22:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:23:46 2021[1,0]: 713/4000 [====>.........................]Mon Sep 6 13:23:46 2021[1,0]: - ETA: 7:31:21 - loss: 0.1478 - auc: 0.7410Mon Sep 6 13:23:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:24:43 2021[1,0]: 720/4000 [====>.........................]Mon Sep 6 13:24:43 2021[1,0]: - ETA: 7:30:24 - loss: 0.1476 - auc: 0.7414Mon Sep 6 13:24:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:25:43 2021[1,0]: 727/4000 [====>.........................]Mon Sep 6 13:25:43 2021[1,0]: - ETA: 7:29:35 - loss: 0.1474 - auc: 0.7417Mon Sep 6 13:25:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:26:44 2021[1,0]: 734/4000 [====>.........................]Mon Sep 6 13:26:44 2021[1,0]: - ETA: 7:28:51 - loss: 0.1472 - auc: 0.7421Mon Sep 6 13:26:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:27:37 2021[1,0]: 740/4000 [====>.........................]Mon Sep 6 13:27:37 2021[1,0]: - ETA: 7:28:19 - loss: 0.1470 - auc: 0.7424Mon Sep 6 13:27:45 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:28:38 2021[1,0]: 747/4000 [====>.........................]Mon Sep 6 13:28:38 2021[1,0]: - ETA: 7:27:33 - loss: 0.1469 - auc: 0.7427Mon Sep 6 13:28:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:29:39 2021[1,0]: 754/4000 [====>.........................]Mon Sep 6 13:29:39 2021[1,0]: - ETA: 7:26:52 - loss: 0.1467 - auc: 0.7431Mon Sep 6 13:29:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:30:38 2021[1,0]: 761/4000 [====>.........................]Mon Sep 6 13:30:38 2021[1,0]: - ETA: 7:25:59 - loss: 0.1465 - auc: 0.7434Mon Sep 6 13:30:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 768/4000 [====>.........................]Mon Sep 6 13:31:38 2021[1,0]: - ETA: 7:25:08 - loss: 0.1463 - auc: 0.7437Mon Sep 6 13:31:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 775/4000 [====>.........................]Mon Sep 6 13:32:37 2021[1,0]: - ETA: 7:24:18 - loss: 0.1461 - auc: 0.7441Mon Sep 6 13:32:46 2021[1,0]:Mon Sep 6 13:32:29 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:33:37 2021[1,0]: 782/4000 [====>.........................]Mon Sep 6 13:33:37 2021[1,0]: - ETA: 7:23:28 - loss: 0.1459 - auc: 0.7444Mon Sep 6 13:33:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:34:37 2021[1,0]: 789/4000 [====>.........................]Mon Sep 6 13:34:37 2021[1,0]: - ETA: 7:22:40 - loss: 0.1458 - auc: 0.7447Mon Sep 6 13:34:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:35:45 2021[1,0]: 797/4000 [====>.........................]Mon Sep 6 13:35:45 2021[1,0]: - ETA: 7:21:40 - loss: 0.1456 - auc: 0.7451Mon Sep 6 13:35:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 803/4000 [=====>........................]Mon Sep 6 13:36:38 2021[1,0]: - ETA: 7:21:02 - loss: 0.1454 - auc: 0.7454Mon Sep 6 13:36:46 2021[1,0]:Mon Sep 6 13:36:20 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:37:46 2021[1,0]: 811/4000 [=====>........................]Mon Sep 6 13:37:46 2021[1,0]: - ETA: 7:20:03 - loss: 0.1452 - auc: 0.7457Mon Sep 6 13:37:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:38:45 2021[1,0]: 818/4000 [=====>........................]Mon Sep 6 13:38:45 2021[1,0]: - ETA: 7:19:11 - loss: 0.1451 - auc: 0.7460Mon Sep 6 13:38:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:39:43 2021[1,0]: 825/4000 [=====>........................]Mon Sep 6 13:39:43 2021[1,0]: - ETA: 7:18:13 - loss: 0.1449 - auc: 0.7463Mon Sep 6 13:39:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:40:40 2021[1,0]: 832/4000 [=====>........................]Mon Sep 6 13:40:40 2021[1,0]: - ETA: 7:17:10 - loss: 0.1448 - auc: 0.7466Mon Sep 6 13:40:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 839/4000 [=====>........................]Mon Sep 6 13:41:39 2021[1,0]: - ETA: 7:16:15 - loss: 0.1446 - auc: 0.7469Mon Sep 6 13:41:47 2021[1,0]:Mon Sep 6 13:41:21 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:42:39 2021[1,0]: 846/4000 [=====>........................]Mon Sep 6 13:42:39 2021[1,0]: - ETA: 7:15:25 - loss: 0.1444 - auc: 0.7472Mon Sep 6 13:42:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:43:44 2021[1,0]: 854/4000 [=====>........................]Mon Sep 6 13:43:44 2021[1,0]: - ETA: 7:14:13 - loss: 0.1443 - auc: 0.7475Mon Sep 6 13:43:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:44:42 2021[1,0]: 861/4000 [=====>........................]Mon Sep 6 13:44:42 2021[1,0]: - ETA: 7:13:19 - loss: 0.1441 - auc: 0.7478Mon Sep 6 13:44:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:45:43 2021[1,0]: 868/4000 [=====>........................]Mon Sep 6 13:45:43 2021[1,0]: - ETA: 7:12:32 - loss: 0.1440 - auc: 0.7481Mon Sep 6 13:45:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:46:43 2021[1,0]: 875/4000 [=====>........................]Mon Sep 6 13:46:43 2021[1,0]: - ETA: 7:11:40 - loss: 0.1438 - auc: 0.7483Mon Sep 6 13:46:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:47:43 2021[1,0]: 882/4000 [=====>........................]Mon Sep 6 13:47:43 2021[1,0]: - ETA: 7:10:49 - loss: 0.1437 - auc: 0.7486Mon Sep 6 13:47:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:48:44 2021[1,0]: 889/4000 [=====>........................]Mon Sep 6 13:48:44 2021[1,0]: - ETA: 7:10:00 - loss: 0.1435 - auc: 0.7489Mon Sep 6 13:48:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:49:43 2021[1,0]: 896/4000 [=====>........................]Mon Sep 6 13:49:43 2021[1,0]: - ETA: 7:09:06 - loss: 0.1434 - auc: 0.7491Mon Sep 6 13:49:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:50:41 2021[1,0]: 903/4000 [=====>........................]Mon Sep 6 13:50:41 2021[1,0]: - ETA: 7:08:08 - loss: 0.1433 - auc: 0.7494Mon Sep 6 13:50:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:51:39 2021[1,0]: 910/4000 [=====>........................]Mon Sep 6 13:51:39 2021[1,0]: - ETA: 7:07:11 - loss: 0.1431 - auc: 0.7496Mon Sep 6 13:51:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:52:41 2021[1,0]: 917/4000 [=====>........................]Mon Sep 6 13:52:41 2021[1,0]: - ETA: 7:06:25 - loss: 0.1430 - auc: 0.7498Mon Sep 6 13:52:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:53:42 2021[1,0]: 924/4000 [=====>........................]Mon Sep 6 13:53:42 2021[1,0]: - ETA: 7:05:35 - loss: 0.1429 - auc: 0.7501Mon Sep 6 13:53:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 931/4000 [=====>........................]Mon Sep 6 13:54:43 2021[1,0]: - ETA: 7:04:47 - loss: 0.1427 - auc: 0.7503Mon Sep 6 13:54:51 2021[1,0]:Mon Sep 6 13:54:34 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:55:43 2021[1,0]: 938/4000 [======>.......................]Mon Sep 6 13:55:43 2021[1,0]: - ETA: 7:03:56 - loss: 0.1426 - auc: 0.7505Mon Sep 6 13:55:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:56:43 2021[1,0]: 945/4000 [======>.......................]Mon Sep 6 13:56:43 2021[1,0]: - ETA: 7:03:04 - loss: 0.1425 - auc: 0.7508Mon Sep 6 13:56:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 952/4000 [======>.......................]Mon Sep 6 13:57:43 2021[1,0]: - ETA: 7:02:11 - loss: 0.1424 - auc: 0.7510Mon Sep 6 13:57:51 2021[1,0]:Mon Sep 6 13:57:43 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:58:42 2021[1,0]: 959/4000 [======>.......................]Mon Sep 6 13:58:42 2021[1,0]: - ETA: 7:01:16 - loss: 0.1422 - auc: 0.7512Mon Sep 6 13:58:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:59:42 2021[1,0]: 966/4000 [======>.......................]Mon Sep 6 13:59:42 2021[1,0]: - ETA: 7:00:24 - loss: 0.1421 - auc: 0.7515Mon Sep 6 13:59:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 973/4000 [======>.......................]Mon Sep 6 14:00:40 2021[1,0]: - ETA: 6:59:25 - loss: 0.1420 - auc: 0.7517Mon Sep 6 14:00:48 2021[1,0]:Mon Sep 6 14:00:40 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:01:40 2021[1,0]: 980/4000 [======>.......................]Mon Sep 6 14:01:40 2021[1,0]: - ETA: 6:58:30 - loss: 0.1419 - auc: 0.7519Mon Sep 6 14:01:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:02:39 2021[1,0]: 987/4000 [======>.......................]Mon Sep 6 14:02:39 2021[1,0]: - ETA: 6:57:34 - loss: 0.1418 - auc: 0.7521Mon Sep 6 14:02:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:03:46 2021[1,0]: 995/4000 [======>.......................]Mon Sep 6 14:03:46 2021[1,0]: - ETA: 6:56:32 - loss: 0.1416 - auc: 0.7524Mon Sep 6 14:03:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:04:45 2021[1,0]:1002/4000 [======>.......................]Mon Sep 6 14:04:45 2021[1,0]: - ETA: 6:55:34 - loss: 0.1415 - auc: 0.7526Mon Sep 6 14:04:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:05:45 2021[1,0]:1009/4000 [======>.......................]Mon Sep 6 14:05:45 2021[1,0]: - ETA: 6:54:42 - loss: 0.1414 - auc: 0.7528Mon Sep 6 14:05:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:06:45 2021[1,0]:1016/4000 [======>.......................]Mon Sep 6 14:06:45 2021[1,0]: - ETA: 6:53:49 - loss: 0.1413 - auc: 0.7530Mon Sep 6 14:06:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:07:45 2021[1,0]:1023/4000 [======>.......................]Mon Sep 6 14:07:45 2021[1,0]: - ETA: 6:52:55 - loss: 0.1412 - auc: 0.7532Mon Sep 6 14:07:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1030/4000 [======>.......................]Mon Sep 6 14:08:45 2021[1,0]: - ETA: 6:52:02 - loss: 0.1411 - auc: 0.7534Mon Sep 6 14:08:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:09:44 2021[1,0]:1037/4000 [======>.......................]Mon Sep 6 14:09:44 2021[1,0]: - ETA: 6:51:06 - loss: 0.1410 - auc: 0.7536Mon Sep 6 14:09:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1044/4000 [======>.......................]Mon Sep 6 14:10:42 2021[1,0]: - ETA: 6:50:08 - loss: 0.1409 - auc: 0.7538Mon Sep 6 14:10:51 2021[1,0]:Mon Sep 6 14:10:25 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:11:41 2021[1,0]:1051/4000 [======>.......................]Mon Sep 6 14:11:41 2021[1,0]: - ETA: 6:49:12 - loss: 0.1408 - auc: 0.7540Mon Sep 6 14:11:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:12:41 2021[1,0]:1058/4000 [======>.......................]Mon Sep 6 14:12:41 2021[1,0]: - ETA: 6:48:18 - loss: 0.1407 - auc: 0.7542Mon Sep 6 14:12:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:13:42 2021[1,0]:1065/4000 [======>.......................]Mon Sep 6 14:13:42 2021[1,0]: - ETA: 6:47:28 - loss: 0.1406 - auc: 0.7544Mon Sep 6 14:13:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1072/4000 [=======>......................]Mon Sep 6 14:14:43 2021[1,0]: - ETA: 6:46:35 - loss: 0.1405 - auc: 0.7546Mon Sep 6 14:14:51 2021[1,0]:Mon Sep 6 14:14:43 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:15:42 2021[1,0]:1079/4000 [=======>......................]Mon Sep 6 14:15:42 2021[1,0]: - ETA: 6:45:40 - loss: 0.1404 - auc: 0.7548Mon Sep 6 14:15:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1086/4000 [=======>......................]Mon Sep 6 14:16:42 2021[1,0]: - ETA: 6:44:46 - loss: 0.1403 - auc: 0.7550Mon Sep 6 14:16:51 2021[1,0]:Mon Sep 6 14:16:42 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:17:42 2021[1,0]:1093/4000 [=======>......................]Mon Sep 6 14:17:42 2021[1,0]: - ETA: 6:43:52 - loss: 0.1402 - auc: 0.7551Mon Sep 6 14:17:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:18:43 2021[1,0]:1100/4000 [=======>......................]Mon Sep 6 14:18:43 2021[1,0]: - ETA: 6:42:59 - loss: 0.1402 - auc: 0.7553Mon Sep 6 14:18:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:19:43 2021[1,0]:1107/4000 [=======>......................]Mon Sep 6 14:19:43 2021[1,0]: - ETA: 6:42:06 - loss: 0.1401 - auc: 0.7555Mon Sep 6 14:19:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:20:44 2021[1,0]:1114/4000 [=======>......................]Mon Sep 6 14:20:44 2021[1,0]: - ETA: 6:41:13 - loss: 0.1400 - auc: 0.7556Mon Sep 6 14:20:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:21:45 2021[1,0]:1121/4000 [=======>......................]Mon Sep 6 14:21:45 2021[1,0]: - ETA: 6:40:23 - loss: 0.1399 - auc: 0.7558Mon Sep 6 14:21:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:22:45 2021[1,0]:1128/4000 [=======>......................]Mon Sep 6 14:22:45 2021[1,0]: - ETA: 6:39:29 - loss: 0.1398 - auc: 0.7560Mon Sep 6 14:22:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:23:46 2021[1,0]:1135/4000 [=======>......................]Mon Sep 6 14:23:46 2021[1,0]: - ETA: 6:38:37 - loss: 0.1397 - auc: 0.7562Mon Sep 6 14:23:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:24:39 2021[1,0]:1141/4000 [=======>......................]Mon Sep 6 14:24:39 2021[1,0]: - ETA: 6:37:54 - loss: 0.1396 - auc: 0.7563Mon Sep 6 14:24:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:25:40 2021[1,0]:1148/4000 [=======>......................]Mon Sep 6 14:25:40 2021[1,0]: - ETA: 6:37:01 - loss: 0.1395 - auc: 0.7565Mon Sep 6 14:25:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1156/4000 [=======>......................]Mon Sep 6 14:26:46 2021[1,0]: - ETA: 6:35:52 - loss: 0.1394 - auc: 0.7567Mon Sep 6 14:26:54 2021[1,0]:Mon Sep 6 14:26:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:27:46 2021[1,0]:1163/4000 [=======>......................]Mon Sep 6 14:27:46 2021[1,0]: - ETA: 6:34:58 - loss: 0.1393 - auc: 0.7568Mon Sep 6 14:27:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1170/4000 [=======>......................]Mon Sep 6 14:28:46 2021[1,0]: - ETA: 6:34:04 - loss: 0.1393 - auc: 0.7570Mon Sep 6 14:28:55 2021[1,0]:Mon Sep 6 14:28:38 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:29:47 2021[1,0]:1177/4000 [=======>......................]Mon Sep 6 14:29:47 2021[1,0]: - ETA: 6:33:10 - loss: 0.1392 - auc: 0.7571Mon Sep 6 14:29:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:30:47 2021[1,0]:1184/4000 [=======>......................]Mon Sep 6 14:30:47 2021[1,0]: - ETA: 6:32:16 - loss: 0.1391 - auc: 0.7573Mon Sep 6 14:30:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1190/4000 [=======>......................]Mon Sep 6 14:31:39 2021[1,0]: - ETA: 6:31:30 - loss: 0.1390 - auc: 0.7575Mon Sep 6 14:31:47 2021[1,0]:Mon Sep 6 14:31:30 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:32:39 2021[1,0]:1197/4000 [=======>......................]Mon Sep 6 14:32:39 2021[1,0]: - ETA: 6:30:35 - loss: 0.1389 - auc: 0.7576Mon Sep 6 14:32:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:33:38 2021[1,0]:1204/4000 [========>.....................]Mon Sep 6 14:33:38 2021[1,0]: - ETA: 6:29:38 - loss: 0.1389 - auc: 0.7578Mon Sep 6 14:33:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:34:39 2021[1,0]:1211/4000 [========>.....................]Mon Sep 6 14:34:39 2021[1,0]: - ETA: 6:28:45 - loss: 0.1388 - auc: 0.7579Mon Sep 6 14:34:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:35:45 2021[1,0]:1219/4000 [========>.....................] - ETA: 6:27:36 - loss: 0.1387 - auc: 0.7581Mon Sep 6 14:35:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:36:44 2021[1,0]:1226/4000 [========>.....................]Mon Sep 6 14:36:44 2021[1,0]: - ETA: 6:26:39 - loss: 0.1386 - auc: 0.7582Mon Sep 6 14:36:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1233/4000 [========>.....................]Mon Sep 6 14:37:45 2021[1,0]: - ETA: 6:25:45 - loss: 0.1386 - auc: 0.7583Mon Sep 6 14:37:53 2021[1,0]:Mon Sep 6 14:37:19 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:38:44 2021[1,0]:1240/4000 [========>.....................]Mon Sep 6 14:38:44 2021[1,0]: - ETA: 6:24:47 - loss: 0.1385 - auc: 0.7585Mon Sep 6 14:38:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1247/4000 [========>.....................]Mon Sep 6 14:39:40 2021[1,0]: - ETA: 6:23:43 - loss: 0.1384 - auc: 0.7586Mon Sep 6 14:39:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:40:45 2021[1,0]:1255/4000 [========>.....................]Mon Sep 6 14:40:45 2021[1,0]: - ETA: 6:22:32 - loss: 0.1383 - auc: 0.7588Mon Sep 6 14:40:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:41:45 2021[1,0]:1262/4000 [========>.....................]Mon Sep 6 14:41:45 2021[1,0]: - ETA: 6:21:38 - loss: 0.1383 - auc: 0.7589Mon Sep 6 14:41:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:42:43 2021[1,0]:1269/4000 [========>.....................]Mon Sep 6 14:42:43 2021[1,0]: - ETA: 6:20:38 - loss: 0.1382 - auc: 0.7591Mon Sep 6 14:42:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1276/4000 [========>.....................]Mon Sep 6 14:43:43 2021[1,0]: - ETA: 6:19:41 - loss: 0.1381 - auc: 0.7592Mon Sep 6 14:43:51 2021[1,0]:Mon Sep 6 14:43:43 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:44:43 2021[1,0]:1283/4000 [========>.....................]Mon Sep 6 14:44:43 2021[1,0]: - ETA: 6:18:47 - loss: 0.1381 - auc: 0.7594Mon Sep 6 14:44:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:45:43 2021[1,0]:1290/4000 [========>.....................]Mon Sep 6 14:45:43 2021[1,0]: - ETA: 6:17:50 - loss: 0.1380 - auc: 0.7595Mon Sep 6 14:45:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:46:43 2021[1,0]:1297/4000 [========>.....................]Mon Sep 6 14:46:43 2021[1,0]: - ETA: 6:16:55 - loss: 0.1379 - auc: 0.7596Mon Sep 6 14:46:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1304/4000 [========>.....................]Mon Sep 6 14:47:43 2021[1,0]: - ETA: 6:16:01 - loss: 0.1379 - auc: 0.7598Mon Sep 6 14:47:52 2021[1,0]:Mon Sep 6 14:47:43 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:48:43 2021[1,0]:1311/4000 [========>.....................]Mon Sep 6 14:48:43 2021[1,0]: - ETA: 6:15:04 - loss: 0.1378 - auc: 0.7599Mon Sep 6 14:48:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1318/4000 [========>.....................]Mon Sep 6 14:49:43 2021[1,0]: - ETA: 6:14:09 - loss: 0.1377 - auc: 0.7601Mon Sep 6 14:49:52 2021[1,0]:Mon Sep 6 14:49:26 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:50:42 2021[1,0]:1325/4000 [========>.....................]Mon Sep 6 14:50:42 2021[1,0]: - ETA: 6:13:10 - loss: 0.1377 - auc: 0.7602Mon Sep 6 14:50:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:51:41 2021[1,0]:1332/4000 [========>.....................]Mon Sep 6 14:51:41 2021[1,0]: - ETA: 6:12:13 - loss: 0.1376 - auc: 0.7603Mon Sep 6 14:51:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:52:40 2021[1,0]:1339/4000 [=========>....................]Mon Sep 6 14:52:40 2021[1,0]: - ETA: 6:11:14 - loss: 0.1375 - auc: 0.7605Mon Sep 6 14:52:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:53:46 2021[1,0]:1347/4000 [=========>....................]Mon Sep 6 14:53:46 2021[1,0]: - ETA: 6:10:06 - loss: 0.1375 - auc: 0.7606Mon Sep 6 14:53:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:54:46 2021[1,0]:1354/4000 [=========>....................]Mon Sep 6 14:54:46 2021[1,0]: - ETA: 6:09:10 - loss: 0.1374 - auc: 0.7608Mon Sep 6 14:54:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:55:44 2021[1,0]:1361/4000 [=========>....................]Mon Sep 6 14:55:44 2021[1,0]: - ETA: 6:08:11 - loss: 0.1373 - auc: 0.7609Mon Sep 6 14:55:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:56:43 2021[1,0]:1368/4000 [=========>....................]Mon Sep 6 14:56:43 2021[1,0]: - ETA: 6:07:12 - loss: 0.1373 - auc: 0.7610Mon Sep 6 14:56:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1375/4000 [=========>....................]Mon Sep 6 14:57:41 2021[1,0]: - ETA: 6:06:13 - loss: 0.1372 - auc: 0.7611Mon Sep 6 14:57:50 2021[1,0]:Mon Sep 6 14:57:33 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1382/4000 [=========>....................]Mon Sep 6 14:58:41 2021[1,0]: - ETA: 6:05:17 - loss: 0.1372 - auc: 0.7613Mon Sep 6 14:58:49 2021[1,0]:Mon Sep 6 14:58:41 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:59:42 2021[1,0]:1389/4000 [=========>....................]Mon Sep 6 14:59:42 2021[1,0]: - ETA: 6:04:23 - loss: 0.1371 - auc: 0.7614Mon Sep 6 14:59:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:00:43 2021[1,0]:1396/4000 [=========>....................]Mon Sep 6 15:00:43 2021[1,0]: - ETA: 6:03:29 - loss: 0.1370 - auc: 0.7615Mon Sep 6 15:00:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:01:43 2021[1,0]:1403/4000 [=========>....................]Mon Sep 6 15:01:43 2021[1,0]: - ETA: 6:02:33 - loss: 0.1370 - auc: 0.7616Mon Sep 6 15:01:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1410/4000 [=========>....................]Mon Sep 6 15:02:43 2021[1,0]: - ETA: 6:01:37 - loss: 0.1369 - auc: 0.7618Mon Sep 6 15:02:52 2021[1,0]:Mon Sep 6 15:02:43 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:03:43 2021[1,0]:1417/4000 [=========>....................]Mon Sep 6 15:03:43 2021[1,0]: - ETA: 6:00:40 - loss: 0.1369 - auc: 0.7619Mon Sep 6 15:03:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:04:43 2021[1,0]:1424/4000 [=========>....................]Mon Sep 6 15:04:43 2021[1,0]: - ETA: 5:59:44 - loss: 0.1368 - auc: 0.7620Mon Sep 6 15:04:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:05:43 2021[1,0]:1431/4000 [=========>....................]Mon Sep 6 15:05:43 2021[1,0]: - ETA: 5:58:48 - loss: 0.1368 - auc: 0.7621Mon Sep 6 15:05:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:06:44 2021[1,0]:1438/4000 [=========>....................]Mon Sep 6 15:06:44 2021[1,0]: - ETA: 5:57:53 - loss: 0.1367 - auc: 0.7622Mon Sep 6 15:06:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:07:45 2021[1,0]:1445/4000 [=========>....................]Mon Sep 6 15:07:45 2021[1,0]: - ETA: 5:56:58 - loss: 0.1367 - auc: 0.7623Mon Sep 6 15:07:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1452/4000 [=========>....................]Mon Sep 6 15:08:46 2021[1,0]: - ETA: 5:56:04 - loss: 0.1366 - auc: 0.7624Mon Sep 6 15:08:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:09:45 2021[1,0]:1459/4000 [=========>....................]Mon Sep 6 15:09:45 2021[1,0]: - ETA: 5:55:05 - loss: 0.1365 - auc: 0.7625Mon Sep 6 15:09:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:10:44 2021[1,0]:1466/4000 [=========>....................]Mon Sep 6 15:10:44 2021[1,0]: - ETA: 5:54:08 - loss: 0.1365 - auc: 0.7627Mon Sep 6 15:10:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1473/4000 [==========>...................]Mon Sep 6 15:11:42 2021[1,0]: - ETA: 5:53:08 - loss: 0.1364 - auc: 0.7627Mon Sep 6 15:11:51 2021[1,0]:Mon Sep 6 15:11:33 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:12:47 2021[1,0]:1481/4000 [==========>...................]Mon Sep 6 15:12:47 2021[1,0]: - ETA: 5:51:58 - loss: 0.1364 - auc: 0.7629Mon Sep 6 15:12:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1488/4000 [==========>...................]Mon Sep 6 15:13:44 2021[1,0]: - ETA: 5:50:57 - loss: 0.1363 - auc: 0.7630Mon Sep 6 15:13:52 2021[1,0]:Mon Sep 6 15:13:36 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:14:40 2021[1,0]:1495/4000 [==========>...................]Mon Sep 6 15:14:40 2021[1,0]: - ETA: 5:49:53 - loss: 0.1363 - auc: 0.7631Mon Sep 6 15:14:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:15:40 2021[1,0]:1502/4000 [==========>...................]Mon Sep 6 15:15:40 2021[1,0]: - ETA: 5:48:57 - loss: 0.1362 - auc: 0.7632Mon Sep 6 15:15:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:16:41 2021[1,0]:1509/4000 [==========>...................]Mon Sep 6 15:16:41 2021[1,0]: - ETA: 5:48:02 - loss: 0.1362 - auc: 0.7633Mon Sep 6 15:16:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:17:41 2021[1,0]:1516/4000 [==========>...................]Mon Sep 6 15:17:41 2021[1,0]: - ETA: 5:47:05 - loss: 0.1361 - auc: 0.7634Mon Sep 6 15:17:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1523/4000 [==========>...................]Mon Sep 6 15:18:40 2021[1,0]: - ETA: 5:46:08 - loss: 0.1361 - auc: 0.7635Mon Sep 6 15:18:49 2021[1,0]:Mon Sep 6 15:18:32 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:19:48 2021[1,0]:1531/4000 [==========>...................]Mon Sep 6 15:19:48 2021[1,0]: - ETA: 5:45:01 - loss: 0.1360 - auc: 0.7636Mon Sep 6 15:19:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:20:45 2021[1,0]:1538/4000 [==========>...................]Mon Sep 6 15:20:45 2021[1,0]: - ETA: 5:44:00 - loss: 0.1360 - auc: 0.7637Mon Sep 6 15:20:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:21:46 2021[1,0]:1545/4000 [==========>...................] - ETA: 5:43:05 - loss: 0.1359 - auc: 0.7638Mon Sep 6 15:21:54 2021[1,0]:Mon Sep 6 15:21:20 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1552/4000 [==========>...................]Mon Sep 6 15:22:46 2021[1,0]: - ETA: 5:42:08 - loss: 0.1359 - auc: 0.7640Mon Sep 6 15:22:54 2021[1,0]:Mon Sep 6 15:22:37 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:23:45 2021[1,0]:1559/4000 [==========>...................]Mon Sep 6 15:23:45 2021[1,0]: - ETA: 5:41:10 - loss: 0.1358 - auc: 0.7640Mon Sep 6 15:23:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:24:44 2021[1,0]:1566/4000 [==========>...................]Mon Sep 6 15:24:44 2021[1,0]: - ETA: 5:40:12 - loss: 0.1358 - auc: 0.7641Mon Sep 6 15:24:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:25:42 2021[1,0]:1573/4000 [==========>...................]Mon Sep 6 15:25:42 2021[1,0]: - ETA: 5:39:12 - loss: 0.1357 - auc: 0.7642Mon Sep 6 15:25:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:26:48 2021[1,0]:1581/4000 [==========>...................]Mon Sep 6 15:26:48 2021[1,0]: - ETA: 5:38:04 - loss: 0.1357 - auc: 0.7643Mon Sep 6 15:26:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:27:47 2021[1,0]:1588/4000 [==========>...................]Mon Sep 6 15:27:47 2021[1,0]: - ETA: 5:37:06 - loss: 0.1356 - auc: 0.7645Mon Sep 6 15:27:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:28:48 2021[1,0]:1595/4000 [==========>...................]Mon Sep 6 15:28:48 2021[1,0]: - ETA: 5:36:10 - loss: 0.1356 - auc: 0.7646Mon Sep 6 15:28:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:29:47 2021[1,0]:1602/4000 [===========>..................]Mon Sep 6 15:29:47 2021[1,0]: - ETA: 5:35:12 - loss: 0.1356 - auc: 0.7647Mon Sep 6 15:29:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1609/4000 [===========>..................]Mon Sep 6 15:30:47 2021[1,0]: - ETA: 5:34:15 - loss: 0.1355 - auc: 0.7648Mon Sep 6 15:30:55 2021[1,0]:Mon Sep 6 15:30:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:31:45 2021[1,0]:1616/4000 [===========>..................]Mon Sep 6 15:31:45 2021[1,0]: - ETA: 5:33:16 - loss: 0.1355 - auc: 0.7649Mon Sep 6 15:31:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:32:44 2021[1,0]:1623/4000 [===========>..................]Mon Sep 6 15:32:44 2021[1,0]: - ETA: 5:32:17 - loss: 0.1354 - auc: 0.7650Mon Sep 6 15:32:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:33:43 2021[1,0]:1630/4000 [===========>..................]Mon Sep 6 15:33:43 2021[1,0]: - ETA: 5:31:19 - loss: 0.1354 - auc: 0.7651Mon Sep 6 15:33:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:34:43 2021[1,0]:1637/4000 [===========>..................]Mon Sep 6 15:34:43 2021[1,0]: - ETA: 5:30:21 - loss: 0.1353 - auc: 0.7651Mon Sep 6 15:34:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:35:43 2021[1,0]:1644/4000 [===========>..................]Mon Sep 6 15:35:43 2021[1,0]: - ETA: 5:29:24 - loss: 0.1353 - auc: 0.7652Mon Sep 6 15:35:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:36:41 2021[1,0]:1651/4000 [===========>..................]Mon Sep 6 15:36:41 2021[1,0]: - ETA: 5:28:25 - loss: 0.1352 - auc: 0.7653Mon Sep 6 15:36:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:37:47 2021[1,0]:1659/4000 [===========>..................]Mon Sep 6 15:37:47 2021[1,0]: - ETA: 5:27:16 - loss: 0.1352 - auc: 0.7654Mon Sep 6 15:37:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:38:44 2021[1,0]:1666/4000 [===========>..................]Mon Sep 6 15:38:44 2021[1,0]: - ETA: 5:26:15 - loss: 0.1352 - auc: 0.7655Mon Sep 6 15:38:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:39:42 2021[1,0]:1673/4000 [===========>..................]Mon Sep 6 15:39:42 2021[1,0]: - ETA: 5:25:15 - loss: 0.1351 - auc: 0.7656Mon Sep 6 15:39:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:40:48 2021[1,0]:1681/4000 [===========>..................]Mon Sep 6 15:40:48 2021[1,0]: - ETA: 5:24:07 - loss: 0.1351 - auc: 0.7657Mon Sep 6 15:40:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:41:40 2021[1,0]:1687/4000 [===========>..................]Mon Sep 6 15:41:40 2021[1,0]: - ETA: 5:23:19 - loss: 0.1350 - auc: 0.7658Mon Sep 6 15:41:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:42:41 2021[1,0]:1694/4000 [===========>..................]Mon Sep 6 15:42:41 2021[1,0]: - ETA: 5:22:23 - loss: 0.1350 - auc: 0.7659Mon Sep 6 15:42:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:43:41 2021[1,0]:1701/4000 [===========>..................]Mon Sep 6 15:43:41 2021[1,0]: - ETA: 5:21:26 - loss: 0.1349 - auc: 0.7660Mon Sep 6 15:43:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1708/4000 [===========>..................]Mon Sep 6 15:44:42 2021[1,0]: - ETA: 5:20:30 - loss: 0.1349 - auc: 0.7661Mon Sep 6 15:44:50 2021[1,0]:Mon Sep 6 15:44:33 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1715/4000 [===========>..................]Mon Sep 6 15:45:42 2021[1,0]: - ETA: 5:19:34 - loss: 0.1349 - auc: 0.7661Mon Sep 6 15:45:51 2021[1,0]:Mon Sep 6 15:45:42 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:46:43 2021[1,0]:1722/4000 [===========>..................]Mon Sep 6 15:46:43 2021[1,0]: - ETA: 5:18:38 - loss: 0.1348 - auc: 0.7662Mon Sep 6 15:46:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:47:44 2021[1,0]:1729/4000 [===========>..................]Mon Sep 6 15:47:44 2021[1,0]: - ETA: 5:17:42 - loss: 0.1348 - auc: 0.7663Mon Sep 6 15:47:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:48:44 2021[1,0]:1736/4000 [============>.................]Mon Sep 6 15:48:44 2021[1,0]: - ETA: 5:16:45 - loss: 0.1348 - auc: 0.7664Mon Sep 6 15:48:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:49:41 2021[1,0]:1743/4000 [============>.................]Mon Sep 6 15:49:41 2021[1,0]: - ETA: 5:15:44 - loss: 0.1347 - auc: 0.7665Mon Sep 6 15:49:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:50:45 2021[1,0]:1751/4000 [============>.................]Mon Sep 6 15:50:45 2021[1,0]: - ETA: 5:14:33 - loss: 0.1347 - auc: 0.7666Mon Sep 6 15:50:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:51:47 2021[1,0]:1759/4000 [============>.................]Mon Sep 6 15:51:47 2021[1,0]: - ETA: 5:13:19 - loss: 0.1346 - auc: 0.7667Mon Sep 6 15:51:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:52:43 2021[1,0]:1766/4000 [============>.................]Mon Sep 6 15:52:43 2021[1,0]: - ETA: 5:12:16 - loss: 0.1346 - auc: 0.7667Mon Sep 6 15:52:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:53:49 2021[1,0]:1774/4000 [============>.................]Mon Sep 6 15:53:49 2021[1,0]: - ETA: 5:11:08 - loss: 0.1346 - auc: 0.7668Mon Sep 6 15:53:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:54:48 2021[1,0]:1781/4000 [============>.................]Mon Sep 6 15:54:48 2021[1,0]: - ETA: 5:10:10 - loss: 0.1345 - auc: 0.7669Mon Sep 6 15:54:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:55:47 2021[1,0]:1788/4000 [============>.................]Mon Sep 6 15:55:47 2021[1,0]: - ETA: 5:09:12 - loss: 0.1345 - auc: 0.7670Mon Sep 6 15:55:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:56:46 2021[1,0]:1795/4000 [============>.................]Mon Sep 6 15:56:46 2021[1,0]: - ETA: 5:08:13 - loss: 0.1344 - auc: 0.7671Mon Sep 6 15:56:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:57:44 2021[1,0]:1802/4000 [============>.................]Mon Sep 6 15:57:44 2021[1,0]: - ETA: 5:07:13 - loss: 0.1344 - auc: 0.7671Mon Sep 6 15:57:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:58:43 2021[1,0]:1809/4000 [============>.................]Mon Sep 6 15:58:43 2021[1,0]: - ETA: 5:06:15 - loss: 0.1344 - auc: 0.7672Mon Sep 6 15:58:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:59:42 2021[1,0]:1816/4000 [============>.................]Mon Sep 6 15:59:42 2021[1,0]: - ETA: 5:05:17 - loss: 0.1343 - auc: 0.7673Mon Sep 6 15:59:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:00:41 2021[1,0]:1823/4000 [============>.................]Mon Sep 6 16:00:41 2021[1,0]: - ETA: 5:04:19 - loss: 0.1343 - auc: 0.7674Mon Sep 6 16:00:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:01:41 2021[1,0]:1830/4000 [============>.................]Mon Sep 6 16:01:41 2021[1,0]: - ETA: 5:03:21 - loss: 0.1343 - auc: 0.7675Mon Sep 6 16:01:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1838/4000 [============>.................]Mon Sep 6 16:02:49 2021[1,0]: - ETA: 5:02:15 - loss: 0.1342 - auc: 0.7676Mon Sep 6 16:02:57 2021[1,0]:Mon Sep 6 16:02:40 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:03:41 2021[1,0]:1844/4000 [============>.................]Mon Sep 6 16:03:41 2021[1,0]: - ETA: 5:01:27 - loss: 0.1342 - auc: 0.7676Mon Sep 6 16:03:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:04:41 2021[1,0]:1851/4000 [============>.................]Mon Sep 6 16:04:41 2021[1,0]: - ETA: 5:00:30 - loss: 0.1342 - auc: 0.7677Mon Sep 6 16:04:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:05:41 2021[1,0]:1858/4000 [============>.................]Mon Sep 6 16:05:41 2021[1,0]: - ETA: 4:59:32 - loss: 0.1341 - auc: 0.7678Mon Sep 6 16:05:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:06:41 2021[1,0]:1865/4000 [============>.................] - ETA: 4:58:35 - loss: 0.1341 - auc: 0.7679Mon Sep 6 16:06:49 2021[1,0]:Mon Sep 6 16:06:32 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:07:41 2021[1,0]:1872/4000 [=============>................]Mon Sep 6 16:07:41 2021[1,0]: - ETA: 4:57:38 - loss: 0.1341 - auc: 0.7679Mon Sep 6 16:07:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:08:49 2021[1,0]:1880/4000 [=============>................]Mon Sep 6 16:08:49 2021[1,0]: - ETA: 4:56:31 - loss: 0.1340 - auc: 0.7680Mon Sep 6 16:08:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:09:47 2021[1,0]:1887/4000 [=============>................]Mon Sep 6 16:09:47 2021[1,0]: - ETA: 4:55:32 - loss: 0.1340 - auc: 0.7681Mon Sep 6 16:09:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:10:44 2021[1,0]:1894/4000 [=============>................]Mon Sep 6 16:10:44 2021[1,0]: - ETA: 4:54:31 - loss: 0.1340 - auc: 0.7682Mon Sep 6 16:10:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:11:43 2021[1,0]:1901/4000 [=============>................]Mon Sep 6 16:11:43 2021[1,0]: - ETA: 4:53:33 - loss: 0.1339 - auc: 0.7682Mon Sep 6 16:11:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:12:49 2021[1,0]:1909/4000 [=============>................]Mon Sep 6 16:12:49 2021[1,0]: - ETA: 4:52:24 - loss: 0.1339 - auc: 0.7683Mon Sep 6 16:12:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:13:45 2021[1,0]:1916/4000 [=============>................]Mon Sep 6 16:13:45 2021[1,0]: - ETA: 4:51:23 - loss: 0.1339 - auc: 0.7684Mon Sep 6 16:13:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:14:44 2021[1,0]:1923/4000 [=============>................]Mon Sep 6 16:14:44 2021[1,0]: - ETA: 4:50:24 - loss: 0.1338 - auc: 0.7685Mon Sep 6 16:14:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:15:48 2021[1,0]:1931/4000 [=============>................]Mon Sep 6 16:15:48 2021[1,0]: - ETA: 4:49:15 - loss: 0.1338 - auc: 0.7686Mon Sep 6 16:15:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:16:46 2021[1,0]:1938/4000 [=============>................]Mon Sep 6 16:16:46 2021[1,0]: - ETA: 4:48:14 - loss: 0.1338 - auc: 0.7686Mon Sep 6 16:16:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:17:43 2021[1,0]:1945/4000 [=============>................]Mon Sep 6 16:17:43 2021[1,0]: - ETA: 4:47:14 - loss: 0.1337 - auc: 0.7687Mon Sep 6 16:17:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:18:49 2021[1,0]:1953/4000 [=============>................]Mon Sep 6 16:18:49 2021[1,0]: - ETA: 4:46:06 - loss: 0.1337 - auc: 0.7688Mon Sep 6 16:18:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:19:48 2021[1,0]:1960/4000 [=============>................]Mon Sep 6 16:19:48 2021[1,0]: - ETA: 4:45:07 - loss: 0.1337 - auc: 0.7689Mon Sep 6 16:19:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:20:49 2021[1,0]:1967/4000 [=============>................]Mon Sep 6 16:20:49 2021[1,0]: - ETA: 4:44:11 - loss: 0.1336 - auc: 0.7689Mon Sep 6 16:20:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:21:41 2021[1,0]:1973/4000 [=============>................]Mon Sep 6 16:21:41 2021[1,0]: - ETA: 4:43:22 - loss: 0.1336 - auc: 0.7690Mon Sep 6 16:21:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:22:49 2021[1,0]:1981/4000 [=============>................]Mon Sep 6 16:22:49 2021[1,0]: - ETA: 4:42:16 - loss: 0.1336 - auc: 0.7691Mon Sep 6 16:22:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:23:45 2021[1,0]:1988/4000 [=============>................]Mon Sep 6 16:23:45 2021[1,0]: - ETA: 4:41:15 - loss: 0.1335 - auc: 0.7692Mon Sep 6 16:23:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:24:49 2021[1,0]:1996/4000 [=============>................]Mon Sep 6 16:24:49 2021[1,0]: - ETA: 4:40:04 - loss: 0.1335 - auc: 0.7692Mon Sep 6 16:24:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2003/4000 [==============>...............]Mon Sep 6 16:25:44 2021[1,0]: - ETA: 4:39:02 - loss: 0.1335 - auc: 0.7693Mon Sep 6 16:25:51 2021[1,0]:Mon Sep 6 16:25:44 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:26:45 2021[1,0]:2011/4000 [==============>...............]Mon Sep 6 16:26:45 2021[1,0]: - ETA: 4:37:49 - loss: 0.1334 - auc: 0.7694Mon Sep 6 16:26:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2018/4000 [==============>...............]Mon Sep 6 16:27:42 2021[1,0]: - ETA: 4:36:48 - loss: 0.1334 - auc: 0.7695Mon Sep 6 16:27:50 2021[1,0]:Mon Sep 6 16:27:25 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:28:49 2021[1,0]:2026/4000 [==============>...............]Mon Sep 6 16:28:49 2021[1,0]: - ETA: 4:35:42 - loss: 0.1334 - auc: 0.7695Mon Sep 6 16:28:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:29:49 2021[1,0]:2033/4000 [==============>...............]Mon Sep 6 16:29:49 2021[1,0]: - ETA: 4:34:45 - loss: 0.1333 - auc: 0.7696Mon Sep 6 16:29:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:30:49 2021[1,0]:2040/4000 [==============>...............] - ETA: 4:33:47 - loss: 0.1333 - auc: 0.7696Mon Sep 6 16:30:58 2021[1,0]:Mon Sep 6 16:30:41 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:31:49 2021[1,0]:2047/4000 [==============>...............]Mon Sep 6 16:31:49 2021[1,0]: - ETA: 4:32:49 - loss: 0.1333 - auc: 0.7697Mon Sep 6 16:31:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:32:47 2021[1,0]:2054/4000 [==============>...............]Mon Sep 6 16:32:47 2021[1,0]: - ETA: 4:31:51 - loss: 0.1333 - auc: 0.7697Mon Sep 6 16:32:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:33:47 2021[1,0]:2061/4000 [==============>...............]Mon Sep 6 16:33:47 2021[1,0]: - ETA: 4:30:52 - loss: 0.1332 - auc: 0.7698Mon Sep 6 16:33:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2068/4000 [==============>...............]Mon Sep 6 16:34:45 2021[1,0]: - ETA: 4:29:54 - loss: 0.1332 - auc: 0.7699Mon Sep 6 16:34:54 2021[1,0]:Mon Sep 6 16:34:45 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:35:46 2021[1,0]:2075/4000 [==============>...............] - ETA: 4:28:57 - loss: 0.1332 - auc: 0.7699Mon Sep 6 16:35:54 2021[1,0]:Mon Sep 6 16:35:20 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:36:44 2021[1,0]:2082/4000 [==============>...............]Mon Sep 6 16:36:44 2021[1,0]: - ETA: 4:27:57 - loss: 0.1331 - auc: 0.7700Mon Sep 6 16:36:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:37:47 2021[1,0]:2090/4000 [==============>...............]Mon Sep 6 16:37:47 2021[1,0]: - ETA: 4:26:47 - loss: 0.1331 - auc: 0.7700Mon Sep 6 16:37:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2097/4000 [==============>...............]Mon Sep 6 16:38:43 2021[1,0]: - ETA: 4:25:46 - loss: 0.1331 - auc: 0.7701Mon Sep 6 16:38:51 2021[1,0]:Mon Sep 6 16:38:43 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2105/4000 [==============>...............]Mon Sep 6 16:39:48 2021[1,0]: - ETA: 4:24:37 - loss: 0.1331 - auc: 0.7702Mon Sep 6 16:39:57 2021[1,0]:Mon Sep 6 16:39:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:40:48 2021[1,0]:2112/4000 [==============>...............]Mon Sep 6 16:40:48 2021[1,0]: - ETA: 4:23:39 - loss: 0.1330 - auc: 0.7702Mon Sep 6 16:40:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:41:48 2021[1,0]:2119/4000 [==============>...............] - ETA: 4:22:42 - loss: 0.1330 - auc: 0.7703Mon Sep 6 16:41:57 2021[1,0]:Mon Sep 6 16:41:39 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:42:48 2021[1,0]:2126/4000 [==============>...............]Mon Sep 6 16:42:48 2021[1,0]: - ETA: 4:21:45 - loss: 0.1330 - auc: 0.7703Mon Sep 6 16:42:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:43:48 2021[1,0]:2133/4000 [==============>...............]Mon Sep 6 16:43:48 2021[1,0]: - ETA: 4:20:47 - loss: 0.1330 - auc: 0.7704Mon Sep 6 16:43:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:44:48 2021[1,0]:2140/4000 [===============>..............]Mon Sep 6 16:44:48 2021[1,0]: - ETA: 4:19:50 - loss: 0.1329 - auc: 0.7704Mon Sep 6 16:44:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:45:47 2021[1,0]:2147/4000 [===============>..............]Mon Sep 6 16:45:47 2021[1,0]: - ETA: 4:18:51 - loss: 0.1329 - auc: 0.7705Mon Sep 6 16:45:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:46:47 2021[1,0]:2154/4000 [===============>..............]Mon Sep 6 16:46:47 2021[1,0]: - ETA: 4:17:54 - loss: 0.1329 - auc: 0.7705Mon Sep 6 16:46:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:47:48 2021[1,0]:2161/4000 [===============>..............]Mon Sep 6 16:47:48 2021[1,0]: - ETA: 4:16:57 - loss: 0.1328 - auc: 0.7706Mon Sep 6 16:47:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2168/4000 [===============>..............]Mon Sep 6 16:48:47 2021[1,0]: - ETA: 4:15:58 - loss: 0.1328 - auc: 0.7707Mon Sep 6 16:48:55 2021[1,0]:Mon Sep 6 16:48:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:49:44 2021[1,0]:2175/4000 [===============>..............]Mon Sep 6 16:49:44 2021[1,0]: - ETA: 4:14:59 - loss: 0.1328 - auc: 0.7707Mon Sep 6 16:49:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:50:43 2021[1,0]:2182/4000 [===============>..............]Mon Sep 6 16:50:43 2021[1,0]: - ETA: 4:14:00 - loss: 0.1328 - auc: 0.7708Mon Sep 6 16:50:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:51:49 2021[1,0]:2190/4000 [===============>..............]Mon Sep 6 16:51:49 2021[1,0]: - ETA: 4:12:52 - loss: 0.1327 - auc: 0.7708Mon Sep 6 16:51:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:52:48 2021[1,0]:2197/4000 [===============>..............]Mon Sep 6 16:52:48 2021[1,0]: - ETA: 4:11:53 - loss: 0.1327 - auc: 0.7709Mon Sep 6 16:52:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:53:47 2021[1,0]:2204/4000 [===============>..............]Mon Sep 6 16:53:47 2021[1,0]: - ETA: 4:10:55 - loss: 0.1327 - auc: 0.7710Mon Sep 6 16:53:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:54:47 2021[1,0]:2211/4000 [===============>..............]Mon Sep 6 16:54:47 2021[1,0]: - ETA: 4:09:58 - loss: 0.1327 - auc: 0.7710Mon Sep 6 16:54:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2218/4000 [===============>..............]Mon Sep 6 16:55:49 2021[1,0]: - ETA: 4:09:02 - loss: 0.1326 - auc: 0.7711Mon Sep 6 16:55:57 2021[1,0]:Mon Sep 6 16:55:40 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:56:48 2021[1,0]:2225/4000 [===============>..............]Mon Sep 6 16:56:48 2021[1,0]: - ETA: 4:08:03 - loss: 0.1326 - auc: 0.7711Mon Sep 6 16:56:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:57:46 2021[1,0]:2232/4000 [===============>..............]Mon Sep 6 16:57:46 2021[1,0]: - ETA: 4:07:04 - loss: 0.1326 - auc: 0.7712Mon Sep 6 16:57:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:58:44 2021[1,0]:2239/4000 [===============>..............]Mon Sep 6 16:58:44 2021[1,0]: - ETA: 4:06:05 - loss: 0.1326 - auc: 0.7712Mon Sep 6 16:58:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:59:42 2021[1,0]:2246/4000 [===============>..............]Mon Sep 6 16:59:42 2021[1,0]: - ETA: 4:05:06 - loss: 0.1326 - auc: 0.7713Mon Sep 6 16:59:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2254/4000 [===============>..............]Mon Sep 6 17:00:48 2021[1,0]: - ETA: 4:03:58 - loss: 0.1325 - auc: 0.7713Mon Sep 6 17:00:56 2021[1,0]:Mon Sep 6 17:00:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:01:47 2021[1,0]:2261/4000 [===============>..............]Mon Sep 6 17:01:47 2021[1,0]: - ETA: 4:02:59 - loss: 0.1325 - auc: 0.7714Mon Sep 6 17:01:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:02:45 2021[1,0]:2268/4000 [================>.............]Mon Sep 6 17:02:45 2021[1,0]: - ETA: 4:02:00 - loss: 0.1325 - auc: 0.7714Mon Sep 6 17:02:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:03:43 2021[1,0]:2275/4000 [================>.............]Mon Sep 6 17:03:43 2021[1,0]: - ETA: 4:01:01 - loss: 0.1325 - auc: 0.7715Mon Sep 6 17:03:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:04:43 2021[1,0]:2282/4000 [================>.............] - ETA: 4:00:03 - loss: 0.1324 - auc: 0.7715Mon Sep 6 17:04:51 2021[1,0]:Mon Sep 6 17:04:17 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:05:44 2021[1,0]:2289/4000 [================>.............]Mon Sep 6 17:05:44 2021[1,0]: - ETA: 3:59:06 - loss: 0.1324 - auc: 0.7716Mon Sep 6 17:05:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:06:45 2021[1,0]:2296/4000 [================>.............]Mon Sep 6 17:06:45 2021[1,0]: - ETA: 3:58:09 - loss: 0.1324 - auc: 0.7716Mon Sep 6 17:06:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:07:46 2021[1,0]:2303/4000 [================>.............]Mon Sep 6 17:07:46 2021[1,0]: - ETA: 3:57:12 - loss: 0.1324 - auc: 0.7717Mon Sep 6 17:07:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:08:46 2021[1,0]:2310/4000 [================>.............]Mon Sep 6 17:08:46 2021[1,0]: - ETA: 3:56:14 - loss: 0.1324 - auc: 0.7717Mon Sep 6 17:08:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:09:46 2021[1,0]:2317/4000 [================>.............]Mon Sep 6 17:09:46 2021[1,0]: - ETA: 3:55:16 - loss: 0.1323 - auc: 0.7718Mon Sep 6 17:09:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:10:44 2021[1,0]:2324/4000 [================>.............]Mon Sep 6 17:10:44 2021[1,0]: - ETA: 3:54:17 - loss: 0.1323 - auc: 0.7718Mon Sep 6 17:10:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:11:42 2021[1,0]:2331/4000 [================>.............]Mon Sep 6 17:11:42 2021[1,0]: - ETA: 3:53:18 - loss: 0.1323 - auc: 0.7719Mon Sep 6 17:11:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:12:48 2021[1,0]:2339/4000 [================>.............]Mon Sep 6 17:12:48 2021[1,0]: - ETA: 3:52:11 - loss: 0.1323 - auc: 0.7719Mon Sep 6 17:12:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:13:48 2021[1,0]:2346/4000 [================>.............]Mon Sep 6 17:13:48 2021[1,0]: - ETA: 3:51:12 - loss: 0.1322 - auc: 0.7720Mon Sep 6 17:13:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2353/4000 [================>.............]Mon Sep 6 17:14:49 2021[1,0]: - ETA: 3:50:15 - loss: 0.1322 - auc: 0.7720Mon Sep 6 17:14:57 2021[1,0]:Mon Sep 6 17:14:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:15:48 2021[1,0]:2360/4000 [================>.............]Mon Sep 6 17:15:48 2021[1,0]: - ETA: 3:49:17 - loss: 0.1322 - auc: 0.7721Mon Sep 6 17:15:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:16:48 2021[1,0]:2367/4000 [================>.............]Mon Sep 6 17:16:48 2021[1,0]: - ETA: 3:48:19 - loss: 0.1322 - auc: 0.7721Mon Sep 6 17:16:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2374/4000 [================>.............]Mon Sep 6 17:17:48 2021[1,0]: - ETA: 3:47:21 - loss: 0.1322 - auc: 0.7722Mon Sep 6 17:17:56 2021[1,0]:Mon Sep 6 17:17:30 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:18:46 2021[1,0]:2381/4000 [================>.............]Mon Sep 6 17:18:46 2021[1,0]: - ETA: 3:46:22 - loss: 0.1321 - auc: 0.7722Mon Sep 6 17:18:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:19:44 2021[1,0]:2388/4000 [================>.............]Mon Sep 6 17:19:44 2021[1,0]: - ETA: 3:45:22 - loss: 0.1321 - auc: 0.7723Mon Sep 6 17:19:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:20:49 2021[1,0]:2396/4000 [================>.............]Mon Sep 6 17:20:49 2021[1,0]: - ETA: 3:44:14 - loss: 0.1321 - auc: 0.7723Mon Sep 6 17:20:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:21:46 2021[1,0]:2403/4000 [=================>............]Mon Sep 6 17:21:46 2021[1,0]: - ETA: 3:43:14 - loss: 0.1321 - auc: 0.7724Mon Sep 6 17:21:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:22:43 2021[1,0]:2410/4000 [=================>............]Mon Sep 6 17:22:43 2021[1,0]: - ETA: 3:42:14 - loss: 0.1320 - auc: 0.7724Mon Sep 6 17:22:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:23:50 2021[1,0]:2418/4000 [=================>............]Mon Sep 6 17:23:50 2021[1,0]: - ETA: 3:41:07 - loss: 0.1320 - auc: 0.7725Mon Sep 6 17:23:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:24:49 2021[1,0]:2425/4000 [=================>............]Mon Sep 6 17:24:49 2021[1,0]: - ETA: 3:40:09 - loss: 0.1320 - auc: 0.7725Mon Sep 6 17:24:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:25:49 2021[1,0]:2432/4000 [=================>............]Mon Sep 6 17:25:49 2021[1,0]: - ETA: 3:39:11 - loss: 0.1320 - auc: 0.7726Mon Sep 6 17:25:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:26:46 2021[1,0]:2439/4000 [=================>............]Mon Sep 6 17:26:46 2021[1,0]: - ETA: 3:38:11 - loss: 0.1320 - auc: 0.7726Mon Sep 6 17:26:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:27:50 2021[1,0]:2447/4000 [=================>............]Mon Sep 6 17:27:50 2021[1,0]: - ETA: 3:37:02 - loss: 0.1319 - auc: 0.7727Mon Sep 6 17:27:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2454/4000 [=================>............]Mon Sep 6 17:28:45 2021[1,0]: - ETA: 3:36:01 - loss: 0.1319 - auc: 0.7727Mon Sep 6 17:28:53 2021[1,0]:Mon Sep 6 17:28:45 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:29:51 2021[1,0]:2462/4000 [=================>............]Mon Sep 6 17:29:51 2021[1,0]: - ETA: 3:34:53 - loss: 0.1319 - auc: 0.7728Mon Sep 6 17:29:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:30:49 2021[1,0]:2469/4000 [=================>............]Mon Sep 6 17:30:49 2021[1,0]: - ETA: 3:33:54 - loss: 0.1319 - auc: 0.7728Mon Sep 6 17:30:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:31:48 2021[1,0]:2476/4000 [=================>............]Mon Sep 6 17:31:48 2021[1,0]: - ETA: 3:32:56 - loss: 0.1318 - auc: 0.7729Mon Sep 6 17:31:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:32:48 2021[1,0]:2483/4000 [=================>............]Mon Sep 6 17:32:48 2021[1,0]: - ETA: 3:31:58 - loss: 0.1318 - auc: 0.7729Mon Sep 6 17:32:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:33:48 2021[1,0]:2490/4000 [=================>............]Mon Sep 6 17:33:48 2021[1,0]: - ETA: 3:31:00 - loss: 0.1318 - auc: 0.7730Mon Sep 6 17:33:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:34:48 2021[1,0]:2497/4000 [=================>............]Mon Sep 6 17:34:48 2021[1,0]: - ETA: 3:30:02 - loss: 0.1318 - auc: 0.7730Mon Sep 6 17:34:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:35:48 2021[1,0]:2504/4000 [=================>............]Mon Sep 6 17:35:48 2021[1,0]: - ETA: 3:29:04 - loss: 0.1318 - auc: 0.7730Mon Sep 6 17:35:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:36:44 2021[1,0]:2511/4000 [=================>............]Mon Sep 6 17:36:44 2021[1,0]: - ETA: 3:28:04 - loss: 0.1317 - auc: 0.7731Mon Sep 6 17:36:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2519/4000 [=================>............]Mon Sep 6 17:37:47 2021[1,0]: - ETA: 3:26:54 - loss: 0.1317 - auc: 0.7731Mon Sep 6 17:37:55 2021[1,0]:Mon Sep 6 17:37:24 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:38:44 2021[1,0]:2526/4000 [=================>............]Mon Sep 6 17:38:44 2021[1,0]: - ETA: 3:25:55 - loss: 0.1317 - auc: 0.7732Mon Sep 6 17:38:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:39:44 2021[1,0]:2533/4000 [=================>............]Mon Sep 6 17:39:44 2021[1,0]: - ETA: 3:24:57 - loss: 0.1317 - auc: 0.7733Mon Sep 6 17:39:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2540/4000 [==================>...........]Mon Sep 6 17:40:44 2021[1,0]: - ETA: 3:23:59 - loss: 0.1317 - auc: 0.7733Mon Sep 6 17:40:53 2021[1,0]:Mon Sep 6 17:40:18 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2547/4000 [==================>...........]Mon Sep 6 17:41:44 2021[1,0]: - ETA: 3:23:01 - loss: 0.1316 - auc: 0.7733Mon Sep 6 17:41:53 2021[1,0]:Mon Sep 6 17:41:44 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2554/4000 [==================>...........]Mon Sep 6 17:42:43 2021[1,0]: - ETA: 3:22:03 - loss: 0.1316 - auc: 0.7734Mon Sep 6 17:42:52 2021[1,0]:Mon Sep 6 17:42:43 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:43:42 2021[1,0]:2561/4000 [==================>...........]Mon Sep 6 17:43:42 2021[1,0]: - ETA: 3:21:04 - loss: 0.1316 - auc: 0.7734Mon Sep 6 17:43:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:44:50 2021[1,0]:2569/4000 [==================>...........]Mon Sep 6 17:44:50 2021[1,0]: - ETA: 3:19:57 - loss: 0.1316 - auc: 0.7735Mon Sep 6 17:44:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:45:49 2021[1,0]:2576/4000 [==================>...........]Mon Sep 6 17:45:49 2021[1,0]: - ETA: 3:18:59 - loss: 0.1316 - auc: 0.7735Mon Sep 6 17:45:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:46:47 2021[1,0]:2583/4000 [==================>...........]Mon Sep 6 17:46:47 2021[1,0]: - ETA: 3:18:00 - loss: 0.1315 - auc: 0.7735Mon Sep 6 17:46:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:47:46 2021[1,0]:2590/4000 [==================>...........]Mon Sep 6 17:47:46 2021[1,0]: - ETA: 3:17:01 - loss: 0.1315 - auc: 0.7736Mon Sep 6 17:47:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:48:44 2021[1,0]:2597/4000 [==================>...........]Mon Sep 6 17:48:44 2021[1,0]: - ETA: 3:16:02 - loss: 0.1315 - auc: 0.7736Mon Sep 6 17:48:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:49:43 2021[1,0]:2604/4000 [==================>...........]Mon Sep 6 17:49:43 2021[1,0]: - ETA: 3:15:03 - loss: 0.1315 - auc: 0.7737Mon Sep 6 17:49:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:50:50 2021[1,0]:2612/4000 [==================>...........]Mon Sep 6 17:50:50 2021[1,0]: - ETA: 3:13:57 - loss: 0.1315 - auc: 0.7737Mon Sep 6 17:50:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:51:49 2021[1,0]:2619/4000 [==================>...........]Mon Sep 6 17:51:49 2021[1,0]: - ETA: 3:12:58 - loss: 0.1314 - auc: 0.7738Mon Sep 6 17:51:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:52:49 2021[1,0]:2626/4000 [==================>...........]Mon Sep 6 17:52:49 2021[1,0]: - ETA: 3:12:00 - loss: 0.1314 - auc: 0.7738Mon Sep 6 17:52:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:53:48 2021[1,0]:2633/4000 [==================>...........]Mon Sep 6 17:53:48 2021[1,0]: - ETA: 3:11:01 - loss: 0.1314 - auc: 0.7739Mon Sep 6 17:53:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:54:47 2021[1,0]:2640/4000 [==================>...........]Mon Sep 6 17:54:47 2021[1,0]: - ETA: 3:10:03 - loss: 0.1314 - auc: 0.7739Mon Sep 6 17:54:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2647/4000 [==================>...........]Mon Sep 6 17:55:47 2021[1,0]: - ETA: 3:09:05 - loss: 0.1314 - auc: 0.7740Mon Sep 6 17:55:56 2021[1,0]:Mon Sep 6 17:55:30 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:56:47 2021[1,0]:2654/4000 [==================>...........]Mon Sep 6 17:56:47 2021[1,0]: - ETA: 3:08:07 - loss: 0.1313 - auc: 0.7740Mon Sep 6 17:56:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:57:46 2021[1,0]:2661/4000 [==================>...........]Mon Sep 6 17:57:46 2021[1,0]: - ETA: 3:07:08 - loss: 0.1313 - auc: 0.7740Mon Sep 6 17:57:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:58:46 2021[1,0]:2668/4000 [===================>..........]Mon Sep 6 17:58:46 2021[1,0]: - ETA: 3:06:10 - loss: 0.1313 - auc: 0.7741Mon Sep 6 17:58:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:59:51 2021[1,0]:2676/4000 [===================>..........]Mon Sep 6 17:59:51 2021[1,0]: - ETA: 3:05:02 - loss: 0.1313 - auc: 0.7741Mon Sep 6 17:59:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:00:49 2021[1,0]:2683/4000 [===================>..........]Mon Sep 6 18:00:49 2021[1,0]: - ETA: 3:04:03 - loss: 0.1313 - auc: 0.7742Mon Sep 6 18:00:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:01:49 2021[1,0]:2690/4000 [===================>..........]Mon Sep 6 18:01:49 2021[1,0]: - ETA: 3:03:05 - loss: 0.1313 - auc: 0.7742Mon Sep 6 18:01:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2697/4000 [===================>..........]Mon Sep 6 18:02:48 2021[1,0]: - ETA: 3:02:06 - loss: 0.1313 - auc: 0.7742Mon Sep 6 18:02:56 2021[1,0]:Mon Sep 6 18:02:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:03:48 2021[1,0]:2704/4000 [===================>..........]Mon Sep 6 18:03:48 2021[1,0]: - ETA: 3:01:09 - loss: 0.1312 - auc: 0.7743Mon Sep 6 18:03:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2711/4000 [===================>..........]Mon Sep 6 18:04:48 2021[1,0]: - ETA: 3:00:10 - loss: 0.1312 - auc: 0.7743Mon Sep 6 18:04:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:05:47 2021[1,0]:2718/4000 [===================>..........]Mon Sep 6 18:05:47 2021[1,0]: - ETA: 2:59:12 - loss: 0.1312 - auc: 0.7744Mon Sep 6 18:05:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:06:45 2021[1,0]:2725/4000 [===================>..........]Mon Sep 6 18:06:45 2021[1,0]: - ETA: 2:58:13 - loss: 0.1312 - auc: 0.7744Mon Sep 6 18:06:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:07:51 2021[1,0]:2733/4000 [===================>..........]Mon Sep 6 18:07:51 2021[1,0]: - ETA: 2:57:05 - loss: 0.1312 - auc: 0.7745Mon Sep 6 18:07:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:08:48 2021[1,0]:2740/4000 [===================>..........]Mon Sep 6 18:08:48 2021[1,0]: - ETA: 2:56:06 - loss: 0.1312 - auc: 0.7745Mon Sep 6 18:08:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:09:46 2021[1,0]:2747/4000 [===================>..........]Mon Sep 6 18:09:46 2021[1,0]: - ETA: 2:55:07 - loss: 0.1311 - auc: 0.7745Mon Sep 6 18:09:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2754/4000 [===================>..........]Mon Sep 6 18:10:44 2021[1,0]: - ETA: 2:54:08 - loss: 0.1311 - auc: 0.7746Mon Sep 6 18:10:52 2021[1,0]:Mon Sep 6 18:10:28 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:11:50 2021[1,0]:2762/4000 [===================>..........]Mon Sep 6 18:11:50 2021[1,0]: - ETA: 2:53:00 - loss: 0.1311 - auc: 0.7746Mon Sep 6 18:11:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:12:49 2021[1,0]:2769/4000 [===================>..........]Mon Sep 6 18:12:49 2021[1,0]: - ETA: 2:52:01 - loss: 0.1311 - auc: 0.7747Mon Sep 6 18:12:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:13:48 2021[1,0]:2776/4000 [===================>..........]Mon Sep 6 18:13:48 2021[1,0]: - ETA: 2:51:03 - loss: 0.1311 - auc: 0.7747Mon Sep 6 18:13:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:14:47 2021[1,0]:2783/4000 [===================>..........]Mon Sep 6 18:14:47 2021[1,0]: - ETA: 2:50:04 - loss: 0.1310 - auc: 0.7748Mon Sep 6 18:14:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:15:47 2021[1,0]:2790/4000 [===================>..........]Mon Sep 6 18:15:47 2021[1,0]: - ETA: 2:49:06 - loss: 0.1310 - auc: 0.7748Mon Sep 6 18:15:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:16:48 2021[1,0]:2797/4000 [===================>..........]Mon Sep 6 18:16:48 2021[1,0]: - ETA: 2:48:08 - loss: 0.1310 - auc: 0.7748Mon Sep 6 18:16:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2804/4000 [====================>.........]Mon Sep 6 18:17:47 2021[1,0]: - ETA: 2:47:10 - loss: 0.1310 - auc: 0.7749Mon Sep 6 18:17:56 2021[1,0]:Mon Sep 6 18:17:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:18:46 2021[1,0]:2811/4000 [====================>.........]Mon Sep 6 18:18:46 2021[1,0]: - ETA: 2:46:11 - loss: 0.1310 - auc: 0.7749Mon Sep 6 18:18:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2818/4000 [====================>.........]Mon Sep 6 18:19:47 2021[1,0]: - ETA: 2:45:14 - loss: 0.1310 - auc: 0.7749Mon Sep 6 18:19:56 2021[1,0]:Mon Sep 6 18:19:38 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:20:47 2021[1,0]:2825/4000 [====================>.........]Mon Sep 6 18:20:47 2021[1,0]: - ETA: 2:44:15 - loss: 0.1310 - auc: 0.7750Mon Sep 6 18:20:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:21:46 2021[1,0]:2832/4000 [====================>.........]Mon Sep 6 18:21:46 2021[1,0]: - ETA: 2:43:17 - loss: 0.1309 - auc: 0.7750Mon Sep 6 18:21:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2840/4000 [====================>.........]Mon Sep 6 18:22:52 2021[1,0]: - ETA: 2:42:09 - loss: 0.1309 - auc: 0.7751Mon Sep 6 18:23:00 2021[1,0]:Mon Sep 6 18:22:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2847/4000 [====================>.........]Mon Sep 6 18:23:51 2021[1,0]: - ETA: 2:41:11 - loss: 0.1309 - auc: 0.7751Mon Sep 6 18:24:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:24:44 2021[1,0]:2853/4000 [====================>.........]Mon Sep 6 18:24:44 2021[1,0]: - ETA: 2:40:21 - loss: 0.1309 - auc: 0.7751Mon Sep 6 18:24:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:25:45 2021[1,0]:2860/4000 [====================>.........]Mon Sep 6 18:25:45 2021[1,0]: - ETA: 2:39:23 - loss: 0.1309 - auc: 0.7752Mon Sep 6 18:25:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:26:45 2021[1,0]:2867/4000 [====================>.........]Mon Sep 6 18:26:45 2021[1,0]: - ETA: 2:38:25 - loss: 0.1309 - auc: 0.7752Mon Sep 6 18:26:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:27:45 2021[1,0]:2874/4000 [====================>.........]Mon Sep 6 18:27:45 2021[1,0]: - ETA: 2:37:27 - loss: 0.1308 - auc: 0.7752Mon Sep 6 18:27:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:28:45 2021[1,0]:2881/4000 [====================>.........]Mon Sep 6 18:28:45 2021[1,0]: - ETA: 2:36:29 - loss: 0.1308 - auc: 0.7753Mon Sep 6 18:28:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:29:44 2021[1,0]:2888/4000 [====================>.........]Mon Sep 6 18:29:44 2021[1,0]: - ETA: 2:35:30 - loss: 0.1308 - auc: 0.7753Mon Sep 6 18:29:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:30:52 2021[1,0]:2896/4000 [====================>.........]Mon Sep 6 18:30:52 2021[1,0]: - ETA: 2:34:23 - loss: 0.1308 - auc: 0.7754Mon Sep 6 18:31:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:31:50 2021[1,0]:2903/4000 [====================>.........]Mon Sep 6 18:31:50 2021[1,0]: - ETA: 2:33:24 - loss: 0.1308 - auc: 0.7754Mon Sep 6 18:31:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2910/4000 [====================>.........]Mon Sep 6 18:32:48 2021[1,0]: - ETA: 2:32:25 - loss: 0.1308 - auc: 0.7754Mon Sep 6 18:32:56 2021[1,0]:Mon Sep 6 18:32:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:33:47 2021[1,0]:2917/4000 [====================>.........]Mon Sep 6 18:33:47 2021[1,0]: - ETA: 2:31:27 - loss: 0.1308 - auc: 0.7755Mon Sep 6 18:33:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:34:45 2021[1,0]:2924/4000 [====================>.........]Mon Sep 6 18:34:45 2021[1,0]: - ETA: 2:30:28 - loss: 0.1307 - auc: 0.7755Mon Sep 6 18:34:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:35:45 2021[1,0]:2931/4000 [====================>.........]Mon Sep 6 18:35:45 2021[1,0]: - ETA: 2:29:30 - loss: 0.1307 - auc: 0.7755Mon Sep 6 18:35:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2938/4000 [=====================>........]Mon Sep 6 18:36:46 2021[1,0]: - ETA: 2:28:32 - loss: 0.1307 - auc: 0.7756Mon Sep 6 18:36:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:37:45 2021[1,0]:2945/4000 [=====================>........]Mon Sep 6 18:37:45 2021[1,0]: - ETA: 2:27:33 - loss: 0.1307 - auc: 0.7756Mon Sep 6 18:37:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:38:46 2021[1,0]:2952/4000 [=====================>........]Mon Sep 6 18:38:46 2021[1,0]: - ETA: 2:26:35 - loss: 0.1307 - auc: 0.7756Mon Sep 6 18:38:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:39:45 2021[1,0]:2959/4000 [=====================>........]Mon Sep 6 18:39:45 2021[1,0]: - ETA: 2:25:36 - loss: 0.1307 - auc: 0.7757Mon Sep 6 18:39:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2966/4000 [=====================>........]Mon Sep 6 18:40:45 2021[1,0]: - ETA: 2:24:38 - loss: 0.1307 - auc: 0.7757Mon Sep 6 18:40:54 2021[1,0]:Mon Sep 6 18:40:36 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:41:45 2021[1,0]:2973/4000 [=====================>........] - ETA: 2:23:40 - loss: 0.1306 - auc: 0.7757Mon Sep 6 18:41:54 2021[1,0]:Mon Sep 6 18:41:28 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:42:44 2021[1,0]:2980/4000 [=====================>........]Mon Sep 6 18:42:44 2021[1,0]: - ETA: 2:22:41 - loss: 0.1306 - auc: 0.7758Mon Sep 6 18:42:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:43:51 2021[1,0]:2988/4000 [=====================>........]Mon Sep 6 18:43:51 2021[1,0]: - ETA: 2:21:34 - loss: 0.1306 - auc: 0.7758Mon Sep 6 18:43:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2995/4000 [=====================>........]Mon Sep 6 18:44:49 2021[1,0]: - ETA: 2:20:35 - loss: 0.1306 - auc: 0.7758Mon Sep 6 18:44:58 2021[1,0]:Mon Sep 6 18:44:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:45:48 2021[1,0]:3002/4000 [=====================>........]Mon Sep 6 18:45:48 2021[1,0]: - ETA: 2:19:36 - loss: 0.1306 - auc: 0.7759Mon Sep 6 18:45:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:46:48 2021[1,0]:3009/4000 [=====================>........]Mon Sep 6 18:46:48 2021[1,0]: - ETA: 2:18:38 - loss: 0.1306 - auc: 0.7759Mon Sep 6 18:46:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:47:48 2021[1,0]:3016/4000 [=====================>........]Mon Sep 6 18:47:48 2021[1,0]: - ETA: 2:17:39 - loss: 0.1306 - auc: 0.7759Mon Sep 6 18:47:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3023/4000 [=====================>........]Mon Sep 6 18:48:48 2021[1,0]: - ETA: 2:16:41 - loss: 0.1305 - auc: 0.7759Mon Sep 6 18:48:56 2021[1,0]:Mon Sep 6 18:48:30 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:49:46 2021[1,0]:3030/4000 [=====================>........]Mon Sep 6 18:49:46 2021[1,0]: - ETA: 2:15:42 - loss: 0.1305 - auc: 0.7760Mon Sep 6 18:49:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:50:45 2021[1,0]:3037/4000 [=====================>........]Mon Sep 6 18:50:45 2021[1,0]: - ETA: 2:14:43 - loss: 0.1305 - auc: 0.7760Mon Sep 6 18:50:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3044/4000 [=====================>........]Mon Sep 6 18:51:45 2021[1,0]: - ETA: 2:13:45 - loss: 0.1305 - auc: 0.7760Mon Sep 6 18:51:53 2021[1,0]:Mon Sep 6 18:51:45 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:52:45 2021[1,0]:3051/4000 [=====================>........]Mon Sep 6 18:52:45 2021[1,0]: - ETA: 2:12:47 - loss: 0.1305 - auc: 0.7761Mon Sep 6 18:52:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:53:45 2021[1,0]:3058/4000 [=====================>........]Mon Sep 6 18:53:45 2021[1,0]: - ETA: 2:11:48 - loss: 0.1305 - auc: 0.7761Mon Sep 6 18:53:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:54:46 2021[1,0]:3065/4000 [=====================>........]Mon Sep 6 18:54:46 2021[1,0]: - ETA: 2:10:50 - loss: 0.1305 - auc: 0.7761Mon Sep 6 18:54:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:55:46 2021[1,0]:3072/4000 [======================>.......]Mon Sep 6 18:55:46 2021[1,0]: - ETA: 2:09:52 - loss: 0.1305 - auc: 0.7762Mon Sep 6 18:55:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:56:45 2021[1,0]:3079/4000 [======================>.......]Mon Sep 6 18:56:45 2021[1,0]: - ETA: 2:08:53 - loss: 0.1304 - auc: 0.7762Mon Sep 6 18:56:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:57:45 2021[1,0]:3086/4000 [======================>.......]Mon Sep 6 18:57:45 2021[1,0]: - ETA: 2:07:55 - loss: 0.1304 - auc: 0.7762Mon Sep 6 18:57:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:58:46 2021[1,0]:3093/4000 [======================>.......]Mon Sep 6 18:58:46 2021[1,0]: - ETA: 2:06:56 - loss: 0.1304 - auc: 0.7763Mon Sep 6 18:58:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:59:46 2021[1,0]:3100/4000 [======================>.......]Mon Sep 6 18:59:46 2021[1,0]: - ETA: 2:05:58 - loss: 0.1304 - auc: 0.7763Mon Sep 6 18:59:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3107/4000 [======================>.......]Mon Sep 6 19:00:47 2021[1,0]: - ETA: 2:05:00 - loss: 0.1304 - auc: 0.7763Mon Sep 6 19:00:56 2021[1,0]:Mon Sep 6 19:00:39 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:01:49 2021[1,0]:3114/4000 [======================>.......]Mon Sep 6 19:01:49 2021[1,0]: - ETA: 2:04:02 - loss: 0.1304 - auc: 0.7763Mon Sep 6 19:01:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:02:50 2021[1,0]:3121/4000 [======================>.......]Mon Sep 6 19:02:50 2021[1,0]: - ETA: 2:03:04 - loss: 0.1304 - auc: 0.7764Mon Sep 6 19:02:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:03:51 2021[1,0]:3128/4000 [======================>.......]Mon Sep 6 19:03:51 2021[1,0]: - ETA: 2:02:06 - loss: 0.1304 - auc: 0.7764Mon Sep 6 19:04:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:04:52 2021[1,0]:3135/4000 [======================>.......]Mon Sep 6 19:04:52 2021[1,0]: - ETA: 2:01:07 - loss: 0.1303 - auc: 0.7764Mon Sep 6 19:05:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:05:44 2021[1,0]:3141/4000 [======================>.......]Mon Sep 6 19:05:44 2021[1,0]: - ETA: 2:00:17 - loss: 0.1303 - auc: 0.7764Mon Sep 6 19:05:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:06:45 2021[1,0]:3148/4000 [======================>.......]Mon Sep 6 19:06:45 2021[1,0]: - ETA: 1:59:19 - loss: 0.1303 - auc: 0.7765Mon Sep 6 19:06:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:07:46 2021[1,0]:3155/4000 [======================>.......]Mon Sep 6 19:07:46 2021[1,0]: - ETA: 1:58:21 - loss: 0.1303 - auc: 0.7765Mon Sep 6 19:07:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:08:47 2021[1,0]:3162/4000 [======================>.......]Mon Sep 6 19:08:47 2021[1,0]: - ETA: 1:57:23 - loss: 0.1303 - auc: 0.7765Mon Sep 6 19:08:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3169/4000 [======================>.......]Mon Sep 6 19:09:49 2021[1,0]: - ETA: 1:56:25 - loss: 0.1303 - auc: 0.7766Mon Sep 6 19:09:57 2021[1,0]:Mon Sep 6 19:09:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:10:49 2021[1,0]:3176/4000 [======================>.......]Mon Sep 6 19:10:49 2021[1,0]: - ETA: 1:55:26 - loss: 0.1303 - auc: 0.7766Mon Sep 6 19:10:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:11:51 2021[1,0]:3183/4000 [======================>.......]Mon Sep 6 19:11:51 2021[1,0]: - ETA: 1:54:28 - loss: 0.1303 - auc: 0.7766Mon Sep 6 19:12:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:12:51 2021[1,0]:3190/4000 [======================>.......] - ETA: 1:53:30 - loss: 0.1303 - auc: 0.7766Mon Sep 6 19:13:00 2021[1,0]:Mon Sep 6 19:12:43 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:13:52 2021[1,0]:3197/4000 [======================>.......]Mon Sep 6 19:13:52 2021[1,0]: - ETA: 1:52:31 - loss: 0.1302 - auc: 0.7767Mon Sep 6 19:14:01 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:14:52 2021[1,0]:3204/4000 [=======================>......]Mon Sep 6 19:14:52 2021[1,0]: - ETA: 1:51:33 - loss: 0.1302 - auc: 0.7767Mon Sep 6 19:15:01 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:15:44 2021[1,0]:3210/4000 [=======================>......]Mon Sep 6 19:15:44 2021[1,0]: - ETA: 1:50:43 - loss: 0.1302 - auc: 0.7767Mon Sep 6 19:15:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:16:45 2021[1,0]:3217/4000 [=======================>......]Mon Sep 6 19:16:45 2021[1,0]: - ETA: 1:49:44 - loss: 0.1302 - auc: 0.7767Mon Sep 6 19:16:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3224/4000 [=======================>......]Mon Sep 6 19:17:45 2021[1,0]: - ETA: 1:48:46 - loss: 0.1302 - auc: 0.7768Mon Sep 6 19:17:53 2021[1,0]:Mon Sep 6 19:17:36 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:18:44 2021[1,0]:3231/4000 [=======================>......]Mon Sep 6 19:18:44 2021[1,0]: - ETA: 1:47:47 - loss: 0.1302 - auc: 0.7768Mon Sep 6 19:18:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:19:53 2021[1,0]:3239/4000 [=======================>......]Mon Sep 6 19:19:53 2021[1,0]: - ETA: 1:46:40 - loss: 0.1302 - auc: 0.7768Mon Sep 6 19:20:01 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3245/4000 [=======================>......]Mon Sep 6 19:20:45 2021[1,0]: - ETA: 1:45:50 - loss: 0.1302 - auc: 0.7768Mon Sep 6 19:20:53 2021[1,0]:Mon Sep 6 19:20:36 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:21:46 2021[1,0]:3252/4000 [=======================>......]Mon Sep 6 19:21:46 2021[1,0]: - ETA: 1:44:51 - loss: 0.1302 - auc: 0.7769Mon Sep 6 19:21:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:22:45 2021[1,0]:3259/4000 [=======================>......]Mon Sep 6 19:22:45 2021[1,0]: - ETA: 1:43:53 - loss: 0.1301 - auc: 0.7769Mon Sep 6 19:22:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:23:53 2021[1,0]:3267/4000 [=======================>......]Mon Sep 6 19:23:53 2021[1,0]: - ETA: 1:42:45 - loss: 0.1301 - auc: 0.7769Mon Sep 6 19:24:01 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:24:52 2021[1,0]:3274/4000 [=======================>......]Mon Sep 6 19:24:52 2021[1,0]: - ETA: 1:41:47 - loss: 0.1301 - auc: 0.7770Mon Sep 6 19:25:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3281/4000 [=======================>......]Mon Sep 6 19:25:51 2021[1,0]: - ETA: 1:40:48 - loss: 0.1301 - auc: 0.7770Mon Sep 6 19:25:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:26:50 2021[1,0]:3288/4000 [=======================>......]Mon Sep 6 19:26:50 2021[1,0]: - ETA: 1:39:49 - loss: 0.1301 - auc: 0.7770Mon Sep 6 19:26:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:27:49 2021[1,0]:3295/4000 [=======================>......]Mon Sep 6 19:27:49 2021[1,0]: - ETA: 1:38:50 - loss: 0.1301 - auc: 0.7770Mon Sep 6 19:27:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:28:48 2021[1,0]:3302/4000 [=======================>......]Mon Sep 6 19:28:48 2021[1,0]: - ETA: 1:37:51 - loss: 0.1301 - auc: 0.7771Mon Sep 6 19:28:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:29:47 2021[1,0]:3309/4000 [=======================>......]Mon Sep 6 19:29:47 2021[1,0]: - ETA: 1:36:52 - loss: 0.1301 - auc: 0.7771Mon Sep 6 19:29:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:30:47 2021[1,0]:3316/4000 [=======================>......] - ETA: 1:35:54 - loss: 0.1301 - auc: 0.7771Mon Sep 6 19:30:55 2021[1,0]:Mon Sep 6 19:30:38 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:31:47 2021[1,0]:3323/4000 [=======================>......]Mon Sep 6 19:31:47 2021[1,0]: - ETA: 1:34:55 - loss: 0.1300 - auc: 0.7771Mon Sep 6 19:31:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:32:46 2021[1,0]:3330/4000 [=======================>......]Mon Sep 6 19:32:46 2021[1,0]: - ETA: 1:33:56 - loss: 0.1300 - auc: 0.7772Mon Sep 6 19:32:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:33:46 2021[1,0]:3337/4000 [========================>.....]Mon Sep 6 19:33:46 2021[1,0]: - ETA: 1:32:58 - loss: 0.1300 - auc: 0.7772Mon Sep 6 19:33:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3344/4000 [========================>.....]Mon Sep 6 19:34:46 2021[1,0]: - ETA: 1:31:59 - loss: 0.1300 - auc: 0.7772Mon Sep 6 19:34:54 2021[1,0]:Mon Sep 6 19:34:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3351/4000 [========================>.....]Mon Sep 6 19:35:46 2021[1,0]: - ETA: 1:31:00 - loss: 0.1300 - auc: 0.7773Mon Sep 6 19:35:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:36:47 2021[1,0]:3358/4000 [========================>.....]Mon Sep 6 19:36:47 2021[1,0]: - ETA: 1:30:02 - loss: 0.1300 - auc: 0.7773Mon Sep 6 19:36:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:37:47 2021[1,0]:3365/4000 [========================>.....]Mon Sep 6 19:37:47 2021[1,0]: - ETA: 1:29:03 - loss: 0.1300 - auc: 0.7773Mon Sep 6 19:37:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:38:46 2021[1,0]:3372/4000 [========================>.....]Mon Sep 6 19:38:46 2021[1,0]: - ETA: 1:28:04 - loss: 0.1300 - auc: 0.7773Mon Sep 6 19:38:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:39:46 2021[1,0]:3379/4000 [========================>.....]Mon Sep 6 19:39:46 2021[1,0]: - ETA: 1:27:05 - loss: 0.1300 - auc: 0.7774Mon Sep 6 19:39:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:40:46 2021[1,0]:3386/4000 [========================>.....]Mon Sep 6 19:40:46 2021[1,0]: - ETA: 1:26:07 - loss: 0.1300 - auc: 0.7774Mon Sep 6 19:40:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:41:46 2021[1,0]:3393/4000 [========================>.....]Mon Sep 6 19:41:46 2021[1,0]: - ETA: 1:25:08 - loss: 0.1299 - auc: 0.7774Mon Sep 6 19:41:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:42:46 2021[1,0]:3400/4000 [========================>.....]Mon Sep 6 19:42:46 2021[1,0]: - ETA: 1:24:09 - loss: 0.1299 - auc: 0.7774Mon Sep 6 19:42:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:43:46 2021[1,0]:3407/4000 [========================>.....]Mon Sep 6 19:43:46 2021[1,0]: - ETA: 1:23:10 - loss: 0.1299 - auc: 0.7774Mon Sep 6 19:43:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:44:46 2021[1,0]:3414/4000 [========================>.....]Mon Sep 6 19:44:46 2021[1,0]: - ETA: 1:22:12 - loss: 0.1299 - auc: 0.7775Mon Sep 6 19:44:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:45:53 2021[1,0]:3422/4000 [========================>.....]Mon Sep 6 19:45:53 2021[1,0]: - ETA: 1:21:04 - loss: 0.1299 - auc: 0.7775Mon Sep 6 19:46:01 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:46:52 2021[1,0]:3429/4000 [========================>.....]Mon Sep 6 19:46:52 2021[1,0]: - ETA: 1:20:05 - loss: 0.1299 - auc: 0.7775Mon Sep 6 19:47:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:47:52 2021[1,0]:3436/4000 [========================>.....]Mon Sep 6 19:47:52 2021[1,0]: - ETA: 1:19:07 - loss: 0.1299 - auc: 0.7775Mon Sep 6 19:48:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:48:51 2021[1,0]:3443/4000 [========================>.....]Mon Sep 6 19:48:51 2021[1,0]: - ETA: 1:18:08 - loss: 0.1299 - auc: 0.7776Mon Sep 6 19:48:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3450/4000 [========================>.....]Mon Sep 6 19:49:49 2021[1,0]: - ETA: 1:17:09 - loss: 0.1299 - auc: 0.7776Mon Sep 6 19:49:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3457/4000 [========================>.....]Mon Sep 6 19:50:47 2021[1,0]: - ETA: 1:16:10 - loss: 0.1298 - auc: 0.7776Mon Sep 6 19:50:55 2021[1,0]:Mon Sep 6 19:50:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:51:52 2021[1,0]:3465/4000 [========================>.....]Mon Sep 6 19:51:52 2021[1,0]: - ETA: 1:15:02 - loss: 0.1298 - auc: 0.7777Mon Sep 6 19:52:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3472/4000 [=========================>....]Mon Sep 6 19:52:50 2021[1,0]: - ETA: 1:14:03 - loss: 0.1298 - auc: 0.7777Mon Sep 6 19:52:58 2021[1,0]:Mon Sep 6 19:52:41 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3479/4000 [=========================>....]Mon Sep 6 19:53:47 2021[1,0]: - ETA: 1:13:04 - loss: 0.1298 - auc: 0.7777Mon Sep 6 19:53:55 2021[1,0]:Mon Sep 6 19:53:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:54:52 2021[1,0]:3487/4000 [=========================>....]Mon Sep 6 19:54:52 2021[1,0]: - ETA: 1:11:56 - loss: 0.1298 - auc: 0.7777Mon Sep 6 19:55:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:55:50 2021[1,0]:3494/4000 [=========================>....]Mon Sep 6 19:55:50 2021[1,0]: - ETA: 1:10:57 - loss: 0.1298 - auc: 0.7777Mon Sep 6 19:55:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:56:48 2021[1,0]:3501/4000 [=========================>....]Mon Sep 6 19:56:48 2021[1,0]: - ETA: 1:09:58 - loss: 0.1298 - auc: 0.7778Mon Sep 6 19:56:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:57:47 2021[1,0]:3508/4000 [=========================>....]Mon Sep 6 19:57:47 2021[1,0]: - ETA: 1:08:59 - loss: 0.1298 - auc: 0.7778Mon Sep 6 19:57:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:58:46 2021[1,0]:3515/4000 [=========================>....]Mon Sep 6 19:58:46 2021[1,0]: - ETA: 1:08:00 - loss: 0.1298 - auc: 0.7778Mon Sep 6 19:58:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:59:52 2021[1,0]:3523/4000 [=========================>....]Mon Sep 6 19:59:52 2021[1,0]: - ETA: 1:06:53 - loss: 0.1298 - auc: 0.7778Mon Sep 6 20:00:01 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:00:49 2021[1,0]:3530/4000 [=========================>....]Mon Sep 6 20:00:49 2021[1,0]: - ETA: 1:05:54 - loss: 0.1297 - auc: 0.7779Mon Sep 6 20:00:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:01:48 2021[1,0]:3537/4000 [=========================>....]Mon Sep 6 20:01:48 2021[1,0]: - ETA: 1:04:55 - loss: 0.1297 - auc: 0.7779Mon Sep 6 20:01:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:02:47 2021[1,0]:3544/4000 [=========================>....]Mon Sep 6 20:02:47 2021[1,0]: - ETA: 1:03:56 - loss: 0.1297 - auc: 0.7779Mon Sep 6 20:02:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:03:53 2021[1,0]:3552/4000 [=========================>....]Mon Sep 6 20:03:53 2021[1,0]: - ETA: 1:02:48 - loss: 0.1297 - auc: 0.7780Mon Sep 6 20:04:01 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:04:52 2021[1,0]:3559/4000 [=========================>....]Mon Sep 6 20:04:52 2021[1,0]: - ETA: 1:01:50 - loss: 0.1297 - auc: 0.7780Mon Sep 6 20:05:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:05:52 2021[1,0]:3566/4000 [=========================>....]Mon Sep 6 20:05:52 2021[1,0]: - ETA: 1:00:51 - loss: 0.1297 - auc: 0.7780Mon Sep 6 20:06:01 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:06:52 2021[1,0]:3573/4000 [=========================>....]Mon Sep 6 20:06:52 2021[1,0]: - ETA: 59:52 - loss: 0.1297 - auc: 0.7780 Mon Sep 6 20:07:01 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3580/4000 [=========================>....]Mon Sep 6 20:07:52 2021[1,0]: - ETA: 58:53 - loss: 0.1297 - auc: 0.7780Mon Sep 6 20:08:01 2021[1,0]:Mon Sep 6 20:07:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3587/4000 [=========================>....]Mon Sep 6 20:08:49 2021[1,0]: - ETA: 57:54 - loss: 0.1297 - auc: 0.7781Mon Sep 6 20:08:57 2021[1,0]:Mon Sep 6 20:08:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:09:46 2021[1,0]:3594/4000 [=========================>....]Mon Sep 6 20:09:46 2021[1,0]: - ETA: 56:55 - loss: 0.1297 - auc: 0.7781Mon Sep 6 20:09:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3602/4000 [==========================>...]Mon Sep 6 20:10:52 2021[1,0]: - ETA: 55:48 - loss: 0.1296 - auc: 0.7781Mon Sep 6 20:11:01 2021[1,0]:Mon Sep 6 20:10:36 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3609/4000 [==========================>...]Mon Sep 6 20:11:51 2021[1,0]: - ETA: 54:49 - loss: 0.1296 - auc: 0.7782Mon Sep 6 20:11:59 2021[1,0]:Mon Sep 6 20:11:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:12:49 2021[1,0]:3616/4000 [==========================>...]Mon Sep 6 20:12:49 2021[1,0]: - ETA: 53:50 - loss: 0.1296 - auc: 0.7782Mon Sep 6 20:12:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:13:49 2021[1,0]:3623/4000 [==========================>...]Mon Sep 6 20:13:49 2021[1,0]: - ETA: 52:51 - loss: 0.1296 - auc: 0.7782Mon Sep 6 20:13:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3630/4000 [==========================>...]Mon Sep 6 20:14:48 2021[1,0]: - ETA: 51:52 - loss: 0.1296 - auc: 0.7782Mon Sep 6 20:14:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:15:49 2021[1,0]:3637/4000 [==========================>...]Mon Sep 6 20:15:49 2021[1,0]: - ETA: 50:53 - loss: 0.1296 - auc: 0.7782Mon Sep 6 20:15:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:16:49 2021[1,0]:3644/4000 [==========================>...]Mon Sep 6 20:16:49 2021[1,0]: - ETA: 49:55 - loss: 0.1296 - auc: 0.7783Mon Sep 6 20:16:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3651/4000 [==========================>...]Mon Sep 6 20:17:49 2021[1,0]: - ETA: 48:56 - loss: 0.1296 - auc: 0.7783Mon Sep 6 20:17:58 2021[1,0]:Mon Sep 6 20:17:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3658/4000 [==========================>...]Mon Sep 6 20:18:50 2021[1,0]: - ETA: 47:57 - loss: 0.1296 - auc: 0.7783Mon Sep 6 20:18:59 2021[1,0]:Mon Sep 6 20:18:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:19:50 2021[1,0]:3665/4000 [==========================>...]Mon Sep 6 20:19:50 2021[1,0]: - ETA: 46:58 - loss: 0.1296 - auc: 0.7783Mon Sep 6 20:19:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:20:49 2021[1,0]:3672/4000 [==========================>...]Mon Sep 6 20:20:49 2021[1,0]: - ETA: 45:59 - loss: 0.1296 - auc: 0.7784Mon Sep 6 20:20:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:21:49 2021[1,0]:3679/4000 [==========================>...]Mon Sep 6 20:21:49 2021[1,0]: - ETA: 45:01 - loss: 0.1295 - auc: 0.7784Mon Sep 6 20:21:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:22:49 2021[1,0]:3686/4000 [==========================>...]Mon Sep 6 20:22:49 2021[1,0]: - ETA: 44:02 - loss: 0.1295 - auc: 0.7784Mon Sep 6 20:22:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3693/4000 [==========================>...]Mon Sep 6 20:23:49 2021[1,0]: - ETA: 43:03 - loss: 0.1295 - auc: 0.7784Mon Sep 6 20:23:58 2021[1,0]:Mon Sep 6 20:23:41 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:24:50 2021[1,0]:3700/4000 [==========================>...]Mon Sep 6 20:24:50 2021[1,0]: - ETA: 42:04 - loss: 0.1295 - auc: 0.7785Mon Sep 6 20:24:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3707/4000 [==========================>...]Mon Sep 6 20:25:49 2021[1,0]: - ETA: 41:05 - loss: 0.1295 - auc: 0.7785Mon Sep 6 20:25:58 2021[1,0]:Mon Sep 6 20:25:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:26:48 2021[1,0]:3714/4000 [==========================>...]Mon Sep 6 20:26:48 2021[1,0]: - ETA: 40:07 - loss: 0.1295 - auc: 0.7785Mon Sep 6 20:26:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:27:46 2021[1,0]:3721/4000 [==========================>...]Mon Sep 6 20:27:46 2021[1,0]: - ETA: 39:07 - loss: 0.1295 - auc: 0.7785Mon Sep 6 20:27:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:28:54 2021[1,0]:3729/4000 [==========================>...]Mon Sep 6 20:28:54 2021[1,0]: - ETA: 38:00 - loss: 0.1295 - auc: 0.7785Mon Sep 6 20:29:02 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3735/4000 [===========================>..]Mon Sep 6 20:29:46 2021[1,0]: - ETA: 37:10 - loss: 0.1295 - auc: 0.7786Mon Sep 6 20:29:54 2021[1,0]:Mon Sep 6 20:29:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3742/4000 [===========================>..] - ETA: 36:11 - loss: 0.1295 - auc: 0.7786Mon Sep 6 20:30:55 2021[1,0]:86Mon Sep 6 20:30:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:31:47 2021[1,0]:3749/4000 [===========================>..]Mon Sep 6 20:31:47 2021[1,0]: - ETA: 35:12 - loss: 0.1295 - auc: 0.7786Mon Sep 6 20:31:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:32:47 2021[1,0]:3756/4000 [===========================>..]Mon Sep 6 20:32:47 2021[1,0]: - ETA: 34:13 - loss: 0.1294 - auc: 0.7786Mon Sep 6 20:32:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:33:48 2021[1,0]:3763/4000 [===========================>..]Mon Sep 6 20:33:48 2021[1,0]: - ETA: 33:15 - loss: 0.1294 - auc: 0.7787Mon Sep 6 20:33:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3770/4000 [===========================>..]Mon Sep 6 20:34:49 2021[1,0]: - ETA: 32:16 - loss: 0.1294 - auc: 0.7787Mon Sep 6 20:34:58 2021[1,0]:Mon Sep 6 20:34:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:35:50 2021[1,0]:3777/4000 [===========================>..]Mon Sep 6 20:35:50 2021[1,0]: - ETA: 31:17 - loss: 0.1294 - auc: 0.7787Mon Sep 6 20:35:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:36:49 2021[1,0]:3784/4000 [===========================>..]Mon Sep 6 20:36:49 2021[1,0]: - ETA: 30:18 - loss: 0.1294 - auc: 0.7787Mon Sep 6 20:36:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:37:47 2021[1,0]:3791/4000 [===========================>..]Mon Sep 6 20:37:47 2021[1,0]: - ETA: 29:19 - loss: 0.1294 - auc: 0.7788Mon Sep 6 20:37:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3799/4000 [===========================>..]Mon Sep 6 20:38:53 2021[1,0]: - ETA: 28:12 - loss: 0.1294 - auc: 0.7788Mon Sep 6 20:39:02 2021[1,0]:Mon Sep 6 20:38:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3806/4000 [===========================>..]Mon Sep 6 20:39:51 2021[1,0]: - ETA: 27:13 - loss: 0.1294 - auc: 0.7788Mon Sep 6 20:39:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:40:50 2021[1,0]:3813/4000 [===========================>..]Mon Sep 6 20:40:50 2021[1,0]: - ETA: 26:14 - loss: 0.1294 - auc: 0.7788Mon Sep 6 20:40:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:41:47 2021[1,0]:3820/4000 [===========================>..]Mon Sep 6 20:41:47 2021[1,0]: - ETA: 25:15 - loss: 0.1294 - auc: 0.7788Mon Sep 6 20:41:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:42:54 2021[1,0]:3828/4000 [===========================>..]Mon Sep 6 20:42:54 2021[1,0]: - ETA: 24:07 - loss: 0.1293 - auc: 0.7789Mon Sep 6 20:43:02 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:43:51 2021[1,0]:3835/4000 [===========================>..]Mon Sep 6 20:43:51 2021[1,0]: - ETA: 23:08 - loss: 0.1293 - auc: 0.7789Mon Sep 6 20:44:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:44:49 2021[1,0]:3842/4000 [===========================>..]Mon Sep 6 20:44:49 2021[1,0]: - ETA: 22:09 - loss: 0.1293 - auc: 0.7789Mon Sep 6 20:44:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3850/4000 [===========================>..]Mon Sep 6 20:45:54 2021[1,0]: - ETA: 21:02 - loss: 0.1293 - auc: 0.7789Mon Sep 6 20:46:02 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:46:52 2021[1,0]:3857/4000 [===========================>..]Mon Sep 6 20:46:52 2021[1,0]: - ETA: 20:03 - loss: 0.1293 - auc: 0.7790Mon Sep 6 20:47:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:47:50 2021[1,0]:3864/4000 [===========================>..]Mon Sep 6 20:47:50 2021[1,0]: - ETA: 19:04 - loss: 0.1293 - auc: 0.7790Mon Sep 6 20:47:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:48:49 2021[1,0]:3871/4000 [============================>.]Mon Sep 6 20:48:49 2021[1,0]: - ETA: 18:05 - loss: 0.1293 - auc: 0.7790Mon Sep 6 20:48:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:49:48 2021[1,0]:3878/4000 [============================>.]Mon Sep 6 20:49:48 2021[1,0]: - ETA: 17:06 - loss: 0.1293 - auc: 0.7790Mon Sep 6 20:49:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:50:54 2021[1,0]:3886/4000 [============================>.]Mon Sep 6 20:50:54 2021[1,0]: - ETA: 15:59 - loss: 0.1293 - auc: 0.7791Mon Sep 6 20:51:02 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3893/4000 [============================>.]Mon Sep 6 20:51:51 2021[1,0]: - ETA: 15:00 - loss: 0.1293 - auc: 0.7791Mon Sep 6 20:52:00 2021[1,0]:Mon Sep 6 20:51:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:52:49 2021[1,0]:3900/4000 [============================>.]Mon Sep 6 20:52:49 2021[1,0]: - ETA: 14:01 - loss: 0.1292 - auc: 0.7791Mon Sep 6 20:52:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:53:55 2021[1,0]:3908/4000 [============================>.]Mon Sep 6 20:53:55 2021[1,0]: - ETA: 12:54 - loss: 0.1292 - auc: 0.7791Mon Sep 6 20:54:03 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:54:52 2021[1,0]:3915/4000 [============================>.]Mon Sep 6 20:54:52 2021[1,0]: - ETA: 11:55 - loss: 0.1292 - auc: 0.7791Mon Sep 6 20:55:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:55:49 2021[1,0]:3922/4000 [============================>.]Mon Sep 6 20:55:49 2021[1,0]: - ETA: 10:56 - loss: 0.1292 - auc: 0.7792Mon Sep 6 20:55:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3930/4000 [============================>.]Mon Sep 6 20:56:53 2021[1,0]: - ETA: 9:48 - loss: 0.1292 - auc: 0.7792Mon Sep 6 20:57:01 2021[1,0]Mon Sep 6 20:57:01 2021[1,0]:>Mon Sep 6 20:56:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:57:50 2021[1,0]:3937/4000 [============================>.]Mon Sep 6 20:57:50 2021[1,0]: - ETA: 8:49 - loss: 0.1292 - auc: 0.7792Mon Sep 6 20:57:58 2021[1,0]Mon Sep 6 20:57:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3944/4000 [============================>.]Mon Sep 6 20:58:47 2021[1,0]: - ETA: 7:51 - loss: 0.1292 - auc: 0.7792Mon Sep 6 20:58:55 2021[1,0]Mon Sep 6 20:58:55 2021[1,0]:Mon Sep 6 20:58:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3952/4000 [============================>.]Mon Sep 6 20:59:52 2021[1,0]: - ETA: 6:43 - loss: 0.1292 - auc: 0.7792Mon Sep 6 21:00:00 2021[1,0]Mon Sep 6 21:00:00 2021[1,0]:Mon Sep 6 20:59:44 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 21:00:55 2021[1,0]:3960/4000 [============================>.]Mon Sep 6 21:00:55 2021[1,0]: - ETA: 5:36 - loss: 0.1292 - auc: 0.7793Mon Sep 6 21:01:03 2021[1,0]Mon Sep 6 21:01:03 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 21:01:51 2021[1,0]:3967/4000 [============================>.]Mon Sep 6 21:01:51 2021[1,0]: - ETA: 4:37 - loss: 0.1292 - auc: 0.7793Mon Sep 6 21:01:59 2021[1,0]Mon Sep 6 21:01:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 21:02:54 2021[1,0]:3975/4000 [============================>.]Mon Sep 6 21:02:54 2021[1,0]: - ETA: 3:30 - loss: 0.1292 - auc: 0.7793Mon Sep 6 21:03:02 2021[1,0]Mon Sep 6 21:02:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3982/4000 [============================>.]Mon Sep 6 21:03:50 2021[1,0]: - ETA: 2:31 - loss: 0.1292 - auc: 0.7793Mon Sep 6 21:03:58 2021[1,0]Mon Sep 6 21:03:58 2021[1,0]:Mon Sep 6 21:03:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 21:04:55 2021[1,0]:3990/4000 [============================>.]Mon Sep 6 21:04:55 2021[1,0]: - ETA: 1:24 - loss: 0.1291 - auc: 0.7793Mon Sep 6 21:05:03 2021[1,0]Mon Sep 6 21:05:03 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 21:05:54 2021[1,0]:3997/4000 [============================>.]Mon Sep 6 21:05:54 2021[1,0]: - ETA: 25s - loss: 0.1291 - auc: 0.7793Mon Sep 6 21:06:03 2021[1,0]:::\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 21:06:11 2021[1,0]:3999/4000 [============================>.]Mon Sep 6 21:06:11 2021[1,0]: - ETA: 8s - loss: 0.1291 - auc: 0.7793 Mon Sep 6 21:06:20 2021[1,0]on Sep 6 21:06:11 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 21:09:31 2021[1,0]:4000/4000 [==============================]Mon Sep 6 21:09:31 2021[1,0]: - 33843s 8s/step - loss: 0.1291 - auc: 0.7793 - val_loss: 0.1287 - val_auc: 0.7880 6 21:09:31 2021[1,0]:\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Best Loss: 0.128178\n" - ] - } - ], - "source": [ - "main_petastorm()" - ] - }, - { - "cell_type": "markdown", - "id": "49a4020d", - "metadata": {}, - "source": [ - "### Conclusion\n", - "\n", - "From cell[8] and cell[10], we can see the total training time respectively: \n", - "\n", - "- NVTabular: 845s\n", - "- Petastorm: 33843s \n", - "\n", - "So the speedup : 33843 / 845 = `40.05`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "51361729", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.2" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/dimensions/._SUCCESS.crc b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/dimensions/._SUCCESS.crc deleted file mode 100644 index 3b7b04493..000000000 Binary files a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/dimensions/._SUCCESS.crc and /dev/null differ diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/dimensions/.part-00000-1bfe973d-db55-449a-8c6a-9aaaa58b0f3e-c000.csv.crc b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/dimensions/.part-00000-1bfe973d-db55-449a-8c6a-9aaaa58b0f3e-c000.csv.crc deleted file mode 100644 index 33674bbb1..000000000 Binary files a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/dimensions/.part-00000-1bfe973d-db55-449a-8c6a-9aaaa58b0f3e-c000.csv.crc and /dev/null differ diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/dimensions/_SUCCESS b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/dimensions/_SUCCESS deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/dimensions/part-00000-1bfe973d-db55-449a-8c6a-9aaaa58b0f3e-c000.csv b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/dimensions/part-00000-1bfe973d-db55-449a-8c6a-9aaaa58b0f3e-c000.csv deleted file mode 100644 index b9e01a929..000000000 --- a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/dimensions/part-00000-1bfe973d-db55-449a-8c6a-9aaaa58b0f3e-c000.csv +++ /dev/null @@ -1,2 +0,0 @@ -c0,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25 -7912888,33822,17138,7338,20045,3,7104,1381,62,5554113,582468,245827,10,2208,10666,103,3,967,14,8165895,2675939,7156452,302515,12021,96,34 diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/test_small/._SUCCESS.crc b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/test_small/._SUCCESS.crc deleted file mode 100644 index 3b7b04493..000000000 Binary files a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/test_small/._SUCCESS.crc and /dev/null differ diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/test_small/.part-00000-39895f81-1e2c-4cb1-a883-b21c9328bf3f-c000.snappy.parquet.crc b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/test_small/.part-00000-39895f81-1e2c-4cb1-a883-b21c9328bf3f-c000.snappy.parquet.crc deleted file mode 100644 index 72624f728..000000000 Binary files a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/test_small/.part-00000-39895f81-1e2c-4cb1-a883-b21c9328bf3f-c000.snappy.parquet.crc and /dev/null differ diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/test_small/_SUCCESS b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/test_small/_SUCCESS deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/test_small/part-00000-39895f81-1e2c-4cb1-a883-b21c9328bf3f-c000.snappy.parquet b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/test_small/part-00000-39895f81-1e2c-4cb1-a883-b21c9328bf3f-c000.snappy.parquet deleted file mode 100644 index 272949e41..000000000 Binary files a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/test_small/part-00000-39895f81-1e2c-4cb1-a883-b21c9328bf3f-c000.snappy.parquet and /dev/null differ diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/train_small/._SUCCESS.crc b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/train_small/._SUCCESS.crc deleted file mode 100644 index 3b7b04493..000000000 Binary files a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/train_small/._SUCCESS.crc and /dev/null differ diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/train_small/.part-00000-b14c1424-bac6-4e03-9935-e969a5a05f34-c000.snappy.parquet.crc b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/train_small/.part-00000-b14c1424-bac6-4e03-9935-e969a5a05f34-c000.snappy.parquet.crc deleted file mode 100644 index 3b6f24a55..000000000 Binary files a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/train_small/.part-00000-b14c1424-bac6-4e03-9935-e969a5a05f34-c000.snappy.parquet.crc and /dev/null differ diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/train_small/_SUCCESS b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/train_small/_SUCCESS deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/train_small/part-00000-b14c1424-bac6-4e03-9935-e969a5a05f34-c000.snappy.parquet b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/train_small/part-00000-b14c1424-bac6-4e03-9935-e969a5a05f34-c000.snappy.parquet deleted file mode 100644 index 4311ea0d6..000000000 Binary files a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/train_small/part-00000-b14c1424-bac6-4e03-9935-e969a5a05f34-c000.snappy.parquet and /dev/null differ diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/val_small/._SUCCESS.crc b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/val_small/._SUCCESS.crc deleted file mode 100644 index 3b7b04493..000000000 Binary files a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/val_small/._SUCCESS.crc and /dev/null differ diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/val_small/.part-00000-d22ab04e-dacf-4c66-96cc-2f5cdbaa6177-c000.snappy.parquet.crc b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/val_small/.part-00000-d22ab04e-dacf-4c66-96cc-2f5cdbaa6177-c000.snappy.parquet.crc deleted file mode 100644 index 1043f863a..000000000 Binary files a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/val_small/.part-00000-d22ab04e-dacf-4c66-96cc-2f5cdbaa6177-c000.snappy.parquet.crc and /dev/null differ diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/val_small/_SUCCESS b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/val_small/_SUCCESS deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/val_small/part-00000-d22ab04e-dacf-4c66-96cc-2f5cdbaa6177-c000.snappy.parquet b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/val_small/part-00000-d22ab04e-dacf-4c66-96cc-2f5cdbaa6177-c000.snappy.parquet deleted file mode 100644 index 88793ad65..000000000 Binary files a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/val_small/part-00000-d22ab04e-dacf-4c66-96cc-2f5cdbaa6177-c000.snappy.parquet and /dev/null differ diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/spark-env.sh b/examples/ML+DL-Examples/Spark-DL/criteo_train/spark-env.sh deleted file mode 100755 index 58af1b354..000000000 --- a/examples/ML+DL-Examples/Spark-DL/criteo_train/spark-env.sh +++ /dev/null @@ -1,2 +0,0 @@ -SPARK_MASTER_HOST=127.0.0.1 -SPARK_WORKER_OPTS="-Dspark.worker.resource.gpu.amount=16 -Dspark.worker.resource.gpu.discoveryScript=/opt/spark-3.1.2-bin-hadoop3.2/examples/src/main/scripts/getGpusResources.sh" \ No newline at end of file diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/start-spark.sh b/examples/ML+DL-Examples/Spark-DL/criteo_train/start-spark.sh deleted file mode 100755 index e4e248e47..000000000 --- a/examples/ML+DL-Examples/Spark-DL/criteo_train/start-spark.sh +++ /dev/null @@ -1,2 +0,0 @@ -/opt/spark-3.1.2-bin-hadoop3.2/sbin/start-master.sh -/opt/spark-3.1.2-bin-hadoop3.2/sbin/start-worker.sh spark://127.0.0.1:7077 diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/submit.sh b/examples/ML+DL-Examples/Spark-DL/criteo_train/submit.sh deleted file mode 100755 index aaea59d9b..000000000 --- a/examples/ML+DL-Examples/Spark-DL/criteo_train/submit.sh +++ /dev/null @@ -1,27 +0,0 @@ -SPARK_HOME=/opt/spark-3.1.2-bin-hadoop3.2 -SPARK_URL=spark://127.0.0.1:7077 - -$SPARK_HOME/bin/spark-submit --master $SPARK_URL --deploy-mode client \ ---driver-memory 20G \ ---executor-memory 50G \ ---executor-cores 6 \ ---conf spark.cores.max=96 \ ---conf spark.task.cpus=6 \ ---conf spark.locality.wait=0 \ ---conf spark.yarn.maxAppAttempts=1 \ ---conf spark.sql.shuffle.partitions=4 \ ---conf spark.sql.files.maxPartitionBytes=1024m \ ---conf spark.sql.warehouse.dir=$OUT \ ---conf spark.task.resource.gpu.amount=0.08 \ ---conf spark.executor.resource.gpu.amount=1 \ ---conf spark.executor.resource.gpu.discoveryScript=./getGpusResources.sh \ ---files $SPARK_HOME/examples/src/main/scripts/getGpusResources.sh \ -./criteo_keras.py \ ---num-proc 16 \ ---data-dir file:///data/parquet \ ---logs-dir $PWD/tf_logs \ ---dataloader nvtabular \ ---learning-rate 0.001 \ ---batch-size 65536 \ ---epochs 1 \ ---local-checkpoint-file ckpt_file \ No newline at end of file