From cdfd5eba544ee0971c7c38ffd86f690afe7c7ef9 Mon Sep 17 00:00:00 2001 From: Kyunggeun Lee Date: Fri, 15 Nov 2024 16:19:19 -0800 Subject: [PATCH] Replace progressbar with tqdm (#3494) Signed-off-by: Kyunggeun Lee --- Examples/onnx/utils/image_net_evaluator.py | 26 ++++----- .../qat_range_learning_ddp_eval.py | 18 +++--- Examples/torch/utils/image_net_evaluator.py | 30 ++++------ Examples/torch/utils/image_net_trainer.py | 57 +++++++++---------- Jenkins/Dockerfile.onnx-cpu | 1 - Jenkins/Dockerfile.onnx-gpu | 1 - Jenkins/Dockerfile.tf-cpu | 1 - Jenkins/Dockerfile.tf-gpu | 1 - Jenkins/Dockerfile.tf-torch-cpu | 1 - Jenkins/Dockerfile.torch-cpu | 1 - Jenkins/Dockerfile.torch-cpu-pt113 | 1 - Jenkins/Dockerfile.torch-gpu | 1 - Jenkins/Dockerfile.torch-gpu-pt113 | 1 - .../Dockerfile.torch-gpu-pt21-py310 | 1 - .../Dockerfile.torch-gpu-pt21-py38 | 1 - .../reqs_pip_torch_gpu.txt | 1 - .../reqs_pip_torch_gpu.txt | 1 - packaging/dependencies/reqs_pip_common.txt | 1 - 18 files changed, 56 insertions(+), 89 deletions(-) diff --git a/Examples/onnx/utils/image_net_evaluator.py b/Examples/onnx/utils/image_net_evaluator.py index c8027e4ab2b..248139d0da9 100644 --- a/Examples/onnx/utils/image_net_evaluator.py +++ b/Examples/onnx/utils/image_net_evaluator.py @@ -40,7 +40,7 @@ """ import logging -import progressbar +from tqdm import tqdm import torch import onnxruntime as ort @@ -92,25 +92,19 @@ def evaluate(self, sess: ort.InferenceSession, iterations: int = None) -> float: logger.info("Evaluating nn.Module for %d iterations with batch_size %d", iterations, self._val_data_loader.batch_size) - batch_cntr = 1 - with progressbar.ProgressBar(max_value=iterations) as progress_bar: - for input_data, target_data in self._val_data_loader: + for i, (input_data, target_data) in tqdm(enumerate(self._val_data_loader), total=10): + if i == 10: + break - inputs_batch = input_data.numpy() + inputs_batch = input_data.numpy() - predicted_batch = sess.run(None, {input_name : inputs_batch})[0] + predicted_batch = sess.run(None, {input_name : inputs_batch})[0] - batch_avg_top_1_5 = accuracy(output=torch.from_numpy(predicted_batch), target=target_data, - topk=(1, 5)) + batch_avg_top_1_5 = accuracy(output=torch.from_numpy(predicted_batch), target=target_data, + topk=(1, 5)) - acc_top1 += batch_avg_top_1_5[0].item() - acc_top5 += batch_avg_top_1_5[1].item() - - progress_bar.update(batch_cntr) - - batch_cntr += 1 - if batch_cntr > iterations: - break + acc_top1 += batch_avg_top_1_5[0].item() + acc_top5 += batch_avg_top_1_5[1].item() acc_top1 /= iterations acc_top5 /= iterations diff --git a/Examples/torch/quantization/qat_range_learning_ddp_eval.py b/Examples/torch/quantization/qat_range_learning_ddp_eval.py index 0f27135625f..4c44e461d50 100644 --- a/Examples/torch/quantization/qat_range_learning_ddp_eval.py +++ b/Examples/torch/quantization/qat_range_learning_ddp_eval.py @@ -52,7 +52,7 @@ import socket import argparse -import progressbar +from tqdm import tqdm import torch import torch.distributed as dist import torch.multiprocessing as mp @@ -129,15 +129,13 @@ def evaluate_quant(model, _): model.to(device) model.eval() - with progressbar.ProgressBar(max_value=100) as progress_bar: - with torch.no_grad(): - for i, (images, _) in enumerate(val_loader): - images = images.to(device) - # compute output - _ = model(images) - progress_bar.update(i) - if i == 100: - break + with torch.no_grad(): + for i, (images, _) in tqdm(enumerate(val_loader), total=100): + images = images.to(device) + # compute output + _ = model(images) + if i == 100: + break return evaluate_quant diff --git a/Examples/torch/utils/image_net_evaluator.py b/Examples/torch/utils/image_net_evaluator.py index a2fa7cfee15..67b0bb0e89c 100644 --- a/Examples/torch/utils/image_net_evaluator.py +++ b/Examples/torch/utils/image_net_evaluator.py @@ -40,7 +40,7 @@ """ import logging -import progressbar +from tqdm import tqdm import torch from torch import nn @@ -104,27 +104,21 @@ def evaluate(self, model: nn.Module, iterations: int = None, use_cuda: bool = Fa model = model.to(device) model = model.eval() - batch_cntr = 1 - with progressbar.ProgressBar(max_value=iterations) as progress_bar: - with torch.no_grad(): - for input_data, target_data in self._val_data_loader: + with torch.no_grad(): + for i, (input_data, target_data) in tqdm(enumerate(self._val_data_loader), total=iterations): + if i == iterations: + break - inputs_batch = input_data.to(device) - target_batch = target_data.to(device) + inputs_batch = input_data.to(device) + target_batch = target_data.to(device) - predicted_batch = model(inputs_batch) + predicted_batch = model(inputs_batch) - batch_avg_top_1_5 = accuracy(output=predicted_batch, target=target_batch, - topk=(1, 5)) + batch_avg_top_1_5 = accuracy(output=predicted_batch, target=target_batch, + topk=(1, 5)) - acc_top1 += batch_avg_top_1_5[0].item() - acc_top5 += batch_avg_top_1_5[1].item() - - progress_bar.update(batch_cntr) - - batch_cntr += 1 - if batch_cntr > iterations: - break + acc_top1 += batch_avg_top_1_5[0].item() + acc_top5 += batch_avg_top_1_5[1].item() acc_top1 /= iterations acc_top5 /= iterations diff --git a/Examples/torch/utils/image_net_trainer.py b/Examples/torch/utils/image_net_trainer.py index e25cc1016d0..6db881c5aec 100644 --- a/Examples/torch/utils/image_net_trainer.py +++ b/Examples/torch/utils/image_net_trainer.py @@ -40,7 +40,7 @@ """ import logging -import progressbar +from tqdm import tqdm import torch from torch import nn, optim @@ -103,36 +103,31 @@ def _train_loop(self, model: nn.Module, criterion: torch.nn.modules.loss, optimi model.train() avg_loss = 0.0 - curr_iter = 1 - - with progressbar.ProgressBar(max_value=max_iterations) as progress_bar: - for images, target in self._train_loader: - images = images.to(device) - target = target.to(device) - - # compute model output - output = model(images) - loss = criterion(output, target) - avg_loss += loss.item() - - # compute gradient and do SGD step - optimizer.zero_grad() - loss.backward() - optimizer.step() - - progress_bar.update(curr_iter) - - if curr_iter % debug_steps == 0: - eval_accuracy = self._evaluator.evaluate(model, use_cuda=use_cuda) - logger.info('Epoch #%d/%d: iteration #%d/%d: Global Avg Loss=%f, Eval Accuracy=%f', - current_epoch, max_epochs, curr_iter, max_iterations, - avg_loss / curr_iter, eval_accuracy) - # switch to training mode after evaluation - model.train() - - curr_iter += 1 - if curr_iter > max_iterations: - break + + for i, (images, target) in tqdm(enumerate(self._train_loader), total=max_iterations): + if i == max_iterations: + break + + images = images.to(device) + target = target.to(device) + + # compute model output + output = model(images) + loss = criterion(output, target) + avg_loss += loss.item() + + # compute gradient and do SGD step + optimizer.zero_grad() + loss.backward() + optimizer.step() + + if (i+1) % debug_steps == 0: + eval_accuracy = self._evaluator.evaluate(model, use_cuda=use_cuda) + logger.info('Epoch #%d/%d: iteration #%d/%d: Global Avg Loss=%f, Eval Accuracy=%f', + current_epoch, max_epochs, curr_iter, max_iterations, + avg_loss / curr_iter, eval_accuracy) + # switch to training mode after evaluation + model.train() eval_accuracy = self._evaluator.evaluate(model, use_cuda=use_cuda) print("eval : ", eval_accuracy) diff --git a/Jenkins/Dockerfile.onnx-cpu b/Jenkins/Dockerfile.onnx-cpu index 725ee5b365c..a7de67c612c 100644 --- a/Jenkins/Dockerfile.onnx-cpu +++ b/Jenkins/Dockerfile.onnx-cpu @@ -170,7 +170,6 @@ RUN python3 -m pip --no-cache-dir install \ piccolo-theme \ Pillow==9.3.0 \ pluggy==0.12.0 \ - progressbar2 \ protobuf==3.20.2 \ psutil \ ptflops \ diff --git a/Jenkins/Dockerfile.onnx-gpu b/Jenkins/Dockerfile.onnx-gpu index 9818fa93ff1..ad4d6f11de5 100644 --- a/Jenkins/Dockerfile.onnx-gpu +++ b/Jenkins/Dockerfile.onnx-gpu @@ -181,7 +181,6 @@ RUN python3 -m pip --no-cache-dir install \ piccolo-theme \ Pillow==9.3.0 \ pluggy==0.12.0 \ - progressbar2 \ protobuf==3.20.2 \ psutil \ ptflops \ diff --git a/Jenkins/Dockerfile.tf-cpu b/Jenkins/Dockerfile.tf-cpu index d6fa1033dbe..6adb2ba43b8 100644 --- a/Jenkins/Dockerfile.tf-cpu +++ b/Jenkins/Dockerfile.tf-cpu @@ -153,7 +153,6 @@ RUN python3 -m pip --no-cache-dir install \ piccolo-theme \ Pillow==9.3.0 \ pluggy==0.12.0 \ - progressbar2 \ protobuf==3.19.2 \ psutil \ ptflops \ diff --git a/Jenkins/Dockerfile.tf-gpu b/Jenkins/Dockerfile.tf-gpu index fa0d98b54fa..22b81b4daa4 100644 --- a/Jenkins/Dockerfile.tf-gpu +++ b/Jenkins/Dockerfile.tf-gpu @@ -164,7 +164,6 @@ RUN python3 -m pip --no-cache-dir install \ piccolo-theme \ Pillow==9.3.0 \ pluggy==0.12.0 \ - progressbar2 \ protobuf==3.19.2 \ psutil \ ptflops \ diff --git a/Jenkins/Dockerfile.tf-torch-cpu b/Jenkins/Dockerfile.tf-torch-cpu index 5bba4c5e9db..ca2bc34e4d8 100644 --- a/Jenkins/Dockerfile.tf-torch-cpu +++ b/Jenkins/Dockerfile.tf-torch-cpu @@ -158,7 +158,6 @@ RUN python3 -m pip --no-cache-dir install \ piccolo-theme \ Pillow==9.3.0 \ pluggy==0.12.0 \ - progressbar2 \ # protobuf==3.20.1 \ psutil \ ptflops \ diff --git a/Jenkins/Dockerfile.torch-cpu b/Jenkins/Dockerfile.torch-cpu index 5dcf3d5c0ff..ff7bd00924b 100644 --- a/Jenkins/Dockerfile.torch-cpu +++ b/Jenkins/Dockerfile.torch-cpu @@ -169,7 +169,6 @@ RUN python3 -m pip --no-cache-dir install \ piccolo-theme \ Pillow==9.3.0 \ pluggy==0.12.0 \ - progressbar2 \ protobuf==3.20.2 \ psutil \ ptflops \ diff --git a/Jenkins/Dockerfile.torch-cpu-pt113 b/Jenkins/Dockerfile.torch-cpu-pt113 index 17b2e578cf9..6a240ec444a 100644 --- a/Jenkins/Dockerfile.torch-cpu-pt113 +++ b/Jenkins/Dockerfile.torch-cpu-pt113 @@ -169,7 +169,6 @@ RUN python3 -m pip --no-cache-dir install \ piccolo-theme \ Pillow==9.3.0 \ pluggy==0.12.0 \ - progressbar2 \ protobuf==3.20.2 \ psutil \ ptflops \ diff --git a/Jenkins/Dockerfile.torch-gpu b/Jenkins/Dockerfile.torch-gpu index 3790406d5bf..7076aeaaaee 100644 --- a/Jenkins/Dockerfile.torch-gpu +++ b/Jenkins/Dockerfile.torch-gpu @@ -181,7 +181,6 @@ RUN python3 -m pip --no-cache-dir install \ piccolo-theme \ Pillow==9.3.0 \ pluggy==0.12.0 \ - progressbar2 \ protobuf==3.20.2 \ psutil \ ptflops \ diff --git a/Jenkins/Dockerfile.torch-gpu-pt113 b/Jenkins/Dockerfile.torch-gpu-pt113 index 2880493cd07..7e25ab984c7 100644 --- a/Jenkins/Dockerfile.torch-gpu-pt113 +++ b/Jenkins/Dockerfile.torch-gpu-pt113 @@ -181,7 +181,6 @@ RUN python3 -m pip --no-cache-dir install \ piccolo-theme \ Pillow==9.3.0 \ pluggy==0.12.0 \ - progressbar2 \ protobuf==3.20.2 \ psutil \ ptflops \ diff --git a/Jenkins/fast-release/Dockerfile.torch-gpu-pt21-py310 b/Jenkins/fast-release/Dockerfile.torch-gpu-pt21-py310 index f1864830cc3..6d6096f690f 100644 --- a/Jenkins/fast-release/Dockerfile.torch-gpu-pt21-py310 +++ b/Jenkins/fast-release/Dockerfile.torch-gpu-pt21-py310 @@ -178,7 +178,6 @@ RUN python3 -m pip --no-cache-dir install \ piccolo-theme \ Pillow==9.3.0 \ pluggy==0.12.0 \ - progressbar2 \ protobuf==3.20.2 \ psutil \ ptflops \ diff --git a/Jenkins/fast-release/Dockerfile.torch-gpu-pt21-py38 b/Jenkins/fast-release/Dockerfile.torch-gpu-pt21-py38 index 4d9dc3f4104..eff4b93d987 100644 --- a/Jenkins/fast-release/Dockerfile.torch-gpu-pt21-py38 +++ b/Jenkins/fast-release/Dockerfile.torch-gpu-pt21-py38 @@ -174,7 +174,6 @@ RUN python3 -m pip --no-cache-dir install \ piccolo-theme \ Pillow==9.3.0 \ pluggy==0.12.0 \ - progressbar2 \ protobuf==3.20.2 \ psutil \ ptflops \ diff --git a/packaging/dependencies/fast-release/torch-gpu-pt21-py310/reqs_pip_torch_gpu.txt b/packaging/dependencies/fast-release/torch-gpu-pt21-py310/reqs_pip_torch_gpu.txt index d3f5ca33847..162570aafe7 100644 --- a/packaging/dependencies/fast-release/torch-gpu-pt21-py310/reqs_pip_torch_gpu.txt +++ b/packaging/dependencies/fast-release/torch-gpu-pt21-py310/reqs_pip_torch_gpu.txt @@ -13,7 +13,6 @@ onnx==1.14.1 onnxsim osqp pandas==1.5.3 -progressbar2 protobuf==3.20.2 psutil pybind11 diff --git a/packaging/dependencies/fast-release/torch-gpu-pt21-py38/reqs_pip_torch_gpu.txt b/packaging/dependencies/fast-release/torch-gpu-pt21-py38/reqs_pip_torch_gpu.txt index e46de77a2f8..566973a35ba 100644 --- a/packaging/dependencies/fast-release/torch-gpu-pt21-py38/reqs_pip_torch_gpu.txt +++ b/packaging/dependencies/fast-release/torch-gpu-pt21-py38/reqs_pip_torch_gpu.txt @@ -13,7 +13,6 @@ onnx==1.14.1 onnxsim osqp pandas==1.5.3 -progressbar2 protobuf==3.20.2 psutil pybind11 diff --git a/packaging/dependencies/reqs_pip_common.txt b/packaging/dependencies/reqs_pip_common.txt index 9cdac487d72..cab89f29a7b 100644 --- a/packaging/dependencies/reqs_pip_common.txt +++ b/packaging/dependencies/reqs_pip_common.txt @@ -6,7 +6,6 @@ hvplot==0.9.2 Jinja2==3.0.3 jsonschema osqp -progressbar2 pybind11 PyYAML scikit-learn==1.1.3