NVIDIA · alancucki · Sep 15, 2020 · Sep 13, 2020 · Sep 15, 2020 · Sep 15, 2020
diff --git a/PyTorch/SpeechRecognition/Jasper/tensorrt/Dockerfile b/PyTorch/SpeechRecognition/Jasper/tensorrt/Dockerfile
@@ -1,14 +1,6 @@
-ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:19.10-py3
+ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:20.08-py3
 FROM ${FROM_IMAGE_NAME}
 
-RUN apt-get update && apt-get install -y python3
-
-WORKDIR /tmp/onnx-trt
-COPY tensorrt/onnx-trt.patch .
-RUN git clone https://github.com/onnx/onnx-tensorrt.git && cd onnx-tensorrt && git checkout 8716c9b && git submodule update --init --recursive && \
-    patch -f < ../onnx-trt.patch && mkdir build && cd build && cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DGPU_ARCHS="60 70 75" && make -j16 && make install && mv -f /usr/lib/libnvonnx* /usr/lib/x86_64-linux-gnu/ && ldconfig
-
-
 # Here's a good place to install pip reqs from JoC repo.
 # At the same step, also install TRT pip reqs
 WORKDIR /tmp/pipReqs

diff --git a/PyTorch/SpeechRecognition/Jasper/tensorrt/README.md b/PyTorch/SpeechRecognition/Jasper/tensorrt/README.md
@@ -19,8 +19,8 @@ After optimizing the compute-intensive acoustic model with NVIDIA TensorRT, infe
 - [Advanced](#advanced)
    * [Scripts and sample code](#scripts-and-sample-code)
    * [Parameters](#parameters)
-   * [TensorRT Inference Process](#tensorrt-inference-process)
    * [TensorRT Inference Benchmark Process](#tensorrt-inference-benchmark-process)
+   * [TensorRT Inference Process](#tensorrt-inference-process)
 - [Performance](#performance)
    * [Results](#results)
       * [Inference performance: NVIDIA T4](#inference-performance-nvidia-t4)
@@ -51,10 +51,10 @@ The following software version configuration has been tested and known to work:
 
 |Software|Version|
 |--------|-------|
-|Python|3.6.9|
-|PyTorch|1.2.0|
-|TensorRT|6.0.1.5|
-|CUDA|10.1.243|
+|Python|3.6.10|
+|PyTorch|1.7.0a0+8deb4fe|
+|TensorRT|7.1.3.4|
+|CUDA|11.0.221|
 
 ## Setup
 
@@ -65,16 +65,16 @@ The following section lists the requirements in order to start inference on the
 This repository contains a `Dockerfile` which extends the PyTorch 19.10-py3 NGC container and encapsulates some dependencies. Ensure you have the following components:
 
 * [NVIDIA Docker](https://github.com/NVIDIA/nvidia-docker)
-* [PyTorch 19.10-py3 NGC container](https://ngc.nvidia.com/catalog/containers/nvidia:pytorch)
-* [NVIDIA Volta](https://www.nvidia.com/en-us/data-center/volta-gpu-architecture/) or [Turing](https://www.nvidia.com/en-us/geforce/turing/) based GPU
+* [PyTorch 20.08-py3 NGC container](https://ngc.nvidia.com/catalog/containers/nvidia:pytorch)
+* NVIDIA [Volta](https://www.nvidia.com/en-us/data-center/volta-gpu-architecture/), [Turing](https://www.nvidia.com/en-us/geforce/turing/), or [Ampere](https://www.nvidia.com/en-us/data-center/nvidia-ampere-gpu-architecture/) based GPU
 * [Pretrained Jasper Model Checkpoint](https://ngc.nvidia.com/catalog/models/nvidia:jasperpyt_fp16)
 
-Required Python packages are listed in `requirements.txt` and `trt/requirements.txt`. These packages are automatically installed when the Docker container is built. To manually install them, run:
+Required Python packages are listed in `requirements.txt` and `tensorrt/requirements.txt`. These packages are automatically installed when the Docker container is built. To manually install them, run:
 
 
 ```bash
 pip install -r requirements.txt
-pip install -r trt/requirements.txt
+pip install -r tensorrt/requirements.txt
 ```
 
 
@@ -92,18 +92,18 @@ Running the following scripts will build and launch the container containing all
 2. Build the Jasper PyTorch with TensorRT container:
 
       ```bash
-      bash trt/scripts/docker/build.sh
+      bash tensorrt/scripts/docker/build.sh
       ```
 3. Start an interactive session in the NGC docker container:
 
       ```bash
-      bash trt/scripts/docker/launch.sh <DATA_DIR> <CHECKPOINT_DIR> <RESULT_DIR>
+      bash tensorrt/scripts/docker/launch.sh <DATA_DIR> <CHECKPOINT_DIR> <RESULT_DIR>
       ```
 
       Alternatively, to start a script in the docker container:
 
       ```bash
-      bash trt/scripts/docker/aunch.sh <DATA_DIR> <CHECKPOINT_DIR> <RESULT_DIR> <SCRIPT_PATH>
+      bash tensorrt/scripts/docker/launch.sh <DATA_DIR> <CHECKPOINT_DIR> <RESULT_DIR> <SCRIPT_PATH>
       ```
 
       The `/datasets`, `/checkpoints`, `/results` directories will be mounted as volumes and mapped to the corresponding directories `<DATA_DIR>`, `<CHECKPOINT_DIR>`, `<RESULT_DIR>` on the host. **These three paths should be absolute and should already exist.** The contents of this repository will be mounted to the `/workspace/jasper` directory. Note that `<DATA_DIR>`, `<CHECKPOINT_DIR>`, and `<RESULT_DIR>` directly correspond to the same arguments in `scripts/docker/launch.sh` mentioned in the [Jasper PyTorch README](../README.md).
@@ -117,7 +117,7 @@ Running the following scripts will build and launch the container containing all
       If LibriSpeech has not been downloaded already, note that only a subset of LibriSpeech is typically used for inference (`dev-*` and `test-*`). To acquire the inference subset of LibriSpeech run the following commands inside the container (does not require GPU):
 
       ```bash
-      bash trt/scripts/download_inference_librispeech.sh
+      bash tensorrt/scripts/download_inference_librispeech.sh
       ```
 
       Once the data download is complete, the following folders should exist:
@@ -131,7 +131,7 @@ Running the following scripts will build and launch the container containing all
       Next, preprocessing the data can be performed with the following command:
 
       ```bash
-      bash trt/scripts/preprocess_inference_librispeech.sh
+      bash tensorrt/scripts/preprocess_inference_librispeech.sh
       ```
 
       Once the data is preprocessed, the following additional files should now exist:
@@ -147,16 +147,16 @@ Running the following scripts will build and launch the container containing all
 
 5. Start TensorRT inference prediction
 
-      Inside the container, use the following script to run inference with TensorRT.
+      Inside the container, use the following script to run inference with TensorRT. To learn more about the following env variables see `tensorrt/scripts/inference.sh`.
       ```bash
       export CHECKPOINT=<CHECKPOINT>
       export TRT_PRECISION=<PRECISION>
       export PYTORCH_PRECISION=<PRECISION>
       export TRT_PREDICTION_PATH=<TRT_PREDICTION_PATH>
-      bash trt/scripts/trt_inference.sh
+      bash tensorrt/scripts/inference.sh
       ```
       A pretrained model checkpoint can be downloaded from [NGC model repository](https://ngc.nvidia.com/catalog/models/nvidia:jasperpyt_fp16). 
-      More details can be found in [Advanced](#advanced) under [Scripts and sample code](#scripts-and-sample-code), [Parameters](#parameters) and [TensorRT Inference process](#trt-inference).
+      More details can be found in [Advanced](#advanced) under [Scripts and sample code](#scripts-and-sample-code), [Parameters](#parameters) and [TensorRT Inference process](#tensorrt-inference).
 
 6.  Start TensorRT inference benchmark
 
@@ -169,10 +169,10 @@ Running the following scripts will build and launch the container containing all
       export TRT_PRECISION=<PRECISION>
       export PYTORCH_PRECISION=<PRECISION>
       export CSV_PATH=<CSV_PATH>
-      bash trt/scripts/trt_inference_benchmark.sh
+      bash tensorrt/scripts/inference_benchmark.sh
       ```
       A pretrained model checkpoint can be downloaded from the [NGC model repository](https://ngc.nvidia.com/catalog/models/nvidia:jasperpyt_fp16). 
-      More details can be found in [Advanced](#advanced) under [Scripts and sample code](#scripts-and-sample-code), [Parameters](#parameters) and [TensorRT Inference Benchmark process](#trt-inference-benchmark).
+      More details can be found in [Advanced](#advanced) under [Scripts and sample code](#scripts-and-sample-code), [Parameters](#parameters) and [TensorRT Inference Benchmark process](#tensorrt-inference-benchmark).
 
 7. Start Jupyter notebook to run inference interactively
 
@@ -185,28 +185,27 @@ Running the following scripts will build and launch the container containing all
 The following sections provide greater details on inference benchmarking with TensorRT and show inference results
 
 ### Scripts and sample code
-In the `trt/` directory, the most important files are:
+In the `tensorrt/` directory, the most important files are:
 * `Dockerfile`: Container to run Jasper inference with TensorRT.
 * `requirements.py`: Python package dependencies. Installed when building the Docker container.
 * `perf.py`: Entry point for inference pipeline using TensorRT.
 * `perfprocedures.py`: Contains functionality to run inference through both the PyTorch model and TensorRT Engine, taking runtime measurements of each component of the inference process for comparison.
 * `trtutils.py`: Helper functions for TensorRT components of Jasper inference.
 * `perfutils.py`: Helper functions for non-TensorRT components of Jasper inference.
-* `onnx-trt.patch`: Used to enable Onnx and TensorRT with dynamic shapes.
 
-The `trt/scripts/` directory has one-click scripts to run supported functionalities, such as:
+The `tensorrt/scripts/` directory has one-click scripts to run supported functionalities, such as:
 
 * `download_librispeech.sh`: Downloads LibriSpeech inference dataset.
 * `preprocess_librispeech.sh`: Preprocess LibriSpeech raw data files to be ready for inference.
-* `trt_inference_benchmark.sh`: Benchmarks and compares TensorRT and PyTorch inference pipelines using the `perf.py` script.
-* `trt_inference.sh`: Runs TensorRT and PyTorch inference using the `trt_inference_benchmark.sh` script.
-* `walk_benchmark.sh`: Illustrates an example of using `trt/scripts/trt_inference_benchmark.sh`, which *walks* a variety of values for `BATCH_SIZE` and `NUM_FRAMES`.
+* `inference_benchmark.sh`: Benchmarks and compares TensorRT and PyTorch inference pipelines using the `perf.py` script.
+* `inference.sh`: Runs TensorRT and PyTorch inference using the `inference_benchmark.sh` script.
+* `walk_benchmark.sh`: Illustrates an example of using `tensorrt/scripts/inference_benchmark.sh`, which *walks* a variety of values for `BATCH_SIZE` and `NUM_FRAMES`.
 * `docker/`: Contains the scripts for building and launching the container.
 
 
 ### Parameters
 
-The list of parameters available for `trt/scripts/trt_inference_benchmark.sh` is:
+The list of parameters available for `tensorrt/scripts/inference_benchmark.sh` is:
 
 ```
 Required:
@@ -232,7 +231,7 @@ FORCE_ENGINE_REBUILD: boolean that indicates whether an already-built TensorRT e
 USE_DYNAMIC_SHAPE: if 'yes' uses dynamic shapes (default: ‘yes’). Dynamic shape is always preferred since it allows to reuse engines.
 ```
 
-The complete list of parameters available for `trt/scripts/trt_inference.sh` is the same as `trt/scripts/trt_inference_benchmark.sh` only with different default input arguments. In the following, only the parameters with different default values are listed:
+The complete list of parameters available for `tensorrt/scripts/inference.sh` is the same as `tensorrt/scripts/inference_benchmark.sh` only with different default input arguments. In the following, only the parameters with different default values are listed:
 
 ```
 TRT_PREDICTION_PATH: file to store inference prediction results generated with TensorRT (default: `/results/trt_predictions.txt`)
@@ -244,7 +243,7 @@ NUM_FRAMES: cuts/pads all pre-processed feature tensors to this length. 100 fram
 
 ### TensorRT Inference Benchmark process
 
-The inference benchmarking is performed on a single GPU by ‘trt/scripts/trt_inference_benchmark.sh’ which delegates to `trt/perf.py`,  which takes the following steps:
+The inference benchmarking is performed on a single GPU by ‘tensorrt/scripts/inference_benchmark.sh’ which delegates to `tensorrt/perf.py`,  which takes the following steps:
 
 
 1. Construct Jasper acoustic model in PyTorch.
@@ -259,19 +258,19 @@ The inference benchmarking is performed on a single GPU by ‘trt/scripts/trt_in
 
 4. Compile performance and WER accuracy results in CSV format, written to `CSV_PATH` file.
 
-`trt/perf.py` utilizes `trt/trtutils.py` and `trt/perfutils.py`, helper functions for TensorRT and non-TensorRT components of Jasper inference respectively.
+`tensorrt/perf.py` utilizes `tensorrt/trtutils.py` and `tensorrt/perfutils.py`, helper functions for TensorRT and non-TensorRT components of Jasper inference respectively.
 
 ### TensorRT Inference process
 
-The inference is performed by `trt/scripts/trt_inference.sh` which delegates to `trt/scripts/trt_inference_benchmark.sh`. The script runs on a single GPU. To do inference prediction on the entire dataset `NUM_FRAMES` is set to 3600, which roughly corresponds to 36 seconds. This covers the longest sentences in both LibriSpeech dev and test dataset. By default, `BATCH_SET` is set to 1 to simulate the online inference scenario in deployment. Other batch sizes can be tried by setting a different value to this parameter. By default `TRT_PRECISION` is set to full precision and can be changed by setting `export TRT_PRECISION=fp16`. The prediction results are stored at `/results/trt_predictions.txt` and `/results/pyt_predictions.txt`.
+The inference is performed by `tensorrt/scripts/inference.sh` which delegates to `tensorrt/scripts/inference_benchmark.sh`. The script runs on a single GPU. To do inference prediction on the entire dataset `NUM_FRAMES` is set to 3600, which roughly corresponds to 36 seconds. This covers the longest sentences in both LibriSpeech dev and test dataset. By default, `BATCH_SET` is set to 1 to simulate the online inference scenario in deployment. Other batch sizes can be tried by setting a different value to this parameter. By default `TRT_PRECISION` is set to full precision and can be changed by setting `export TRT_PRECISION=fp16`. The prediction results are stored at `/results/trt_predictions.txt` and `/results/pyt_predictions.txt`.
 
 
 
 ## Performance
 
 To benchmark the inference performance on a specific batch size and audio length refer to [Quick-Start-Guide](#quick-start-guide). To do a sweep over multiple batch sizes and audio durations run:
 ```bash
-bash trt/scripts/walk_benchmark.sh
+bash tensorrt/scripts/walk_benchmark.sh
 ```
 The results are obtained by running inference on LibriSpeech dev-clean dataset on a single T4 GPU using half precision with AMP. We compare the throughput of the acoustic model between TensorRT and native PyTorch.   
 

diff --git a/PyTorch/SpeechRecognition/Jasper/tensorrt/onnx-trt.patch b/PyTorch/SpeechRecognition/Jasper/tensorrt/onnx-trt.patch
diff --git a/PyTorch/SpeechRecognition/Jasper/tensorrt/perf.py b/PyTorch/SpeechRecognition/Jasper/tensorrt/perf.py
@@ -18,7 +18,7 @@
 import sys, os
 # Get local modules in parent directory and current directory (assuming this was called from root of repository)
 sys.path.append("./")
-sys.path.append("./trt")
+sys.path.append("./tensorrt")
 import perfutils
 import trtutils
 import perfprocedures
@@ -116,7 +116,7 @@ def parse_args():
     parser.add_argument("--pyt_fp16", action="store_true", default=False, help="If set, will construct pytorch model with fp16 weights")
     parser.add_argument("--make_onnx", action="store_true", default=False, help="If set, will create an ONNX model and store it at the path specified by --onnx_path")
     parser.add_argument("--csv_path", type=str, default=None, help="File to append csv info about inference time")
-    parser.add_argument("--trt_prediction_path", type=str, default=None, help="File to write predictions inferred with trt")
+    parser.add_argument("--trt_prediction_path", type=str, default=None, help="File to write predictions inferred with tensorrt")
     parser.add_argument("--pyt_prediction_path", type=str, default=None, help="File to write predictions inferred with pytorch")
     parser.add_argument("--verbose", action="store_true", default=False, help="If set, will verbosely describe TRT engine building and deserialization as well as TRT inference")
     parser.add_argument("--wav", type=str, help='absolute path to .wav file (16KHz)')

diff --git a/PyTorch/SpeechRecognition/Jasper/tensorrt/requirements.txt b/PyTorch/SpeechRecognition/Jasper/tensorrt/requirements.txt
@@ -1,4 +1,4 @@
 pycuda
 pillow
-onnx==1.5.0
-onnxruntime==0.5.0
+onnx==1.6.0
+onnxruntime==1.4.0
diff --git a/PyTorch/SpeechRecognition/Jasper/tensorrt/scripts/docker/build.sh b/PyTorch/SpeechRecognition/Jasper/tensorrt/scripts/docker/build.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
 
-# Constructs a docker image containing dependencies for execution of JASPER through TRT
-echo "docker build . -f ./tensorrt/Dockerfile -t jasper:trt6"
-docker build . -f ./tensorrt/Dockerfile -t jasper:trt6
+# Constructs a docker image containing dependencies for execution of JASPER through TensorRT
+echo "docker build . -f ./tensorrt/Dockerfile -t jasper:tensorrt"
+docker build . -f ./tensorrt/Dockerfile -t jasper:tensorrt
diff --git a/PyTorch/SpeechRecognition/Jasper/tensorrt/scripts/docker/launch.sh b/PyTorch/SpeechRecognition/Jasper/tensorrt/scripts/docker/launch.sh
@@ -40,4 +40,4 @@ nvidia-docker run -it --rm \
   -v $RESULT_DIR:/results/ \
   -v ${JASPER_REPO}:/jasper \
   ${EXTRA_JASPER_ENV} \
-  jasper:trt6 bash $PROGRAM_PATH
+  jasper:tensorrt bash $PROGRAM_PATH
diff --git a/.../Jasper/tensorrt/scripts/trt_inference.sh → ...tion/Jasper/tensorrt/scripts/inference.sh b/.../Jasper/tensorrt/scripts/trt_inference.sh → ...tion/Jasper/tensorrt/scripts/inference.sh
@@ -56,7 +56,7 @@ export TRT_PREDICTION_PATH="$TRT_PREDICTION_PATH"
 export PYT_PREDICTION_PATH="$PYT_PREDICTION_PATH"
 export VERBOSE="$VERBOSE"
 
-bash ./trt/scripts/trt_inference_benchmark.sh $1 $2 $3 $4 $5 $6 $7
+bash ./tensorrt/scripts/inference_benchmark.sh $1 $2 $3 $4 $5 $6 $7
 
 trt_word_error_rate=`cat "$CSV_PATH" | awk '{print $3}'`
 pyt_word_error_rate=`cat "$CSV_PATH" | awk '{print $4}'`

diff --git a/...nsorrt/scripts/trt_inference_benchmark.sh → ...r/tensorrt/scripts/inference_benchmark.sh b/...nsorrt/scripts/trt_inference_benchmark.sh → ...r/tensorrt/scripts/inference_benchmark.sh
@@ -155,7 +155,7 @@ CMD+=" $PYT_PREDICTION_PATH"
 
 if [ "$CREATE_LOGFILE" == "true" ] ; then
   export GBS=$(expr $BATCH_SIZE )
-  printf -v TAG "jasper_trt_inference_benchmark_%s_gbs%d" "$PYTORCH_PRECISION" $GBS
+  printf -v TAG "jasper_tensorrt_inference_benchmark_%s_gbs%d" "$PYTORCH_PRECISION" $GBS
   DATESTAMP=`date +'%y%m%d%H%M%S'`
   LOGFILE=$LOG_DIR/$TAG.$DATESTAMP.log
   printf "Logs written to %s\n" "$LOGFILE"

diff --git a/PyTorch/SpeechRecognition/Jasper/tensorrt/scripts/walk_benchmark.sh b/PyTorch/SpeechRecognition/Jasper/tensorrt/scripts/walk_benchmark.sh
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# A usage example of trt_inference_benchmark.sh.
+# A usage example of inference_benchmark.sh.
 
 
 export NUM_STEPS=100
@@ -37,7 +37,7 @@ do
             export BATCH_SIZE=$bs
 
             echo "Doing batch size ${bs}, sequence length ${nf}, precision ${prec}"
-            bash trt/scripts/trt_inference_benchmark.sh $1 $2 $3 $4 $5 $6
+            bash tensorrt/scripts/inference_benchmark.sh $1 $2 $3 $4 $5 $6
         done
     done
 done