NVIDIA · nv-kkudrynski · Aug 14, 2020 · Aug 14, 2020
diff --git a/PyTorch/LanguageModeling/BERT/Dockerfile b/PyTorch/LanguageModeling/BERT/Dockerfile
@@ -34,7 +34,7 @@ WORKDIR /workspace/bert
 RUN pip install --upgrade --no-cache-dir pip \
  && pip install --no-cache-dir \
  tqdm boto3 requests six ipdb h5py html2text nltk progressbar onnxruntime \
- git+https://github.com/NVIDIA/dllogger
+ git+https://github.com/NVIDIA/dllogger wget
 
 RUN apt-get install -y iputils-ping
 

diff --git a/PyTorch/LanguageModeling/BERT/NOTICE b/PyTorch/LanguageModeling/BERT/NOTICE
@@ -2,4 +2,3 @@ BERT PyTorch
 
 This repository includes software from https://github.com/huggingface/pytorch-pretrained-BERT
 licensed under the Apache License 2.0.
-
diff --git a/PyTorch/LanguageModeling/BERT/README.md b/PyTorch/LanguageModeling/BERT/README.md
diff --git a/PyTorch/LanguageModeling/BERT/bind.sh b/PyTorch/LanguageModeling/BERT/bind.sh
@@ -1,3 +1,16 @@
+# Copyright (c) 2020 NVIDIA CORPORATION. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 #! /bin/bash
 set -euo pipefail
 

diff --git a/PyTorch/LanguageModeling/BERT/data/Downloader.py b/PyTorch/LanguageModeling/BERT/data/Downloader.py
@@ -15,16 +15,16 @@
 from NVIDIAPretrainedWeightDownloader import NVIDIAPretrainedWeightDownloader
 from WikiDownloader import WikiDownloader
 from BooksDownloader import BooksDownloader
-from MRPCDownloader import MRPCDownloader
+from GLUEDownloader import GLUEDownloader
 from SquadDownloader import SquadDownloader
 
 
 class Downloader:
+
     def __init__(self, dataset_name, save_path):
         self.dataset_name = dataset_name
         self.save_path = save_path
 
-
     def download(self):
         if self.dataset_name == 'bookscorpus':
             self.download_bookscorpus()
@@ -41,50 +41,45 @@ def download(self):
         elif self.dataset_name == 'nvidia_pretrained_weights':
             self.download_nvidia_pretrained_weights()
 
-        elif self.dataset_name == 'mrpc':
-            self.download_mrpc()
+        elif self.dataset_name in {'mrpc', 'sst-2'}:
+            self.download_glue(self.dataset_name)
 
         elif self.dataset_name == 'squad':
             self.download_squad()
 
         elif self.dataset_name == 'all':
-            self.download_bookscorpus(self.save_path)
-            self.download_wikicorpus('en', self.save_path)
-            self.download_wikicorpus('zh', self.save_path)
-            self.download_google_pretrained_weights(self.save_path)
-            self.download_nvidia_pretrained_weights(self.save_path)
-            self.download_mrpc(self.save_path)
-            self.download_squad(self.save_path)
+            self.download_bookscorpus()
+            self.download_wikicorpus('en')
+            self.download_wikicorpus('zh')
+            self.download_google_pretrained_weights()
+            self.download_nvidia_pretrained_weights()
+            self.download_glue('mrpc')
+            self.download_glue('sst-2')
+            self.download_squad()
 
         else:
             print(self.dataset_name)
             assert False, 'Unknown dataset_name provided to downloader'
 
-
     def download_bookscorpus(self):
         downloader = BooksDownloader(self.save_path)
         downloader.download()
 
-
     def download_wikicorpus(self, language):
         downloader = WikiDownloader(language, self.save_path)
         downloader.download()
 
-
     def download_google_pretrained_weights(self):
         downloader = GooglePretrainedWeightDownloader(self.save_path)
         downloader.download()
 
-
     def download_nvidia_pretrained_weights(self):
         downloader = NVIDIAPretrainedWeightDownloader(self.save_path)
         downloader.download()
 
-
-    def download_mrpc(self):
-        downloader = MRPCDownloader(self.save_path)
-        downloader.download()
-
+    def download_glue(self, task_name):
+        downloader = GLUEDownloader(self.save_path)
+        downloader.download(task_name)
 
     def download_squad(self):
         downloader = SquadDownloader(self.save_path)

diff --git a/PyTorch/LanguageModeling/BERT/data/GLUEDownloader.py b/PyTorch/LanguageModeling/BERT/data/GLUEDownloader.py
@@ -0,0 +1,46 @@
+# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import wget
+
+from pathlib import Path
+
+
+def mkdir(path):
+    Path(path).mkdir(parents=True, exist_ok=True)
+
+
+class GLUEDownloader:
+
+    def __init__(self, save_path):
+        self.save_path = save_path + '/glue'
+
+    def download(self, task_name):
+        mkdir(self.save_path)
+        if task_name in {'mrpc', 'mnli'}:
+            task_name = task_name.upper()
+        elif task_name == 'cola':
+            task_name = 'CoLA'
+        else:  # SST-2
+            assert task_name == 'sst-2'
+            task_name = 'SST'
+        wget.download(
+            'https://gist.githubusercontent.com/W4ngatang/60c2bdb54d156a41194446737ce03e2e/raw/17b8dd0d724281ed7c3b2aeeda662b92809aadd5/download_glue_data.py',
+            out=self.save_path,
+        )
+        sys.path.append(self.save_path)
+        import download_glue_data
+        download_glue_data.main(
+            ['--data_dir', self.save_path, '--tasks', task_name])
+        sys.path.pop()
diff --git a/PyTorch/LanguageModeling/BERT/data/MRPCDownloader.py b/PyTorch/LanguageModeling/BERT/data/MRPCDownloader.py
diff --git a/PyTorch/LanguageModeling/BERT/data/bertPrep.py b/PyTorch/LanguageModeling/BERT/data/bertPrep.py
@@ -101,7 +101,7 @@ def main(args):
             output_filename = directory_structure['formatted'] + '/wikicorpus_zh_one_article_per_line.txt'
             wiki_formatter = WikicorpusTextFormatting.WikicorpusTextFormatting(wiki_path, output_filename, recursive=True)
             wiki_formatter.merge()
-            
+
             assert os.stat(output_filename).st_size > 0, 'File glob did not pick up extracted wiki files from WikiExtractor.'
 
     elif args.action == 'sharding':
@@ -248,6 +248,7 @@ def create_record_worker(filename_prefix, shard_id, output_format='hdf5'):
             'google_pretrained_weights',
             'nvidia_pretrained_weights',
             'mrpc',
+            'sst-2',
             'squad',
             'all'
         }

diff --git a/PyTorch/LanguageModeling/BERT/data/create_datasets_from_start.sh b/PyTorch/LanguageModeling/BERT/data/create_datasets_from_start.sh
@@ -23,6 +23,8 @@ fi
 python3 /workspace/bert/data/bertPrep.py --action download --dataset wikicorpus_en
 python3 /workspace/bert/data/bertPrep.py --action download --dataset google_pretrained_weights  # Includes vocab
 python3 /workspace/bert/data/bertPrep.py --action download --dataset squad
+python3 /workspace/bert/data/bertPrep.py --action download --dataset mrpc
+python3 /workspace/bert/data/bertPrep.py --action download --dataset sst-2
 
 # Properly format the text files
 if [ "$to_download" = "wiki_books" ] ; then

diff --git a/PyTorch/LanguageModeling/BERT/data/glue/download_mrpc.sh b/PyTorch/LanguageModeling/BERT/data/glue/download_mrpc.sh
diff --git a/PyTorch/LanguageModeling/BERT/modeling.py b/PyTorch/LanguageModeling/BERT/modeling.py
@@ -149,7 +149,7 @@ def __init__(self, in_features, out_features, act='gelu', bias=True):
         self.in_features = in_features
         self.out_features = out_features
         self.act_fn = nn.Identity()                                                         #
-        self.biased_act_fn = None                                                           # 
+        self.biased_act_fn = None                                                           #
         self.bias = None                                                                    #
         if isinstance(act, str) or (sys.version_info[0] == 2 and isinstance(act, unicode)): # For TorchScript
             if bias and not 'bias' in act:                                                  # compatibility
@@ -1073,17 +1073,10 @@ def __init__(self, config, num_labels):
         self.classifier = nn.Linear(config.hidden_size, num_labels)
         self.apply(self.init_bert_weights)
 
-    def forward(self, input_ids, token_type_ids=None, attention_mask=None, labels=None):
+    def forward(self, input_ids, token_type_ids=None, attention_mask=None):
         _, pooled_output = self.bert(input_ids, token_type_ids, attention_mask)
         pooled_output = self.dropout(pooled_output)
-        logits = self.classifier(pooled_output)
-
-        if labels is not None:
-            loss_fct = CrossEntropyLoss()
-            loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
-            return loss
-        else:
-            return logits
+        return self.classifier(pooled_output)
 
 
 class BertForMultipleChoice(BertPreTrainedModel):

diff --git a/PyTorch/LanguageModeling/BERT/processors/__init__.py b/PyTorch/LanguageModeling/BERT/processors/__init__.py
Original file line number	Diff line number	Diff line change
Expand Up		@@ -2,4 +2,3 @@ BERT PyTorch

		This repository includes software from https://github.com/huggingface/pytorch-pretrained-BERT
		licensed under the Apache License 2.0.