This repository was archived by the owner on Feb 24, 2022. It is now read-only.
forked from facebookresearch/TransCoder
-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathtest_preprocess.py
More file actions
executable file
·79 lines (68 loc) · 3.06 KB
/
Copy pathtest_preprocess.py
File metadata and controls
executable file
·79 lines (68 loc) · 3.06 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
# Copyright (c) 2019-present, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
#
import shutil
from concurrent.futures import ProcessPoolExecutor
from submitit import AutoExecutor
from pathlib import Path
from preprocessing.src.dataset import Dataset
from preprocessing.src import code_tokenizer as code_tokenizers
from preprocessing.preprocess import preprocess
root = Path(__file__).resolve().parents[1].joinpath("data/test_dataset")
print(f"ROOT{root}")
lang1 = "java"
lang2 = "python"
lang3 = 'cpp'
keep_comments = True
suffix = ".with_comments"
def copy_and_clean_folder():
# clean existing folder/files
for l in [lang1, lang2, lang3]:
for tokenized_file in root.joinpath(l).glob("*.tok"):
tokenized_file.unlink()
langs = sorted([lang1, lang2, lang3])
shutil.rmtree(str(root.joinpath(
f"{langs[0]}-{langs[1]}-{langs[2]}{suffix}")), ignore_errors=True)
shutil.rmtree(str(root.joinpath(
f"{langs[0]}-{langs[1]}-{langs[2]}{suffix}.XLM-syml")), ignore_errors=True)
print(
str(root.joinpath(f"{langs[0]}-{langs[1]}-{langs[2]}{suffix}.XLM-syml")))
shutil.rmtree(
str(root.joinpath(f"{langs[0]}-{langs[1]}-{langs[2]}")), ignore_errors=True)
def preprocess_(dataset, lang_executor=None, tok_executor=None, bpe_executor=None):
dataset.process_languages(
lang_executor=lang_executor, tok_executor=tok_executor)
dataset.train_bpe(ncodes=100, size_gb=None)
dataset.apply_bpe(
f'train{dataset.suffix}.[01234567].tok', use_vocab=False, executor=bpe_executor)
dataset.get_vocab()
dataset.apply_bpe(f'test{dataset.suffix}.tok',
use_vocab=True, executor=None)
dataset.apply_bpe(f'valid{dataset.suffix}.tok',
use_vocab=True, executor=None)
dataset.extract_functions_and_apply_bpe(
lang_executor=lang_executor, function_executor=tok_executor, bpe_executor=bpe_executor)
dataset.binarize_for_XLM(
f'train{dataset.suffix}.[01234567].functions_*.bpe', executor=None)
dataset.binarize_for_XLM(
f'test{dataset.suffix}.functions_*.bpe', executor=None)
dataset.binarize_for_XLM(
f'valid{dataset.suffix}.functions_*.bpe', executor=None)
def test_run_pipeline_locally_3_langs_with_comments():
copy_and_clean_folder()
preprocess(root, lang1, lang2, keep_comments, local=True,
lang3=lang3, test_size=10, size_gb=0)
def test_run_pipeline_submitit_3_langs_with_comments():
copy_and_clean_folder()
dataset = Dataset(root, lang1, lang2, keep_comments,
test_size=10, lang3=lang3)
mp_executor = ProcessPoolExecutor()
cluster_ex1 = AutoExecutor(dataset.folder.joinpath('log'), cluster="local")
cluster_ex1.update_parameters()
cluster_ex2 = AutoExecutor(dataset.folder.joinpath('log'), cluster="local")
cluster_ex2.update_parameters()
preprocess_(dataset, lang_executor=mp_executor,
tok_executor=cluster_ex1, bpe_executor=cluster_ex2)