Skip to content

Commit da133ee

Browse files
danpoveydanpovey
committed
sandbox/online: merging changes from trunk.
git-svn-id: https://svn.code.sf.net/p/kaldi/code/sandbox/online@3563 5e6a8d80-dfce-4ca6-a32a-6e07a63d50c8
2 parents 9e8fddd + b42b9b7 commit da133ee

212 files changed

Lines changed: 9106 additions & 6497 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎egs/babel/s5/run-1-main.sh‎

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
# This is not necessarily the top-level run.sh as it is in other directories. see README.txt first.
44
tri5_only=false
5+
56
[ ! -f ./lang.conf ] && echo "Language configuration does not exist! Use the configurations in conf/lang/* as a startup" && exit 1
67
[ ! -f ./conf/common_vars.sh ] && echo "the file conf/common_vars.sh does not exist!" && exit 1
78

@@ -10,6 +11,8 @@ tri5_only=false
1011

1112
[ -f local.conf ] && . ./local.conf
1213

14+
. ./utils/parse_options.sh
15+
1316
set -e #Exit on non-zero return code from any command
1417
set -o pipefail #Exit if any of the commands in the pipeline will
1518
#return non-zero return code
@@ -24,12 +27,12 @@ if [ ! -d data/raw_train_data ]; then
2427
local/make_corpus_subset.sh "$train_data_dir" "$train_data_list" ./data/raw_train_data
2528
train_data_dir=`readlink -f ./data/raw_train_data`
2629

27-
nj_max=`cat $train_data_list | wc -l`
28-
if [[ "$nj_max" -lt "$train_nj" ]] ; then
29-
echo "The maximum reasonable number of jobs is $nj_max (you have $train_nj)! (The training and decoding process has file-granularity)"
30-
exit 1;
31-
train_nj=$nj_max
32-
fi
30+
fi
31+
nj_max=`cat $train_data_list | wc -l`
32+
if [[ "$nj_max" -lt "$train_nj" ]] ; then
33+
echo "The maximum reasonable number of jobs is $nj_max (you have $train_nj)! (The training and decoding process has file-granularity)"
34+
exit 1;
35+
train_nj=$nj_max
3336
fi
3437
train_data_dir=`readlink -f ./data/raw_train_data`
3538

@@ -59,8 +62,9 @@ if [[ ! -f data/local/lexicon.txt || data/local/lexicon.txt -ot "$lexicon_file"
5962
echo ---------------------------------------------------------------------
6063
echo "Preparing lexicon in data/local on" `date`
6164
echo ---------------------------------------------------------------------
65+
local/make_lexicon_subset.sh $train_data_dir/transcription $lexicon_file data/local/filtered_lexicon.txt
6266
local/prepare_lexicon.pl --phonemap "$phoneme_mapping" \
63-
$lexiconFlags $lexicon_file data/local
67+
$lexiconFlags data/local/filtered_lexicon.txt data/local
6468
fi
6569

6670
mkdir -p data/lang
@@ -116,6 +120,7 @@ if [[ ! -f data/srilm/lm.gz || data/srilm/lm.gz -ot data/train/text ]]; then
116120
local/train_lms_srilm.sh --dev-text data/dev2h/text \
117121
--train-text data/train/text data data/srilm
118122
fi
123+
119124
if [[ ! -f data/lang/G.fst || data/lang/G.fst -ot data/srilm/lm.gz ]]; then
120125
echo ---------------------------------------------------------------------
121126
echo "Creating G.fst on " `date`
@@ -152,8 +157,7 @@ if [ ! -f data/train/.plp.done ]; then
152157
rm -rf {plp,ffv}_tmp_train data/train_{plp,ffv}
153158
fi
154159
utils/fix_data_dir.sh data/train
155-
steps/compute_cmvn_stats.sh \
156-
data/train exp/make_plp/train plp
160+
steps/compute_cmvn_stats.sh data/train exp/make_plp/train plp
157161
# In case plp or pitch extraction failed on some utterances, delist them
158162
utils/fix_data_dir.sh data/train
159163
touch data/train/.plp.done
@@ -273,6 +277,12 @@ if [ ! -f exp/tri5_ali/.done ]; then
273277
touch exp/tri5_ali/.done
274278
fi
275279

280+
if $tri5_only ; then
281+
echo "Exiting after stage TRI5, as requested. "
282+
echo "Everything went fine. Done"
283+
exit 0;
284+
fi
285+
276286
if [ ! -f exp/ubm5/.done ]; then
277287
echo ---------------------------------------------------------------------
278288
echo "Starting exp/ubm5 on" `date`
@@ -311,11 +321,6 @@ if [ ! -f exp/sgmm5_ali/.done ]; then
311321
touch exp/sgmm5_ali/.done
312322
fi
313323

314-
if $tri5_only ; then
315-
echo "Exiting after stage TRI5, as requested. "
316-
echo "Everything went fine. Done"
317-
exit 0;
318-
fi
319324

320325
if [ ! -f exp/sgmm5_denlats/.done ]; then
321326
echo ---------------------------------------------------------------------
@@ -334,7 +339,7 @@ if [ ! -f exp/sgmm5_mmi_b0.1/.done ]; then
334339
echo ---------------------------------------------------------------------
335340
steps/train_mmi_sgmm2.sh \
336341
--cmd "$train_cmd" "${sgmm_mmi_extra_opts[@]}" \
337-
--zero-if-disjoint true --transform-dir exp/tri5_ali --boost 0.1 \
342+
--drop-frames true --transform-dir exp/tri5_ali --boost 0.1 \
338343
data/train data/lang exp/sgmm5_ali exp/sgmm5_denlats \
339344
exp/sgmm5_mmi_b0.1
340345
touch exp/sgmm5_mmi_b0.1/.done

‎egs/babel/s5/run-3-bnf-system.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ fi
130130
if [ ! -f exp_BNF/sgmm7_mmi_b0.1/.done ]; then
131131
steps/train_mmi_sgmm2.sh \
132132
--cmd "$train_cmd" --acwt 0.06 \
133-
--transform-dir exp_BNF/tri6_ali --boost 0.1 --zero-if-disjoint true \
133+
--transform-dir exp_BNF/tri6_ali --boost 0.1 --drop-frames true \
134134
data/app_train data/lang exp_BNF/sgmm7_ali exp_BNF/sgmm7_denlats \
135135
exp_BNF/sgmm7_mmi_b0.1
136136
touch exp_BNF/sgmm7_mmi_b0.1/.done;

‎egs/babel/s5b/conf/common.fullLP‎

Lines changed: 33 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,31 @@
1+
# BNF training parameters
2+
bnf_num_hidden_layers=6
3+
bottleneck_dim=42
4+
bnf_hidden_layer_dim=2048
5+
bnf_minibatch_size=512
6+
bnf_init_learning_rate=0.008
7+
bnf_final_learning_rate=0.0008
8+
bnf_max_change=40
9+
bnf_num_jobs=4
10+
bnf_num_threads=1
11+
bnf_mixup=10000
12+
bnf_mpe_learning_rate=0.00009
13+
bnf_mpe_last_layer_factor=0.1
14+
bnf_num_gauss_ubm=550 # use fewer UBM Gaussians than the
15+
# non-bottleneck system (which has 800)
16+
bnf_num_gauss_sgmm=50000 # use fewer SGMM sub-states than the
17+
# non-bottleneck system (which has 80000).
18+
bnf_decode_acwt=0.066666
19+
120
# DNN hybrid system training parameters
221
dnn_num_hidden_layers=4
322
dnn_input_dim=4000
423
dnn_output_dim=400
524
dnn_init_learning_rate=0.008
625
dnn_final_learning_rate=0.0008
7-
dnn_max_change=10
826
dnn_mixup=12000
927

10-
dnn_mpe_learning_rate=0.00012
28+
dnn_mpe_learning_rate=0.00008
1129
dnn_mpe_last_layer_factor=0.1
1230
dnn_mpe_retroactive=true
1331

@@ -17,8 +35,8 @@ babel_type=full
1735
use_pitch=true
1836

1937
lmwt_plp_extra_opts=( --min-lmwt 8 --max-lmwt 12 )
20-
lmwt_bnf_extra_opts=( --min-lmwt 13 --max-lmwt 18 )
21-
lmwt_dnn_extra_opts=( --min-lmwt 9 --max-lmwt 13 )
38+
lmwt_bnf_extra_opts=( --min-lmwt 15 --max-lmwt 22 )
39+
lmwt_dnn_extra_opts=( --min-lmwt 10 --max-lmwt 15 )
2240

2341
dnn_beam=16.0
2442
dnn_lat_beam=8.5
@@ -32,6 +50,7 @@ if [[ `hostname` == *.tacc.utexas.edu ]] ; then
3250
sgmm_denlats_extra_opts=( --num-threads 2 )
3351
dnn_denlats_extra_opts=( --num-threads 2 )
3452
dnn_parallel_opts="-l gpu=1"
53+
dnn_gpu_mpe_parallel_opts=(--num-jobs-nnet 8 --num-threads 1)
3554
else
3655
decode_extra_opts=(--num-threads 6 --parallel-opts "-pe smp 6 -l mem_free=4G,ram_free=0.7G")
3756
sgmm_train_extra_opts=( --num-iters 25 )
@@ -40,11 +59,13 @@ else
4059
sgmm_mmi_extra_opts=(--cmd "queue.pl -l arch=*64 -l mem_free=3.2G,ram_free=3.2G")
4160
dnn_denlats_extra_opts=(--num-threads 4 --parallel-opts "-pe smp 4" --cmd "queue.pl -l arch=*64 -l mem_free=2G,ram_free=0.8G")
4261

43-
dnn_cpu_parallel_opts=(--minibatch-size 128 --num-jobs-nnet 8 --num-threads 16 \
62+
dnn_cpu_parallel_opts=(--minibatch-size 128 --max-change 10 --num-jobs-nnet 8 --num-threads 16 \
4463
--parallel-opts "-pe smp 16" --cmd "queue.pl -l arch=*64 -l mem_free=2G,ram_free=1G")
45-
dnn_gpu_parallel_opts=(--minibatch-size 128 --num-jobs-nnet 8 --num-threads 1 \
64+
dnn_gpu_parallel_opts=(--minibatch-size 512 --max-change 40 --num-jobs-nnet 8 --num-threads 1 \
4665
--parallel-opts "-l gpu=1" --cmd "queue.pl -l arch=*64 -l mem_free=2G,ram_free=1G")
47-
66+
dnn_parallel_opts="-l gpu=1"
67+
dnn_gpu_mpe_parallel_opts=(--num-jobs-nnet 8 --num-threads 1 \
68+
--parallel-opts "-l gpu=1" --cmd "queue.pl -l arch=*64 -l mem_free=2G,ram_free=1G")
4869
fi
4970

5071
icu_transform="Any-Lower"
@@ -56,3 +77,8 @@ wip=0.5
5677

5778

5879
phoneme_mapping=
80+
81+
extend_lexicon=true
82+
unk_fraction_boost=1.0
83+
num_sent_gen=12000000
84+
num_prons=1000000

‎egs/babel/s5b/conf/common.limitedLP‎

Lines changed: 34 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,29 @@
1-
# DNN hybrid system training parameters
1+
# BNF training parameters
2+
bnf_num_hidden_layers=5
3+
bottleneck_dim=42
4+
bnf_hidden_layer_dim=1024
5+
bnf_minibatch_size=512
6+
bnf_init_learning_rate=0.008
7+
bnf_final_learning_rate=0.0008
8+
bnf_max_change=40
9+
bnf_num_jobs=4
10+
bnf_num_threads=1
11+
bnf_mixup=5000
12+
bnf_mpe_learning_rate=0.00009
13+
bnf_mpe_last_layer_factor=0.1
14+
bnf_num_gauss_ubm=500 # use fewer UBM Gaussians than the
15+
# non-bottleneck system (which has 750)
16+
bnf_num_gauss_sgmm=10000 # use fewer SGMM sub-states than the
17+
# non-bottleneck system (which has 18000).
18+
bnf_decode_acwt=0.066666
19+
20+
21+
## DNN hybrid system training parameters
222
dnn_num_hidden_layers=3
323
dnn_input_dim=2000
424
dnn_output_dim=200
525
dnn_init_learning_rate=0.008
626
dnn_final_learning_rate=0.0008
7-
dnn_max_change=40
827
dnn_mixup=5000
928

1029
dnn_mpe_learning_rate=0.00009
@@ -17,8 +36,8 @@ babel_type=limited
1736
use_pitch=true
1837

1938
lmwt_plp_extra_opts=( --min-lmwt 8 --max-lmwt 12 )
20-
lmwt_bnf_extra_opts=( --min-lmwt 13 --max-lmwt 18 )
21-
lmwt_dnn_extra_opts=( --min-lmwt 9 --max-lmwt 13 )
39+
lmwt_bnf_extra_opts=( --min-lmwt 15 --max-lmwt 22 )
40+
lmwt_dnn_extra_opts=( --min-lmwt 10 --max-lmwt 15 )
2241

2342
dnn_beam=16.0
2443
dnn_lat_beam=8.5
@@ -32,6 +51,7 @@ if [[ `hostname` == *.tacc.utexas.edu ]] ; then
3251
sgmm_denlats_extra_opts=( --num-threads 1 )
3352
dnn_denlats_extra_opts=( --num-threads 1 )
3453
dnn_parallel_opts="-l gpu=1"
54+
dnn_gpu_mpe_parallel_opts=(--num-jobs-nnet 4 --num-threads 1)
3555
else
3656
decode_extra_opts=(--num-threads 6 --parallel-opts "-pe smp 6 -l mem_free=4G,ram_free=0.7G")
3757
sgmm_train_extra_opts=( --num-iters 25 )
@@ -40,10 +60,13 @@ else
4060
sgmm_mmi_extra_opts=(--cmd "queue.pl -l arch=*64 -l mem_free=1.5G,ram_free=1.5G")
4161
dnn_denlats_extra_opts=(--num-threads 4 --parallel-opts "-pe smp 4" --cmd "queue.pl -l arch=*64 -l mem_free=2G,ram_free=0.8G")
4262

43-
dnn_cpu_parallel_opts=(--minibatch-size 128 --num-jobs-nnet 8 --num-threads 16 \
63+
dnn_cpu_parallel_opts=(--minibatch-size 128 --max-change 10 --num-jobs-nnet 8 --num-threads 16 \
4464
--parallel-opts "-pe smp 16" --cmd "queue.pl -l arch=*64 -l mem_free=2G,ram_free=1G")
45-
dnn_gpu_parallel_opts=(--minibatch-size 512 --num-jobs-nnet 4 --num-threads 1 \
65+
dnn_gpu_parallel_opts=(--minibatch-size 512 --max-change 40 --num-jobs-nnet 4 --num-threads 1 \
4666
--parallel-opts "-l gpu=1" --cmd "queue.pl -l arch=*64 -l mem_free=2G,ram_free=1G")
67+
dnn_parallel_opts="-l gpu=1"
68+
dnn_gpu_mpe_parallel_opts=(--num-jobs-nnet 4 --num-threads 1 \
69+
--parallel-opts "-l gpu=1" --cmd "queue.pl -l arch=*64 -l mem_free=2G,ram_free=1G")
4770

4871
fi
4972

@@ -56,3 +79,8 @@ wip=0.5
5679

5780

5881
phoneme_mapping=
82+
83+
extend_lexicon=true
84+
unk_fraction_boost=1.0
85+
num_sent_gen=12000000
86+
num_prons=1000000

‎egs/babel/s5b/conf/common.semisupervised.limitedLP‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
# DNN hybrid system training parameters
2+
dnn_num_hidden_layers=3
3+
dnn_input_dim=2000
4+
dnn_output_dim=200
5+
dnn_init_learning_rate=0.008
6+
dnn_final_learning_rate=0.0008
7+
dnn_max_change=40
8+
dnn_mixup=5000
9+
10+
num_epochs=15
11+
num_epochs_extra=5
12+
num_iters_final=20
13+
14+
dnn_mpe_learning_rate=0.00009
15+
dnn_mpe_last_layer_factor=0.1
16+
dnn_mpe_retroactive=true
17+
18+
bnf_every_nth_frame=1 # take all frames.
19+
babel_type=limited
20+
21+
use_pitch=true
22+
23+
lmwt_plp_extra_opts=( --min-lmwt 8 --max-lmwt 12 )
24+
lmwt_bnf_extra_opts=( --min-lmwt 13 --max-lmwt 18 )
25+
lmwt_dnn_extra_opts=( --min-lmwt 9 --max-lmwt 13 )
26+
27+
dnn_beam=16.0
28+
dnn_lat_beam=8.5
29+
30+
icu_opt=(--use-icu true --icu-transform Any-Lower)
31+
32+
if [[ `hostname` == *.tacc.utexas.edu ]] ; then
33+
decode_extra_opts=( --num-threads 4 --parallel-opts "-pe smp 4" )
34+
sgmm_train_extra_opts=( )
35+
sgmm_group_extra_opts=( )
36+
sgmm_denlats_extra_opts=( --num-threads 1 )
37+
dnn_denlats_extra_opts=( --num-threads 1 )
38+
dnn_parallel_opts="-l gpu=1"
39+
else
40+
decode_extra_opts=(--num-threads 6 --parallel-opts "-pe smp 6 -l mem_free=4G,ram_free=0.7G")
41+
sgmm_train_extra_opts=( --num-iters 25 )
42+
sgmm_group_extra_opts=(--group 3 --parallel-opts "-pe smp 3 -l mem_free=7G,ram_free=2.3G" --cmd "queue.pl -l arch=*64 -l mem_free=2.0G,ram_free=2.0G")
43+
sgmm_denlats_extra_opts=(--num-threads 4 --parallel-opts "-pe smp 4" --cmd "queue.pl -l arch=*64 -l mem_free=2G,ram_free=0.8G")
44+
sgmm_mmi_extra_opts=(--cmd "queue.pl -l arch=*64 -l mem_free=1.5G,ram_free=1.5G")
45+
dnn_denlats_extra_opts=(--num-threads 4 --parallel-opts "-pe smp 4" --cmd "queue.pl -l arch=*64 -l mem_free=2G,ram_free=0.8G")
46+
47+
dnn_cpu_parallel_opts=(--minibatch-size 128 --num-jobs-nnet 8 --num-threads 16 \
48+
--parallel-opts "-pe smp 16" --cmd "queue.pl -l arch=*64 -l mem_free=2G,ram_free=1G")
49+
dnn_gpu_parallel_opts=(--minibatch-size 512 --num-jobs-nnet 4 --num-threads 1 \
50+
--parallel-opts "-l gpu=1" --cmd "queue.pl -l arch=*64 -l mem_free=2G,ram_free=1G")
51+
egs_gpu_opts=(--splice-width 4 --samples-per-iter 200000 --num-jobs-nnet 4 --cmd "queue.pl -l arch=*64 -l mem_free=2G,ram_free=1G")
52+
egs_cpu_opts=(--splice-width 4 --samples-per-iter 200000 --num-jobs-nnet 8 --cmd "queue.pl -l arch=*64 -l mem_free=2G,ram_free=1G")
53+
egs_io_opts="-tc 5"
54+
fi
55+
56+
icu_transform="Any-Lower"
57+
case_insensitive=true
58+
59+
60+
max_states=150000
61+
wip=0.5
62+
63+
64+
phoneme_mapping=
65+

‎egs/babel/s5b/conf/lang/102-assamese-fullLP.official.conf‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,4 +73,3 @@ word_ins_penalty=0.5
7373
#keyword search settings
7474
duptime=0.5
7575
case_insensitive=true
76-

‎egs/babel/s5b/conf/lang/102-assamese-limitedLP.official.conf‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -33,15 +33,23 @@ dev10h_more_kwlists=(
3333
)
3434
dev10h_nj=32
3535

36-
3736
#Official EVAL period evaluation data files (not released yet)
38-
#eval_data_dir=
39-
#eval_data_list=
40-
#eval_data_cmudb=
41-
#eval_ecf_file=
42-
#eval_kwlist_file=
43-
#eval_nj=64
37+
eval_data_dir=
38+
eval_data_list=
39+
eval_data_cmudb=
40+
eval_ecf_file=
41+
eval_kwlist_file=
42+
eval_nj=64
4443

44+
unsup_data_dir=(/export/babel/data/102-assamese//release-current/conversational/training/
45+
/export/babel/data/102-assamese//release-current/conversational/untranscribed-training/
46+
)
47+
unsup_data_list=(
48+
/export/babel/data/splits/Assamese_Babel102/train.LimitedLP.untranscribed.list
49+
/export/babel/data/splits/Assamese_Babel102/train.untranscribed.list
50+
)
51+
unsup_nj=64
52+
4553
# Acoustic model parameters
4654
numLeavesTri1=1000
4755
numGaussTri1=10000
@@ -73,4 +81,3 @@ word_ins_penalty=0.5
7381
#keyword search settings
7482
duptime=0.5
7583
case_insensitive=true
76-

‎egs/babel/s5b/conf/lang/103-bengali-fullLP.official.conf‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,4 +73,3 @@ word_ins_penalty=0.5
7373
#keyword search settings
7474
duptime=0.5
7575
case_insensitive=true
76-

0 commit comments

Comments
 (0)