Skip to content

Commit 8aa4e8d

Browse files
danpoveydanpovey
committed
Merge branch 'phones_id' of https://github.com/hhadian/kaldi into reorthonormalize-cleaned
Conflicts: egs/wsj/s5/steps/nnet3/chain/train.py src/nnet3/nnet-compile-looped.cc src/nnet3/nnet-simple-component.cc
2 parents fe18a16 + 7bfafa4 commit 8aa4e8d

166 files changed

Lines changed: 6248 additions & 847 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎egs/aishell/README.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,5 +5,9 @@ Aishell is an open Chinese Mandarin speech database published by Beijing Shell S
55
The database can be downloaded from openslr:
66
http://www.openslr.org/33/
77

8+
This folder contains two subfolders:
9+
s5: a speech recognition recipe
10+
v1: a speaker recognition recipe
11+
812
For more details, please visit:
913
http://www.aishelltech.com/kysjcp

‎egs/aishell/v1/README.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
The database can be downloaded from openslr:
2+
http://www.openslr.org/33/
3+
4+
For more details, please visit:
5+
http://www.aishelltech.com/kysjcp
6+
7+
We use the training set to train model,
8+
and split the test set into enroll and eval.

‎egs/aishell/v1/cmd.sh‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# you can change cmd.sh depending on what type of queue you are using.
2+
# If you have no queueing system and want to run on a local machine, you
3+
# can change all instances 'queue.pl' to run.pl (but be careful and run
4+
# commands one by one: most recipes will exhaust the memory on your
5+
# machine). queue.pl works with GridEngine (qsub). slurm.pl works
6+
# with slurm. Different queues are configured differently, with different
7+
# queue names and different ways of specifying things like memory;
8+
# to account for these differences you can create and edit the file
9+
# conf/queue.conf to match your queue's configuration. Search for
10+
# conf/queue.conf in http://kaldi-asr.org/doc/queue.html for more information,
11+
# or search for the string 'default_config' in utils/queue.pl or utils/slurm.pl.
12+
13+
export train_cmd="queue.pl --mem 4G"
14+
15+

‎egs/aishell/v1/conf/mfcc.conf‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
--sample-frequency=16000
2+
--num-mel-bins=40 #higher than the default which is 23
3+
--num-ceps=20 # higher than the default which is 12.

‎egs/aishell/v1/conf/vad.conf‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
--vad-energy-threshold=5.5
2+
--vad-energy-mean-scale=0.5

‎egs/aishell/v1/local/aishell_data_prep.sh‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
#!/bin/bash
2+
3+
# Copyright 2017 Xingyu Na
4+
# Apache 2.0
5+
6+
. ./path.sh || exit 1;
7+
8+
if [ $# != 2 ]; then
9+
echo "Usage: $0 <audio-path> <text-path>"
10+
echo " $0 /export/a05/xna/data/data_aishell/wav /export/a05/xna/data/data_aishell/transcript"
11+
exit 1;
12+
fi
13+
14+
aishell_audio_dir=$1
15+
aishell_text_dir=$2
16+
17+
train_dir=data/local/train
18+
dev_dir=data/local/dev
19+
test_dir=data/local/test
20+
21+
mkdir -p $train_dir
22+
mkdir -p $dev_dir
23+
mkdir -p $test_dir
24+
25+
# data directory check
26+
if [ ! -d $aishell_audio_dir ] || [ ! -d $aishell_text_dir ]; then
27+
echo "Error: $0 requires two directory arguments"
28+
exit 1;
29+
fi
30+
31+
# find wav audio file for train, dev and test resp.
32+
find $aishell_audio_dir -iname "*.wav" | grep -i "wav/train" > $train_dir/wav.flist || exit 1;
33+
find $aishell_audio_dir -iname "*.wav" | grep -i "wav/dev" > $dev_dir/wav.flist || exit 1;
34+
find $aishell_audio_dir -iname "*.wav" | grep -i "wav/test" > $test_dir/wav.flist || exit 1;
35+
36+
n=`cat $train_dir/wav.flist $dev_dir/wav.flist $test_dir/wav.flist | wc -l`
37+
[ $n -ne 141925 ] && \
38+
echo Warning: expected 141925 data data files, found $n
39+
40+
# Transcriptions preparation
41+
for dir in $train_dir $test_dir; do
42+
echo Preparing $dir transcriptions
43+
sed -e 's/\.wav//' $dir/wav.flist | awk -F '/' '{print $NF}' |\
44+
sort > $dir/utt.list
45+
sed -e 's/\.wav//' $dir/wav.flist | awk -F '/' '{i=NF-1;printf("%s %s\n",$NF,$i)}' |\
46+
sort > $dir/utt2spk_all
47+
paste -d' ' $dir/utt.list $dir/wav.flist > $dir/wav.scp_all
48+
utils/filter_scp.pl -f 1 $dir/utt.list $aishell_text_dir/*.txt > $dir/transcripts.txt
49+
awk '{print $1}' $dir/transcripts.txt > $dir/utt.list
50+
utils/filter_scp.pl -f 1 $dir/utt.list $dir/utt2spk_all | sort -u > $dir/utt2spk
51+
utils/filter_scp.pl -f 1 $dir/utt.list $dir/wav.scp_all | sort -u > $dir/wav.scp
52+
sort -u $dir/transcripts.txt > $dir/text
53+
utils/utt2spk_to_spk2utt.pl $dir/utt2spk > $dir/spk2utt
54+
done
55+
56+
mkdir -p data/train data/test
57+
for f in spk2utt utt2spk wav.scp text; do
58+
cp $train_dir/$f data/train/$f || exit 1;
59+
cp $test_dir/$f data/test/$f || exit 1;
60+
done
61+
62+
echo "$0: AISHELL data preparation succeeded"
63+
exit 0;

‎egs/aishell/v1/local/download_and_untar.sh‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
#!/bin/bash
2+
3+
# Copyright 2014 Johns Hopkins University (author: Daniel Povey)
4+
# 2017 Xingyu Na
5+
# Apache 2.0
6+
7+
remove_archive=false
8+
9+
if [ "$1" == --remove-archive ]; then
10+
remove_archive=true
11+
shift
12+
fi
13+
14+
if [ $# -ne 3 ]; then
15+
echo "Usage: $0 [--remove-archive] <data-base> <url-base> <corpus-part>"
16+
echo "e.g.: $0 /export/a05/xna/data www.openslr.org/resources/33 data_aishell"
17+
echo "With --remove-archive it will remove the archive after successfully un-tarring it."
18+
echo "<corpus-part> can be one of: data_aishell, resource."
19+
fi
20+
21+
data=$1
22+
url=$2
23+
part=$3
24+
25+
if [ ! -d "$data" ]; then
26+
echo "$0: no such directory $data"
27+
exit 1;
28+
fi
29+
30+
part_ok=false
31+
list="data_aishell resource"
32+
for x in $list; do
33+
if [ "$part" == $x ]; then part_ok=true; fi
34+
done
35+
if ! $part_ok; then
36+
echo "$0: expected <corpus-part> to be one of $list, but got '$part'"
37+
exit 1;
38+
fi
39+
40+
if [ -z "$url" ]; then
41+
echo "$0: empty URL base."
42+
exit 1;
43+
fi
44+
45+
if [ -f $data/$part/.complete ]; then
46+
echo "$0: data part $part was already successfully extracted, nothing to do."
47+
exit 0;
48+
fi
49+
50+
# sizes of the archive files in bytes.
51+
sizes="15582913665 1246920"
52+
53+
if [ -f $data/$part.tgz ]; then
54+
size=$(/bin/ls -l $data/$part.tgz | awk '{print $5}')
55+
size_ok=false
56+
for s in $sizes; do if [ $s == $size ]; then size_ok=true; fi; done
57+
if ! $size_ok; then
58+
echo "$0: removing existing file $data/$part.tgz because its size in bytes $size"
59+
echo "does not equal the size of one of the archives."
60+
rm $data/$part.gz
61+
else
62+
echo "$data/$part.tgz exists and appears to be complete."
63+
fi
64+
fi
65+
66+
if [ ! -f $data/$part.tgz ]; then
67+
if ! which wget >/dev/null; then
68+
echo "$0: wget is not installed."
69+
exit 1;
70+
fi
71+
full_url=$url/$part.tgz
72+
echo "$0: downloading data from $full_url. This may take some time, please be patient."
73+
74+
cd $data
75+
if ! wget --no-check-certificate $full_url; then
76+
echo "$0: error executing wget $full_url"
77+
exit 1;
78+
fi
79+
fi
80+
81+
cd $data
82+
83+
if ! tar -xvzf $part.tgz; then
84+
echo "$0: error un-tarring archive $data/$part.tgz"
85+
exit 1;
86+
fi
87+
88+
touch $data/$part/.complete
89+
90+
if [ $part == "data_aishell" ]; then
91+
cd $data/$part/wav
92+
for wav in ./*.tar.gz; do
93+
echo "Extracting wav from $wav"
94+
tar -zxf $wav && rm $wav
95+
done
96+
fi
97+
98+
echo "$0: Successfully downloaded and un-tarred $data/$part.tgz"
99+
100+
if $remove_archive; then
101+
echo "$0: removing $data/$part.tgz file since --remove-archive option was supplied."
102+
rm $data/$part.tgz
103+
fi
104+
105+
exit 0;

‎egs/aishell/v1/local/produce_trials.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#!/usr/bin/env python3
2+
# Copyright 2017 Bengu Wu
3+
# Apache 2.0.
4+
5+
# This script generate trials file.
6+
# Trial file is formatted as:
7+
# uttid spkid target|nontarget
8+
9+
# If uttid belong to spkid, it is marked 'target',
10+
# otherwise is 'nontarget'.
11+
# input: eval set uttspk file
12+
# output: trial file
13+
14+
import sys
15+
16+
fnutt = sys.argv[1]
17+
ftrial = open(sys.argv[2], 'w')
18+
19+
dictutt = {}
20+
for line in open(fnutt):
21+
utt2spk = line.rstrip('\r\t\n ')
22+
spk = utt2spk.split(' ')[1]
23+
if spk not in dictutt:
24+
dictutt[spk] = spk
25+
26+
for line in open(fnutt):
27+
utt2spk = line.rstrip('\r\t\n ')
28+
utt, spk = utt2spk.split(' ')
29+
for target in dictutt:
30+
if target == spk:
31+
trial = utt + ' ' + target + ' target'
32+
else:
33+
trial = utt + ' ' + target + ' nontarget'
34+
ftrial.write(trial + '\n')
35+
ftrial.close()

‎egs/aishell/v1/local/split_data_enroll_eval.py‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#!/usr/bin/env python3
2+
3+
# Copyright 2017 Bengu Wu
4+
# Apache 2.0.
5+
6+
# This script splits the test set utt2spk into enroll set and eval set
7+
# For each speaker, 3 utterances are randomly selected as enroll samples,
8+
# and the others are used as eval samples for evaluation
9+
# input: test utt2spk
10+
# output: enroll utt2spk, eval utt2spk
11+
12+
import sys,random
13+
14+
dictutt = {}
15+
16+
for line in open(sys.argv[1]):
17+
line = line.rstrip('\r\t\n ')
18+
utt, spk = line.split(' ')
19+
if spk not in dictutt:
20+
dictutt[spk] = []
21+
dictutt[spk].append(utt)
22+
23+
fenroll = open(sys.argv[2], 'w')
24+
feval = open(sys.argv[3], 'w')
25+
26+
for key in dictutt:
27+
utts = dictutt[key]
28+
random.shuffle(utts)
29+
for i in range(0, len(utts)):
30+
line = utts[i] + ' ' + key
31+
if(i < 3):
32+
fenroll.write(line + '\n')
33+
else:
34+
feval.write(line + '\n')
35+
36+
fenroll.close()
37+
feval.close()

‎egs/aishell/v1/path.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
export KALDI_ROOT=`pwd`/../../..
2+
export PATH=$PWD/utils/:$KALDI_ROOT/tools/openfst/bin:$KALDI_ROOT/tools/sph2pipe_v2.5:$PWD:$PATH
3+
[ ! -f $KALDI_ROOT/tools/config/common_path.sh ] && echo >&2 "The standard file $KALDI_ROOT/tools/config/common_path.sh is not present -> Exit!" && exit 1
4+
. $KALDI_ROOT/tools/config/common_path.sh
5+
export LC_ALL=C

0 commit comments

Comments
 (0)