[egs] fixes to babel pipeline; thanks to Fred Richardson (#1509)

egs/babel/s5d/conf/common.fullLP

@@ -35,10 +35,10 @@ babel_type=full
 
 use_pitch=true
 
-lmwt_plp_extra_opts=( --min-lmwt 8 --max-lmwt 18 )
+lmwt_plp_extra_opts=( --min-lmwt 9 --max-lmwt 13 )
 lmwt_bnf_extra_opts=( --min-lmwt 15 --max-lmwt 22 )
 lmwt_dnn_extra_opts=( --min-lmwt 10 --max-lmwt 15 )
-lmwt_chain_extra_opts=( --min-lmwt 4 --max-lmwt 22 )
+lmwt_chain_extra_opts=( --min-lmwt 9 --max-lmwt 13 )
 
 dnn_beam=16.0
 dnn_lat_beam=8.5

egs/babel/s5d/conf/common_vars.sh

@@ -12,6 +12,7 @@ cer=0
 
 #Declaring here to make the definition inside the language conf files more
 # transparent and nice
+declare -A train_kwlists
 declare -A dev10h_kwlists
 declare -A dev2h_kwlists
 declare -A evalpart1_kwlists

egs/babel/s5d/conf/lang/104-pashto-fullLP.official.conf

@@ -3,7 +3,7 @@
 
 #speech corpora files location
 train_data_dir=/export/babel/data/104-pashto/release-current/conversational/training
-train_data_list=/export/babel/data/splits/Pashto_Babel104/train.FullLP.list
+train_data_list=./conf/lists/104-pashto/training.list
 train_nj=32
 
 #RADICAL DEV2H data files
@@ -22,7 +22,7 @@ dev2h_nj=18
 
 #Official DEV data files
 dev10h_data_dir=/export/babel/data/104-pashto/release-current/conversational/dev
-dev10h_data_list=/export/babel/data/splits/Pashto_Babel104/dev.list
+dev10h_data_list=./conf/lists/104-pashto/dev.list
 dev10h_stm_file=/export/babel/data/scoring/IndusDB/IARPA-babel104b-v0.4bY_conv-dev/IARPA-babel104b-v0.4bY_conv-dev.stm
 dev10h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel104b-v0.4bY_conv-dev.ecf.xml
 dev10h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel104b-v0.4bY_conv-dev/IARPA-babel104b-v0.4bY_conv-dev.mitllfa3.rttm

egs/babel/s5d/conf/lang/105-turkish-fullLP.official.conf

@@ -5,7 +5,14 @@
 #speech corpora files location
 train_data_dir=/export/babel/data/105-turkish/release-current-b/conversational/training
 train_data_list=/export/babel/data/splits/Turkish_Babel105/train.fullLP.list
-train_nj=32
+#train_nj=32
+train_ecf_file=./data/train/ecf.train.xml
+train_rttm_file=./exp/tri5/rttm
+train_kwlists=(
+                      [dev]=/export/babel/data/scoring/IndusDB/IARPA-babel105b-v0.4_conv-dev.kwlist.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel105b-v0.4_conv-dev.kwlist2.xml
+)
+train_nj=64
 
 #RADICAL DEV data files
 dev2h_data_dir=/export/babel/data/105-turkish/release-current-b/conversational/dev

egs/babel/s5d/conf/lang/305-guarani.FLP.official.conf

@@ -11,11 +11,12 @@ train_nj=32
 #Radical reduced DEV corpora files location
 dev2h_data_dir=/export/babel/data/305-guarani/IARPA-babel305b-v1.0b-build/BABEL_OP3_305/conversational/dev
 dev2h_data_list=./conf/lists/305-guarani//dev.2h.list
-dev2h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel305b-v1.0a_conv-dev/IARPA-babel305b-v1.0a_conv-dev.mitllfa3.rttm
-dev2h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel305b-v1.0a_conv-dev/IARPA-babel305b-v1.0a_conv-dev.scoring.ecf.xml
-dev2h_stm_file=/export/babel/data/scoring/IndusDB/IARPA-babel305b-v1.0a_conv-dev/IARPA-babel305b-v1.0a_conv-dev.stm
+dev2h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel305b-v1.0c_conv-dev/IARPA-babel305b-v1.0c_conv-dev.mitllfa3.rttm
+dev2h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel305b-v1.0c_conv-dev/IARPA-babel305b-v1.0c_conv-dev.scoring.ecf.xml
+dev2h_stm_file=/export/babel/data/scoring/IndusDB/IARPA-babel305b-v1.0c_conv-dev/IARPA-babel305b-v1.0c_conv-dev.stm
 dev2h_kwlists=(
-    [kwlist]=/export/babel/data/scoring/IndusDB/IARPA-babel305b-v1.0a_conv-dev/IARPA-babel305b-v1.0a_conv-dev.annot.kwlist.xml
+    [dev]=/export/babel/data/scoring/IndusDB/IARPA-babel305b-v1.0c_conv-dev/IARPA-babel305b-v1.0c_conv-dev.annot.kwlist.xml
+    [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel305b-v1.0c_conv-dev.kwlist3.xml
 )  # dev2h_kwlists
 dev2h_nj=16
 dev2h_subset_ecf=true
@@ -24,11 +25,12 @@ dev2h_subset_ecf=true
 #Official DEV corpora files location
 dev10h_data_dir=/export/babel/data/305-guarani/IARPA-babel305b-v1.0b-build/BABEL_OP3_305/conversational/dev
 dev10h_data_list=./conf/lists/305-guarani//dev.list
-dev10h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel305b-v1.0a_conv-dev/IARPA-babel305b-v1.0a_conv-dev.mitllfa3.rttm
-dev10h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel305b-v1.0a_conv-dev/IARPA-babel305b-v1.0a_conv-dev.scoring.ecf.xml
-dev10h_stm_file=/export/babel/data/scoring/IndusDB/IARPA-babel305b-v1.0a_conv-dev/IARPA-babel305b-v1.0a_conv-dev.stm
+dev10h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel305b-v1.0c_conv-dev/IARPA-babel305b-v1.0c_conv-dev.mitllfa3.rttm
+dev10h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel305b-v1.0c_conv-dev/IARPA-babel305b-v1.0c_conv-dev.scoring.ecf.xml
+dev10h_stm_file=/export/babel/data/scoring/IndusDB/IARPA-babel305b-v1.0c_conv-dev/IARPA-babel305b-v1.0c_conv-dev.stm
 dev10h_kwlists=(
-    [kwlist]=/export/babel/data/scoring/IndusDB/IARPA-babel305b-v1.0a_conv-dev/IARPA-babel305b-v1.0a_conv-dev.annot.kwlist.xml
+    [dev]=/export/babel/data/scoring/IndusDB/IARPA-babel305b-v1.0c_conv-dev/IARPA-babel305b-v1.0c_conv-dev.annot.kwlist.xml
+    [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel305b-v1.0c_conv-dev.kwlist3.xml
 )  # dev10h_kwlists
 dev10h_nj=32
 

egs/babel/s5d/conf/lang/306-igbo.FLP.official.conf

@@ -15,7 +15,8 @@ dev2h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel306b-v2.0c_conv-de
 dev2h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel306b-v2.0c_conv-dev/IARPA-babel306b-v2.0c_conv-dev.scoring.ecf.xml
 dev2h_stm_file=/export/babel/data/scoring/IndusDB/IARPA-babel306b-v2.0c_conv-dev/IARPA-babel306b-v2.0c_conv-dev.stm
 dev2h_kwlists=(
-    [kwlist]=/export/babel/data/scoring/IndusDB/IARPA-babel306b-v2.0c_conv-dev/IARPA-babel306b-v2.0c_conv-dev.annot.kwlist.xml
+    [dev]=/export/babel/data/scoring/IndusDB/IARPA-babel306b-v2.0c_conv-dev/IARPA-babel306b-v2.0c_conv-dev.annot.kwlist.xml
+    [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel306b-v2.0c_conv-dev.kwlist3.xml
 )  # dev2h_kwlists
 dev2h_nj=16
 dev2h_subset_ecf=true
@@ -28,7 +29,8 @@ dev10h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel306b-v2.0c_conv-d
 dev10h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel306b-v2.0c_conv-dev/IARPA-babel306b-v2.0c_conv-dev.scoring.ecf.xml
 dev10h_stm_file=/export/babel/data/scoring/IndusDB/IARPA-babel306b-v2.0c_conv-dev/IARPA-babel306b-v2.0c_conv-dev.stm
 dev10h_kwlists=(
-    [kwlist]=/export/babel/data/scoring/IndusDB/IARPA-babel306b-v2.0c_conv-dev/IARPA-babel306b-v2.0c_conv-dev.annot.kwlist.xml
+    [dev]=/export/babel/data/scoring/IndusDB/IARPA-babel306b-v2.0c_conv-dev/IARPA-babel306b-v2.0c_conv-dev.annot.kwlist.xml
+    [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel306b-v2.0c_conv-dev.kwlist3.xml
 )  # dev10h_kwlists
 dev10h_nj=32
 

egs/babel/s5d/conf/lang/307-amharic.FLP.official.conf

@@ -15,7 +15,8 @@ dev2h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel307b-v1.0b_conv-de
 dev2h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel307b-v1.0b_conv-dev/IARPA-babel307b-v1.0b_conv-dev.scoring.ecf.xml
 dev2h_stm_file=/export/babel/data/scoring/IndusDB/IARPA-babel307b-v1.0b_conv-dev/IARPA-babel307b-v1.0b_conv-dev.stm
 dev2h_kwlists=(
-    [kwlist]=/export/babel/data/scoring/IndusDB/IARPA-babel307b-v1.0b_conv-dev/IARPA-babel307b-v1.0b_conv-dev.annot.kwlist.xml
+    [dev]=/export/babel/data/scoring/IndusDB/IARPA-babel307b-v1.0b_conv-dev/IARPA-babel307b-v1.0b_conv-dev.annot.kwlist.xml
+    [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel307b-v1.0b_conv-dev.kwlist4.xml
 )  # dev2h_kwlists
 dev2h_nj=16
 dev2h_subset_ecf=true
@@ -28,7 +29,8 @@ dev10h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel307b-v1.0b_conv-d
 dev10h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel307b-v1.0b_conv-dev/IARPA-babel307b-v1.0b_conv-dev.scoring.ecf.xml
 dev10h_stm_file=/export/babel/data/scoring/IndusDB/IARPA-babel307b-v1.0b_conv-dev/IARPA-babel307b-v1.0b_conv-dev.stm
 dev10h_kwlists=(
-    [kwlist]=/export/babel/data/scoring/IndusDB/IARPA-babel307b-v1.0b_conv-dev/IARPA-babel307b-v1.0b_conv-dev.annot.kwlist.xml
+    [dev]=/export/babel/data/scoring/IndusDB/IARPA-babel307b-v1.0b_conv-dev/IARPA-babel307b-v1.0b_conv-dev.annot.kwlist.xml
+    [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel307b-v1.0b_conv-dev.kwlist4.xml
 )  # dev10h_kwlists
 dev10h_nj=32
 
@@ -42,5 +44,9 @@ unsup_nj=32
 lexicon_file=/export/babel/data/307-amharic/IARPA-babel307b-v1.0b-build/BABEL_OP3_307/conversational/reference_materials/lexicon.txt
 lexiconFlags="--romanized --oov <unk>"
 
+extlex_proxy_phone_beam=5
+extlex_proxy_phone_nbest=300
+extlex_proxy_beam=-1
+extlex_proxy_nbest=-1
 
 

egs/babel/s5d/conf/lang/401-mongolian.FLP.official.conf

@@ -15,7 +15,8 @@ dev2h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel401b-v2.0b_conv-de
 dev2h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel401b-v2.0b_conv-dev/IARPA-babel401b-v2.0b_conv-dev.scoring.ecf.xml
 dev2h_stm_file=/export/babel/data/scoring/IndusDB/IARPA-babel401b-v2.0b_conv-dev/IARPA-babel401b-v2.0b_conv-dev.stm
 dev2h_kwlists=(
-    [kwlist]=/export/babel/data/scoring/IndusDB/IARPA-babel401b-v2.0b_conv-dev/IARPA-babel401b-v2.0b_conv-dev.annot.kwlist.xml
+    [dev]=/export/babel/data/scoring/IndusDB/IARPA-babel401b-v2.0b_conv-dev/IARPA-babel401b-v2.0b_conv-dev.annot.kwlist.xml
+    [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel401b-v2.0b_conv-dev.kwlist3.xml
 )  # dev2h_kwlists
 dev2h_nj=16
 dev2h_subset_ecf=true
@@ -28,7 +29,8 @@ dev10h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel401b-v2.0b_conv-d
 dev10h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel401b-v2.0b_conv-dev/IARPA-babel401b-v2.0b_conv-dev.scoring.ecf.xml
 dev10h_stm_file=/export/babel/data/scoring/IndusDB/IARPA-babel401b-v2.0b_conv-dev/IARPA-babel401b-v2.0b_conv-dev.stm
 dev10h_kwlists=(
-    [kwlist]=/export/babel/data/scoring/IndusDB/IARPA-babel401b-v2.0b_conv-dev/IARPA-babel401b-v2.0b_conv-dev.annot.kwlist.xml
+    [dev]=/export/babel/data/scoring/IndusDB/IARPA-babel401b-v2.0b_conv-dev/IARPA-babel401b-v2.0b_conv-dev.annot.kwlist.xml
+    [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel401b-v2.0b_conv-dev.kwlist3.xml
 )  # dev10h_kwlists
 dev10h_nj=32
 

egs/babel/s5d/conf/lang/402-javanese.FLP.official.conf

@@ -15,7 +15,8 @@ dev2h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel402b-v1.0b_conv-de
 dev2h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel402b-v1.0b_conv-dev/IARPA-babel402b-v1.0b_conv-dev.scoring.ecf.xml
 dev2h_stm_file=/export/babel/data/scoring/IndusDB/IARPA-babel402b-v1.0b_conv-dev/IARPA-babel402b-v1.0b_conv-dev.stm
 dev2h_kwlists=(
-    [kwlist]=/export/babel/data/scoring/IndusDB/IARPA-babel402b-v1.0b_conv-dev/IARPA-babel402b-v1.0b_conv-dev.annot.kwlist.xml
+    [dev]=/export/babel/data/scoring/IndusDB/IARPA-babel402b-v1.0b_conv-dev/IARPA-babel402b-v1.0b_conv-dev.annot.kwlist.xml
+    [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel402b-v1.0b_conv-dev.kwlist3.xml
 )  # dev2h_kwlists
 dev2h_nj=16
 dev2h_subset_ecf=true
@@ -28,9 +29,8 @@ dev10h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel402b-v1.0b_conv-d
 dev10h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel402b-v1.0b_conv-dev/IARPA-babel402b-v1.0b_conv-dev.scoring.ecf.xml
 dev10h_stm_file=/export/babel/data/scoring/IndusDB/IARPA-babel402b-v1.0b_conv-dev/IARPA-babel402b-v1.0b_conv-dev.stm
 dev10h_kwlists=(
-    [kwlist]=/export/babel/data/scoring/IndusDB/IARPA-babel402b-v1.0b_conv-dev/IARPA-babel402b-v1.0b_conv-dev.annot.kwlist.xml
-    [kwlist3]=/export/babel/data/scoring/IndusDB/IARPA-babel402b-v1.0b_conv-dev/IARPA-babel402b-v1.0b_conv-dev.annot.kwlist3.xml
-
+    [dev]=/export/babel/data/scoring/IndusDB/IARPA-babel402b-v1.0b_conv-dev/IARPA-babel402b-v1.0b_conv-dev.annot.kwlist.xml
+    [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel402b-v1.0b_conv-dev.kwlist3.xml
 )  # dev10h_kwlists
 dev10h_nj=32
 

egs/babel/s5d/conf/lang/403-dholuo.FLP.official.conf

@@ -15,7 +15,8 @@ dev2h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel403b-v1.0b_conv-de
 dev2h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel403b-v1.0b_conv-dev/IARPA-babel403b-v1.0b_conv-dev.scoring.ecf.xml
 dev2h_stm_file=/export/babel/data/scoring/IndusDB/IARPA-babel403b-v1.0b_conv-dev/IARPA-babel403b-v1.0b_conv-dev.stm
 dev2h_kwlists=(
-    [kwlist]=/export/babel/data/scoring/IndusDB/IARPA-babel403b-v1.0b_conv-dev/IARPA-babel403b-v1.0b_conv-dev.annot.kwlist.xml
+    [dev]=/export/babel/data/scoring/IndusDB/IARPA-babel403b-v1.0b_conv-dev/IARPA-babel403b-v1.0b_conv-dev.annot.kwlist.xml
+    [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel403b-v1.0b_conv-dev.kwlist4.xml
 )  # dev2h_kwlists
 dev2h_nj=16
 dev2h_subset_ecf=true
@@ -28,7 +29,8 @@ dev10h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel403b-v1.0b_conv-d
 dev10h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel403b-v1.0b_conv-dev/IARPA-babel403b-v1.0b_conv-dev.scoring.ecf.xml
 dev10h_stm_file=/export/babel/data/scoring/IndusDB/IARPA-babel403b-v1.0b_conv-dev/IARPA-babel403b-v1.0b_conv-dev.stm
 dev10h_kwlists=(
-    [kwlist]=/export/babel/data/scoring/IndusDB/IARPA-babel403b-v1.0b_conv-dev/IARPA-babel403b-v1.0b_conv-dev.annot.kwlist.xml
+    [dev]=/export/babel/data/scoring/IndusDB/IARPA-babel403b-v1.0b_conv-dev/IARPA-babel403b-v1.0b_conv-dev.annot.kwlist.xml
+    [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel403b-v1.0b_conv-dev.kwlist4.xml
 )  # dev10h_kwlists
 dev10h_nj=32
 

egs/babel/s5d/conf/lang/404-georgian.FLP.official.conf

+# include common settings for fullLP systems.
+. conf/common.fullLP || exit 1;
+
+
+#speech corpora files location
+train_data_dir=/export/babel/data//404-georgian/IARPA-babel404b-v1.0a-build/BABEL_OP3_404/conversational/training
+train_data_list=./conf/lists/404-georgian//training.list
+train_nj=32
+
+
+#Radical reduced DEV corpora files location
+dev2h_data_dir=/export/babel/data//404-georgian/IARPA-babel404b-v1.0a-build/BABEL_OP3_404/conversational/dev
+dev2h_data_list=./conf/lists/404-georgian//dev.2h.list
+dev2h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-dev/IARPA-babel404b-v1.0a_conv-dev.mitllfa3.rttm
+dev2h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-dev/IARPA-babel404b-v1.0a_conv-dev.scoring.ecf.xml
+dev2h_stm_file=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-dev/IARPA-babel404b-v1.0a_conv-dev.stm
+dev2h_kwlists=(
+    [dev]=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-dev/IARPA-babel404b-v1.0a_conv-dev.annot.kwlist.xml
+    [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-dev.kwlist3.xml
+)  # dev2h_kwlists
+dev2h_nj=16
+dev2h_subset_ecf=true
+
+
+#Official DEV corpora files location
+dev10h_data_dir=/export/babel/data//404-georgian/IARPA-babel404b-v1.0a-build/BABEL_OP3_404/conversational/dev
+dev10h_data_list=./conf/lists/404-georgian//dev.list
+dev10h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-dev/IARPA-babel404b-v1.0a_conv-dev.mitllfa3.rttm
+dev10h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-dev/IARPA-babel404b-v1.0a_conv-dev.scoring.ecf.xml
+dev10h_stm_file=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-dev/IARPA-babel404b-v1.0a_conv-dev.stm
+dev10h_kwlists=(
+    [dev]=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-dev/IARPA-babel404b-v1.0a_conv-dev.annot.kwlist.xml
+    [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-dev.kwlist3.xml
+)  # dev10h_kwlists
+dev10h_nj=32
+
+
+#Official EVAL period evaluation data files
+eval_data_dir=/export/babel/data//404-georgian/IARPA-babel404b-v1.0a-build/BABEL_OP3_404/conversational/eval
+eval_data_list=./conf/lists/404-georgian//eval.list
+eval_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-eval.ecf.xml
+eval_kwlists=(
+    [dev]=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-dev/IARPA-babel404b-v1.0a_conv-dev.annot.kwlist.xml
+    [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-dev.kwlist3.xml
+)  # eval_kwlists
+eval_nj=32
+
+
+#Shadow data files
+shadow_data_dir=(
+    /export/babel/data//404-georgian/IARPA-babel404b-v1.0a-build/BABEL_OP3_404/conversational/dev
+    /export/babel/data//404-georgian/IARPA-babel404b-v1.0a-build/BABEL_OP3_404/conversational/eval
+) # shadow_data_dir
+shadow_data_list=(
+    ./conf/lists/404-georgian//dev.list
+    ./conf/lists/404-georgian//eval.list
+) # shadow_data_dir
+shadow_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-dev/IARPA-babel404b-v1.0a_conv-dev.scoring.ecf.xml
+shadow_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-dev/IARPA-babel404b-v1.0a_conv-dev.mitllfa3.rttm
+shadow_stm_file=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-dev/IARPA-babel404b-v1.0a_conv-dev.stm
+shadow_kwlists=(
+    [dev]=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-dev/IARPA-babel404b-v1.0a_conv-dev.annot.kwlist.xml
+    [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-dev.kwlist3.xml
+)  # shadow_kwlists
+shadow_nj=32
+
+
+#Unsupervised dataset for FullLP condition
+unsup_data_dir=/export/babel/data//404-georgian/IARPA-babel404b-v1.0a-build/BABEL_OP3_404/conversational/untranscribed-training
+unsup_data_list=./conf/lists/404-georgian//untranscribed-training.list
+unsup_nj=32
+
+
+lexicon_file=
+lexiconFlags="--romanized --oov <unk>"
+
+
+

egs/babel/s5d/conf/lang/404-georgian.LLP.official.conf

+# include common settings for fullLP systems.
+. conf/common.limitedLP || exit 1;
+
+
+#speech corpora files location
+train_data_dir=/export/babel/data//404-georgian/IARPA-babel404b-v1.0a-build/BABEL_OP3_404/conversational/training
+train_data_list=./conf/lists/404-georgian//sub-train.list
+train_nj=32
+
+
+#Radical reduced DEV corpora files location
+dev2h_data_dir=/export/babel/data//404-georgian/IARPA-babel404b-v1.0a-build/BABEL_OP3_404/conversational/dev
+dev2h_data_list=./conf/lists/404-georgian//dev.2h.list
+dev2h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-dev/IARPA-babel404b-v1.0a_conv-dev.mitllfa3.rttm
+dev2h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-dev/IARPA-babel404b-v1.0a_conv-dev.scoring.ecf.xml
+dev2h_stm_file=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-dev/IARPA-babel404b-v1.0a_conv-dev.stm
+dev2h_kwlists=(
+    [dev]=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-dev/IARPA-babel404b-v1.0a_conv-dev.annot.kwlist.xml
+    [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-dev/IARPA-babel404b-v1.0a_conv-dev.annot.kwlist3.xml
+)  # dev2h_kwlists
+dev2h_nj=16
+dev2h_subset_ecf=true
+
+
+#Official DEV corpora files location
+dev10h_data_dir=/export/babel/data//404-georgian/IARPA-babel404b-v1.0a-build/BABEL_OP3_404/conversational/dev
+dev10h_data_list=./conf/lists/404-georgian//dev.list
+dev10h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-dev/IARPA-babel404b-v1.0a_conv-dev.mitllfa3.rttm
+dev10h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-dev/IARPA-babel404b-v1.0a_conv-dev.scoring.ecf.xml
+dev10h_stm_file=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-dev/IARPA-babel404b-v1.0a_conv-dev.stm
+dev10h_kwlists=(
+    [dev]=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-dev/IARPA-babel404b-v1.0a_conv-dev.annot.kwlist.xml
+    [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel404b-v1.0a_conv-dev/IARPA-babel404b-v1.0a_conv-dev.annot.kwlist3.xml
+)  # dev10h_kwlists
+dev10h_nj=32
+
+
+#Unsupervised dataset for LimitedLP condition
+unsup_data_list=(
+    ./conf/lists/404-georgian//untranscribed-training.list
+    ./conf/lists/404-georgian//sub-train.untranscribed.list
+)  # unsup_data_list
+unsup_data_dir=(
+    /export/babel/data//404-georgian/IARPA-babel404b-v1.0a-build/BABEL_OP3_404/conversational/untranscribed-training
+    /export/babel/data//404-georgian/IARPA-babel404b-v1.0a-build/BABEL_OP3_404/conversational/training
+)  # unsup_data_dir
+unsup_nj=32
+
+
+lexicon_file=
+lexiconFlags="--romanized --oov <unk>"
+
+
+

egs/babel/s5d/conf/lists/404-georgian/dev.2h.list

+BABEL_OP3_404_10184_20141107_212406_inLine
+BABEL_OP3_404_10184_20141107_212406_outLine
+BABEL_OP3_404_12851_20141013_024620_inLine
+BABEL_OP3_404_12851_20141013_024620_outLine
+BABEL_OP3_404_16184_20141020_233508_inLine
+BABEL_OP3_404_16184_20141020_233508_outLine
+BABEL_OP3_404_17165_20141117_063008_inLine
+BABEL_OP3_404_17165_20141117_063008_outLine
+BABEL_OP3_404_17472_20141201_023731_inLine
+BABEL_OP3_404_17472_20141201_023731_outLine
+BABEL_OP3_404_18380_20141118_001754_inLine
+BABEL_OP3_404_18380_20141118_001754_outLine
+BABEL_OP3_404_18939_20141009_063127_inLine
+BABEL_OP3_404_18939_20141009_063127_outLine
+BABEL_OP3_404_22446_20141013_062554_inLine
+BABEL_OP3_404_22446_20141013_062554_outLine
+BABEL_OP3_404_22466_20141018_193129_inLine
+BABEL_OP3_404_22466_20141018_193129_outLine
+BABEL_OP3_404_22494_20141127_221208_inLine
+BABEL_OP3_404_22494_20141127_221208_outLine
+BABEL_OP3_404_22494_20141127_222057_inLine
+BABEL_OP3_404_22494_20141127_222057_outLine
+BABEL_OP3_404_23239_20141127_054155_inLine
+BABEL_OP3_404_23239_20141127_054155_outLine
+BABEL_OP3_404_24253_20150513_212152_inLine
+BABEL_OP3_404_24253_20150513_212152_outLine
+BABEL_OP3_404_24779_20150620_032949_inLine
+BABEL_OP3_404_24779_20150620_032949_outLine
+BABEL_OP3_404_26074_20141120_050650_inLine
+BABEL_OP3_404_26074_20141120_050650_outLine
+BABEL_OP3_404_28419_20141028_024104_inLine
+BABEL_OP3_404_28419_20141028_024104_outLine
+BABEL_OP3_404_33476_20141114_205102_inLine
+BABEL_OP3_404_33476_20141114_205102_outLine
+BABEL_OP3_404_34564_20141211_015413_inLine
+BABEL_OP3_404_34564_20141211_015413_outLine
+BABEL_OP3_404_35467_20141020_054030_inLine
+BABEL_OP3_404_35467_20141020_054030_outLine
+BABEL_OP3_404_38431_20141130_190122_inLine
+BABEL_OP3_404_38431_20141130_190122_outLine
+BABEL_OP3_404_41592_20141117_033328_inLine
+BABEL_OP3_404_41592_20141117_033328_outLine
+BABEL_OP3_404_41741_20141019_015552_inLine
+BABEL_OP3_404_41741_20141019_015552_outLine
+BABEL_OP3_404_42231_20141130_013425_inLine
+BABEL_OP3_404_42231_20141130_013425_outLine
+BABEL_OP3_404_42231_20141130_014628_inLine
+BABEL_OP3_404_42231_20141130_014628_outLine
+BABEL_OP3_404_42600_20141029_174857_inLine
+BABEL_OP3_404_42600_20141029_174857_outLine
+BABEL_OP3_404_44619_20141028_234639_inLine
+BABEL_OP3_404_44619_20141028_234639_outLine
+BABEL_OP3_404_46535_20150216_024618_inLine
+BABEL_OP3_404_46535_20150216_024618_outLine
+BABEL_OP3_404_46757_20141123_021510_inLine
+BABEL_OP3_404_46757_20141123_021510_outLine
+BABEL_OP3_404_47487_20141030_235808_inLine
+BABEL_OP3_404_47487_20141030_235808_outLine
+BABEL_OP3_404_47866_20150526_162411_inLine
+BABEL_OP3_404_47866_20150526_162411_outLine
+BABEL_OP3_404_47959_20141026_214447_inLine
+BABEL_OP3_404_47959_20141026_214447_outLine
+BABEL_OP3_404_51955_20141024_012212_inLine
+BABEL_OP3_404_51955_20141024_012212_outLine
+BABEL_OP3_404_51968_20141117_023015_inLine
+BABEL_OP3_404_51968_20141117_023015_outLine
+BABEL_OP3_404_52804_20141023_174815_inLine
+BABEL_OP3_404_52804_20141023_174815_outLine
+BABEL_OP3_404_54567_20141119_040337_inLine
+BABEL_OP3_404_54567_20141119_040337_outLine
+BABEL_OP3_404_56677_20141201_065523_inLine
+BABEL_OP3_404_56677_20141201_065523_outLine
+BABEL_OP3_404_56826_20141201_042429_inLine
+BABEL_OP3_404_56826_20141201_042429_outLine
+BABEL_OP3_404_58047_20141110_215330_inLine
+BABEL_OP3_404_58047_20141110_215330_outLine
+BABEL_OP3_404_58313_20141119_234202_inLine
+BABEL_OP3_404_58313_20141119_234202_outLine
+BABEL_OP3_404_59549_20141102_190355_inLine
+BABEL_OP3_404_59549_20141102_190355_outLine
+BABEL_OP3_404_60307_20150625_022621_inLine
+BABEL_OP3_404_60307_20150625_022621_outLine
+BABEL_OP3_404_61040_20141211_011552_inLine
+BABEL_OP3_404_61040_20141211_011552_outLine
+BABEL_OP3_404_61190_20141029_013447_inLine
+BABEL_OP3_404_61190_20141029_013447_outLine
+BABEL_OP3_404_64638_20141130_205157_inLine
+BABEL_OP3_404_64638_20141130_205157_outLine
+BABEL_OP3_404_66472_20141107_204602_inLine
+BABEL_OP3_404_66472_20141107_204602_outLine
+BABEL_OP3_404_66519_20141031_015751_inLine
+BABEL_OP3_404_66519_20141031_015751_outLine
+BABEL_OP3_404_67794_20141103_023323_inLine
+BABEL_OP3_404_67794_20141103_023323_outLine
+BABEL_OP3_404_73696_20150618_060036_inLine
+BABEL_OP3_404_73696_20150618_060036_outLine
+BABEL_OP3_404_73757_20141117_025704_inLine
+BABEL_OP3_404_73757_20141117_025704_outLine
+BABEL_OP3_404_74121_20141120_020705_inLine
+BABEL_OP3_404_74121_20141120_020705_outLine
+BABEL_OP3_404_80781_20141104_212234_inLine
+BABEL_OP3_404_80781_20141104_212234_outLine
+BABEL_OP3_404_80881_20141010_222135_inLine
+BABEL_OP3_404_80881_20141010_222135_outLine
+BABEL_OP3_404_81424_20141123_000421_inLine
+BABEL_OP3_404_81424_20141123_000421_outLine
+BABEL_OP3_404_87298_20141025_213601_inLine
+BABEL_OP3_404_87298_20141025_213601_outLine
+BABEL_OP3_404_87313_20141119_014632_inLine
+BABEL_OP3_404_87313_20141119_014632_outLine
+BABEL_OP3_404_87796_20141120_065537_inLine
+BABEL_OP3_404_87796_20141120_065537_outLine
+BABEL_OP3_404_87884_20141128_211555_inLine
+BABEL_OP3_404_87884_20141128_211555_outLine
+BABEL_OP3_404_88776_20141006_193621_inLine
+BABEL_OP3_404_88776_20141006_193621_outLine
+BABEL_OP3_404_91760_20150609_033824_inLine
+BABEL_OP3_404_91760_20150609_033824_outLine
+BABEL_OP3_404_91930_20150522_034521_inLine
+BABEL_OP3_404_91930_20150522_034521_outLine
+BABEL_OP3_404_92740_20141126_025242_inLine
+BABEL_OP3_404_92740_20141126_025242_outLine
+BABEL_OP3_404_97376_20141126_024552_inLine
+BABEL_OP3_404_97376_20141126_024552_outLine

egs/babel/s5d/conf/lists/404-georgian/dev.list

+BABEL_OP3_404_10184_20141107_212406_inLine
+BABEL_OP3_404_10184_20141107_212406_outLine
+BABEL_OP3_404_12851_20141013_024620_inLine
+BABEL_OP3_404_12851_20141013_024620_outLine
+BABEL_OP3_404_16184_20141020_233508_inLine
+BABEL_OP3_404_16184_20141020_233508_outLine
+BABEL_OP3_404_17165_20141117_063008_inLine
+BABEL_OP3_404_17165_20141117_063008_outLine
+BABEL_OP3_404_17472_20141201_023731_inLine
+BABEL_OP3_404_17472_20141201_023731_outLine
+BABEL_OP3_404_18380_20141118_001754_inLine
+BABEL_OP3_404_18380_20141118_001754_outLine
+BABEL_OP3_404_18939_20141009_063127_inLine
+BABEL_OP3_404_18939_20141009_063127_outLine
+BABEL_OP3_404_22446_20141013_062554_inLine
+BABEL_OP3_404_22446_20141013_062554_outLine
+BABEL_OP3_404_22466_20141018_193129_inLine
+BABEL_OP3_404_22466_20141018_193129_outLine
+BABEL_OP3_404_22494_20141127_221208_inLine
+BABEL_OP3_404_22494_20141127_221208_outLine
+BABEL_OP3_404_22494_20141127_222057_inLine
+BABEL_OP3_404_22494_20141127_222057_outLine
+BABEL_OP3_404_23239_20141127_054155_inLine
+BABEL_OP3_404_23239_20141127_054155_outLine
+BABEL_OP3_404_24253_20150513_212152_inLine
+BABEL_OP3_404_24253_20150513_212152_outLine
+BABEL_OP3_404_24779_20150620_032949_inLine
+BABEL_OP3_404_24779_20150620_032949_outLine
+BABEL_OP3_404_26074_20141120_050650_inLine
+BABEL_OP3_404_26074_20141120_050650_outLine
+BABEL_OP3_404_28419_20141028_024104_inLine
+BABEL_OP3_404_28419_20141028_024104_outLine
+BABEL_OP3_404_33476_20141114_205102_inLine
+BABEL_OP3_404_33476_20141114_205102_outLine
+BABEL_OP3_404_34564_20141211_015413_inLine
+BABEL_OP3_404_34564_20141211_015413_outLine
+BABEL_OP3_404_35467_20141020_054030_inLine
+BABEL_OP3_404_35467_20141020_054030_outLine
+BABEL_OP3_404_38431_20141130_190122_inLine
+BABEL_OP3_404_38431_20141130_190122_outLine
+BABEL_OP3_404_41592_20141117_033328_inLine
+BABEL_OP3_404_41592_20141117_033328_outLine
+BABEL_OP3_404_41741_20141019_015552_inLine
+BABEL_OP3_404_41741_20141019_015552_outLine
+BABEL_OP3_404_42231_20141130_013425_inLine
+BABEL_OP3_404_42231_20141130_013425_outLine
+BABEL_OP3_404_42231_20141130_014628_inLine
+BABEL_OP3_404_42231_20141130_014628_outLine
+BABEL_OP3_404_42600_20141029_174857_inLine
+BABEL_OP3_404_42600_20141029_174857_outLine
+BABEL_OP3_404_44619_20141028_234639_inLine
+BABEL_OP3_404_44619_20141028_234639_outLine
+BABEL_OP3_404_46535_20150216_024618_inLine
+BABEL_OP3_404_46535_20150216_024618_outLine
+BABEL_OP3_404_46757_20141123_021510_inLine
+BABEL_OP3_404_46757_20141123_021510_outLine
+BABEL_OP3_404_47487_20141030_235808_inLine
+BABEL_OP3_404_47487_20141030_235808_outLine
+BABEL_OP3_404_47866_20150526_162411_inLine
+BABEL_OP3_404_47866_20150526_162411_outLine
+BABEL_OP3_404_47959_20141026_214447_inLine
+BABEL_OP3_404_47959_20141026_214447_outLine
+BABEL_OP3_404_51955_20141024_012212_inLine
+BABEL_OP3_404_51955_20141024_012212_outLine
+BABEL_OP3_404_51968_20141117_023015_inLine
+BABEL_OP3_404_51968_20141117_023015_outLine
+BABEL_OP3_404_52804_20141023_174815_inLine
+BABEL_OP3_404_52804_20141023_174815_outLine
+BABEL_OP3_404_54567_20141119_040337_inLine
+BABEL_OP3_404_54567_20141119_040337_outLine
+BABEL_OP3_404_56677_20141201_065523_inLine
+BABEL_OP3_404_56677_20141201_065523_outLine
+BABEL_OP3_404_56826_20141201_042429_inLine
+BABEL_OP3_404_56826_20141201_042429_outLine
+BABEL_OP3_404_58047_20141110_215330_inLine
+BABEL_OP3_404_58047_20141110_215330_outLine
+BABEL_OP3_404_58313_20141119_234202_inLine
+BABEL_OP3_404_58313_20141119_234202_outLine
+BABEL_OP3_404_59549_20141102_190355_inLine
+BABEL_OP3_404_59549_20141102_190355_outLine
+BABEL_OP3_404_60307_20150625_022621_inLine
+BABEL_OP3_404_60307_20150625_022621_outLine
+BABEL_OP3_404_61040_20141211_011552_inLine
+BABEL_OP3_404_61040_20141211_011552_outLine
+BABEL_OP3_404_61190_20141029_013447_inLine
+BABEL_OP3_404_61190_20141029_013447_outLine
+BABEL_OP3_404_64638_20141130_205157_inLine
+BABEL_OP3_404_64638_20141130_205157_outLine
+BABEL_OP3_404_66472_20141107_204602_inLine
+BABEL_OP3_404_66472_20141107_204602_outLine
+BABEL_OP3_404_66519_20141031_015751_inLine
+BABEL_OP3_404_66519_20141031_015751_outLine
+BABEL_OP3_404_67794_20141103_023323_inLine
+BABEL_OP3_404_67794_20141103_023323_outLine
+BABEL_OP3_404_73696_20150618_060036_inLine
+BABEL_OP3_404_73696_20150618_060036_outLine
+BABEL_OP3_404_73757_20141117_025704_inLine
+BABEL_OP3_404_73757_20141117_025704_outLine
+BABEL_OP3_404_74121_20141120_020705_inLine
+BABEL_OP3_404_74121_20141120_020705_outLine
+BABEL_OP3_404_80781_20141104_212234_inLine
+BABEL_OP3_404_80781_20141104_212234_outLine
+BABEL_OP3_404_80881_20141010_222135_inLine
+BABEL_OP3_404_80881_20141010_222135_outLine
+BABEL_OP3_404_81424_20141123_000421_inLine
+BABEL_OP3_404_81424_20141123_000421_outLine
+BABEL_OP3_404_87298_20141025_213601_inLine
+BABEL_OP3_404_87298_20141025_213601_outLine
+BABEL_OP3_404_87313_20141119_014632_inLine
+BABEL_OP3_404_87313_20141119_014632_outLine
+BABEL_OP3_404_87796_20141120_065537_inLine
+BABEL_OP3_404_87796_20141120_065537_outLine
+BABEL_OP3_404_87884_20141128_211555_inLine
+BABEL_OP3_404_87884_20141128_211555_outLine
+BABEL_OP3_404_88776_20141006_193621_inLine
+BABEL_OP3_404_88776_20141006_193621_outLine
+BABEL_OP3_404_91760_20150609_033824_inLine
+BABEL_OP3_404_91760_20150609_033824_outLine
+BABEL_OP3_404_91930_20150522_034521_inLine
+BABEL_OP3_404_91930_20150522_034521_outLine
+BABEL_OP3_404_92740_20141126_025242_inLine
+BABEL_OP3_404_92740_20141126_025242_outLine
+BABEL_OP3_404_97376_20141126_024552_inLine
+BABEL_OP3_404_97376_20141126_024552_outLine

egs/babel/s5d/conf/lists/404-georgian/sub-train.list

+BABEL_OP3_404_11663_20141118_032146_inLine
+BABEL_OP3_404_11663_20141118_032146_outLine
+BABEL_OP3_404_12242_20141028_021853_inLine
+BABEL_OP3_404_12242_20141028_021853_outLine
+BABEL_OP3_404_13178_20141129_192909_inLine
+BABEL_OP3_404_13178_20141129_192909_outLine
+BABEL_OP3_404_14137_20141025_202817_inLine
+BABEL_OP3_404_14137_20141025_202817_outLine
+BABEL_OP3_404_14875_20141026_230227_inLine
+BABEL_OP3_404_14875_20141026_230227_outLine
+BABEL_OP3_404_15869_20150218_225936_inLine
+BABEL_OP3_404_15869_20150218_225936_outLine
+BABEL_OP3_404_17113_20150611_050102_inLine
+BABEL_OP3_404_17113_20150611_050102_outLine
+BABEL_OP3_404_23505_20141021_032033_inLine
+BABEL_OP3_404_23505_20141021_032033_outLine
+BABEL_OP3_404_24470_20141111_184651_inLine
+BABEL_OP3_404_24470_20141111_184651_outLine
+BABEL_OP3_404_24470_20141111_190229_inLine
+BABEL_OP3_404_24470_20141111_190229_outLine
+BABEL_OP3_404_24679_20141018_015615_inLine
+BABEL_OP3_404_24679_20141018_015615_outLine
+BABEL_OP3_404_26388_20141026_014207_inLine
+BABEL_OP3_404_26388_20141026_014207_outLine
+BABEL_OP3_404_27042_20141201_215107_inLine
+BABEL_OP3_404_27042_20141201_215107_outLine
+BABEL_OP3_404_28538_20141119_005526_inLine
+BABEL_OP3_404_28538_20141119_005526_outLine
+BABEL_OP3_404_29208_20141106_013309_inLine
+BABEL_OP3_404_29208_20141106_013309_outLine
+BABEL_OP3_404_30461_20150620_020316_inLine
+BABEL_OP3_404_30461_20150620_020316_outLine
+BABEL_OP3_404_31979_20141106_000523_inLine
+BABEL_OP3_404_31979_20141106_000523_outLine
+BABEL_OP3_404_31992_20141014_221817_inLine
+BABEL_OP3_404_31992_20141014_221817_outLine
+BABEL_OP3_404_37064_20141102_063308_inLine
+BABEL_OP3_404_37064_20141102_063308_outLine
+BABEL_OP3_404_37281_20141119_053453_inLine
+BABEL_OP3_404_37281_20141119_053453_outLine
+BABEL_OP3_404_37853_20150602_030625_inLine
+BABEL_OP3_404_37853_20150602_030625_outLine
+BABEL_OP3_404_40713_20141028_221207_inLine
+BABEL_OP3_404_40713_20141028_221207_outLine
+BABEL_OP3_404_41680_20141012_040411_inLine
+BABEL_OP3_404_41680_20141012_040411_outLine
+BABEL_OP3_404_41920_20141008_040539_inLine
+BABEL_OP3_404_41920_20141008_040539_outLine
+BABEL_OP3_404_42877_20150212_052937_inLine
+BABEL_OP3_404_42877_20150212_052937_outLine
+BABEL_OP3_404_45121_20150609_055234_inLine
+BABEL_OP3_404_45121_20150609_055234_outLine
+BABEL_OP3_404_46169_20141130_224339_inLine
+BABEL_OP3_404_46169_20141130_224339_outLine
+BABEL_OP3_404_46625_20141011_040505_inLine
+BABEL_OP3_404_46625_20141011_040505_outLine
+BABEL_OP3_404_46681_20141021_040451_inLine
+BABEL_OP3_404_46681_20141021_040451_outLine
+BABEL_OP3_404_47270_20150512_053415_inLine
+BABEL_OP3_404_47270_20150512_053415_outLine
+BABEL_OP3_404_48844_20141020_065414_inLine
+BABEL_OP3_404_48844_20141020_065414_outLine
+BABEL_OP3_404_49768_20141026_022902_inLine
+BABEL_OP3_404_49768_20141026_022902_outLine
+BABEL_OP3_404_50175_20141021_025726_inLine
+BABEL_OP3_404_50175_20141021_025726_outLine
+BABEL_OP3_404_52301_20141009_051739_inLine
+BABEL_OP3_404_52301_20141009_051739_outLine
+BABEL_OP3_404_52301_20141009_054049_inLine
+BABEL_OP3_404_52301_20141009_054049_outLine
+BABEL_OP3_404_52490_20141016_020323_inLine
+BABEL_OP3_404_52490_20141016_020323_outLine
+BABEL_OP3_404_56213_20141201_000837_inLine
+BABEL_OP3_404_56213_20141201_000837_outLine
+BABEL_OP3_404_58103_20141030_002209_inLine
+BABEL_OP3_404_58103_20141030_002209_outLine
+BABEL_OP3_404_59078_20141111_004941_inLine
+BABEL_OP3_404_59078_20141111_004941_outLine
+BABEL_OP3_404_61225_20141009_174003_inLine
+BABEL_OP3_404_61225_20141009_174003_outLine
+BABEL_OP3_404_63220_20141127_033605_inLine
+BABEL_OP3_404_63220_20141127_033605_outLine
+BABEL_OP3_404_64494_20141026_203549_inLine
+BABEL_OP3_404_64494_20141026_203549_outLine
+BABEL_OP3_404_64768_20141027_201818_inLine
+BABEL_OP3_404_64768_20141027_201818_outLine
+BABEL_OP3_404_66916_20141022_000731_inLine
+BABEL_OP3_404_66916_20141022_000731_outLine
+BABEL_OP3_404_67401_20141109_211809_inLine
+BABEL_OP3_404_67401_20141109_211809_outLine
+BABEL_OP3_404_68059_20141109_052011_inLine
+BABEL_OP3_404_68059_20141109_052011_outLine
+BABEL_OP3_404_68068_20141201_054518_inLine
+BABEL_OP3_404_68068_20141201_054518_outLine
+BABEL_OP3_404_68384_20141130_035214_inLine
+BABEL_OP3_404_68384_20141130_035214_outLine
+BABEL_OP3_404_68627_20141105_190511_inLine
+BABEL_OP3_404_68627_20141105_190511_outLine
+BABEL_OP3_404_72844_20141007_033837_inLine
+BABEL_OP3_404_72844_20141007_033837_outLine
+BABEL_OP3_404_73837_20141026_191037_inLine
+BABEL_OP3_404_73837_20141026_191037_outLine
+BABEL_OP3_404_78511_20141201_003606_inLine
+BABEL_OP3_404_78511_20141201_003606_outLine
+BABEL_OP3_404_79139_20141117_054733_inLine
+BABEL_OP3_404_79139_20141117_054733_outLine
+BABEL_OP3_404_81971_20141022_025641_inLine
+BABEL_OP3_404_81971_20141022_025641_outLine
+BABEL_OP3_404_83062_20150523_220236_inLine
+BABEL_OP3_404_83062_20150523_220236_outLine
+BABEL_OP3_404_83775_20141030_230742_inLine
+BABEL_OP3_404_83775_20141030_230742_outLine
+BABEL_OP3_404_84339_20150502_014143_inLine
+BABEL_OP3_404_84339_20150502_014143_outLine
+BABEL_OP3_404_86191_20141027_013544_inLine
+BABEL_OP3_404_86191_20141027_013544_outLine
+BABEL_OP3_404_86888_20141119_022459_inLine
+BABEL_OP3_404_86888_20141119_022459_outLine
+BABEL_OP3_404_95966_20141129_060246_inLine
+BABEL_OP3_404_95966_20141129_060246_outLine
+BABEL_OP3_404_97461_20141118_230730_inLine
+BABEL_OP3_404_97461_20141118_230730_outLine
+BABEL_OP3_404_99487_20141021_053024_inLine
+BABEL_OP3_404_99487_20141021_053024_outLine

egs/babel/s5d/local/arpa2G.sh

@@ -85,7 +85,8 @@ if [ ! -z "$oov_prob_file" ]; then
           print "$log10prob $word\n";
        }
      }} print STDERR "Ceilinged $ceilinged unk-probs\n";' \
-       $oov_prob_file $min_prob $unk_fraction | gzip -c > $destdir/lm_tmp.gz
+       $oov_prob_file $min_prob $unk_fraction | \
+  ngram  -unk -lm - -write-lm $destdir/lm_tmp.gz
   lmfile=$destdir/lm_tmp.gz
 fi