removing the redundant lattice with unigram composition tool,

adding tool to get frame-level confidences from posteriors.



git-svn-id: https://svn.code.sf.net/p/kaldi/code/trunk@2396 5e6a8d80-dfce-4ca6-a32a-6e07a63d50c8

egs/wsj/s5/steps/make_denlats_nnet_union.sh

@@ -140,7 +140,7 @@ echo "Generating the numerator ali and lattices with LM scores (to be unioned wi
 #1) Generate the tracing-fsts based on the lattice with correct paths
 $cmd JOB=1:$nj $dir/log/num_lat.JOB.log \
   compile-train-graphs $scale_opts $srcdir/tree $srcdir/final.mdl  $lang/L.fst "$tra" ark:- \| \
-  compose-with-unigram-fst $dir/lang/G.fst ark:- ark,t:- \| \
+  fsttablecompose ark:- $dir/lang/G.fst ark,t:- \| \
   latgen-faster-mapped --beam=$beam_ali --lattice-beam=$lattice_beam_ali --acoustic-scale=$acwt \
     --word-symbol-table=$lang/words.txt \
     $srcdir/final.mdl ark:- "$feats" ark,t:$dir/num_lat.ark.JOB ark:/dev/null "ark,t:|gzip -c >$dir/ali.JOB.gz"

src/bin/Makefile

@@ -20,7 +20,7 @@ BINFILES = align-equal align-equal-compiled acc-tree-stats \
         hmm-info pdf-to-counts analyze-counts extract-ctx post-to-phone-post \
         post-to-pdf-post duplicate-matrix logprob-to-post prob-to-post copy-post \
         matrix-logprob matrix-sum latgen-tracking-mapped \
-        build-pfile-from-ali compose-with-unigram-fst
+        build-pfile-from-ali get-post-on-ali
 
 OBJFILES = 
 

src/bin/compose-with-unigram-fst.cc

-// bin/compose-with-unigram-fst.cc
-
-// Copyright 2009-2012  Microsoft Corporation
-//           2012-2013  Johns Hopkins University (Author: Daniel Povey)
-
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
-// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
-// MERCHANTABLITY OR NON-INFRINGEMENT.
-// See the Apache 2 License for the specific language governing permissions and
-// limitations under the License.
-
-#include "base/kaldi-common.h"
-#include "util/common-utils.h"
-#include "fstext/fstext-lib.h"
-//#include "fst/fstlib.h"
-#include "fst/compose.h"
-#include "fst/arcsort.h"
-#include "fst/topsort.h"
-
-
-
-int main(int argc, char *argv[]) {
-  try {
-    using namespace kaldi;
-    typedef kaldi::int32 int32;
-    using fst::SymbolTable;
-    using fst::VectorFst;
-    using fst::StdArc;
-    using fst::ComposeFst;
-    using fst::ArcSort;
-    using fst::StdILabelCompare;
-    using fst::StdOLabelCompare;
-    using fst::TopSort;
-
-    const char *usage =
-        "Composes training graphs with unigram grammar FST (ie. injects the LM scores)\n"
-        "\n"
-        "Usage:   compose-with-unigram-fst [options] unigram-fst graphs-rspecifier graphs-wspecifier\n"
-        "e.g.: \n"
-        " compose-with-unigram-fst G.fst ark:graph_in.ark ark:graph_out.ark\n";
-
-    ParseOptions po(usage);
-    /* No configuration options */
-    po.Read(argc, argv);
-
-    if(po.NumArgs() != 3) {
-      po.PrintUsage();
-      exit(1);
-    }
-
-    std::string unigramfst_rxfilename = po.GetArg(1);
-    std::string graph_rspecifier = po.GetArg(2);
-    std::string graph_wspecifier = po.GetArg(3);
-
-    //Read the unigram fst
-    VectorFst<StdArc> *unigram_fst = fst::ReadFstKaldi(unigramfst_rxfilename);
-    //sort it along the input symbols
-    ArcSort(unigram_fst, StdILabelCompare());
-
-    //Iterate over the graphs
-    SequentialTableReader<fst::VectorFstHolder> graph_reader(graph_rspecifier);
-    TableWriter<fst::VectorFstHolder> graph_writer(graph_wspecifier);
-    int32 num_success = 0; 
-    for(; !graph_reader.Done(); graph_reader.Next()) {
-      //get the graph
-      std::string key = graph_reader.Key();
-      VectorFst<StdArc> graph_in = graph_reader.Value();
-      //sort the input graph along the output symbols
-      ArcSort(&graph_in, StdOLabelCompare());
-      //compose with unigram fst
-      VectorFst<StdArc> graph_out;
-
-      //KALDI_LOG << "gr_in: states" << graph_in.NumStates(); //<< ", arcs" << graph_in.NumArcs(0);
-      //KALDI_LOG << "unigr_in: states" << unigram_fst->NumStates(); // << ", arcs" << unigram_fst.NumArcs(0);
-      Compose(graph_in,*unigram_fst,&graph_out);
-      //KALDI_LOG << "gr_out: states" << graph_out.NumStates(); // << ", arcs" << graph_out.NumArcs(0);
-      //write it
-      graph_writer.Write(key,graph_out);
-      //count the files
-      num_success += 1;
-    }
-
-    delete unigram_fst;
-
-    KALDI_LOG << "Successfuly added unigram scores to " 
-              << num_success << " training graphs.";
-    
-    return 0;
-  } catch(const std::exception &e) {
-    std::cerr << e.what();
-    return -1;
-  }
-}
-
-
-
-
-

src/bin/get-post-on-ali.cc

+// bin/post-to-pdf-post.cc
+
+// Copyright 2012  Johns Hopkins University (Author: Daniel Povey)
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABLITY OR NON-INFRINGEMENT.
+// See the Apache 2 License for the specific language governing permissions and
+// limitations under the License.
+
+
+#include "base/kaldi-common.h"
+#include "util/common-utils.h"
+#include "gmm/am-diag-gmm.h"
+#include "hmm/transition-model.h"
+#include "hmm/hmm-utils.h"
+
+
+
+int main(int argc, char *argv[]) {
+  using namespace kaldi;
+  typedef kaldi::int32 int32;
+  try {
+    const char *usage =
+        "This program selects posteriors that are masked by alignment\n"
+        "(ie. posteriors that are under the alignment path)\n"
+        "This can be used as a per-frame confidence measure\n"
+        "It is better to use pdf-posteriors and pdf-alignments\n"
+        "\n"
+        "Usage:  get-post-on-ali [options] <posteriors-rspecifier> <ali-rspecifier> <conf-wspecifier>\n"
+        "e.g.: get-post-on-ali ark:post.ark ark:ali.ark ark:conf.ark\n";
+
+    ParseOptions po(usage);
+
+    po.Read(argc, argv);
+
+    if (po.NumArgs() != 3) {
+      po.PrintUsage();
+      exit(1);
+    }
+
+    std::string posteriors_rspecifier = po.GetArg(1),
+        alignments_rspecifier = po.GetArg(2),
+        confidences_wspecifier = po.GetArg(3);
+
+    int32 num_done = 0, num_no_alignment = 0;
+    SequentialPosteriorReader posterior_reader(posteriors_rspecifier);
+    RandomAccessInt32VectorReader alignments_reader(alignments_rspecifier);
+    BaseFloatVectorWriter confidences_writer(confidences_wspecifier);
+    
+    for (; !posterior_reader.Done(); posterior_reader.Next()) {
+      std::string key = posterior_reader.Key();
+      if (!alignments_reader.HasKey(key)) {
+        num_no_alignment++;
+      } else {
+        //get the posterior
+        const kaldi::Posterior &posterior = posterior_reader.Value();
+        int32 num_frames = static_cast<int32>(posterior.size());
+        //get the alignment
+        const std::vector<int32> &alignment = alignments_reader.Value(key);
+        //check the lengths match
+        KALDI_ASSERT(num_frames == alignment.size());
+
+        //fill the vector with posteriors on the alignment (under the alignment path)
+        Vector<BaseFloat> confidence(num_frames);
+        for(int32 i = 0; i < num_frames; i++) {
+          BaseFloat post_i = 0.0;
+          for(int32 j = 0; j < posterior[i].size(); j++) {
+            if(alignment[i] == posterior[i][j].first) {
+              post_i = posterior[i][j].second;
+              break;
+            }
+          }
+          confidence(i) = post_i;
+        }
+
+        //write the vector with confidences
+        confidences_writer.Write(key,confidence);
+        num_done++;
+      }
+    }
+    KALDI_LOG << "Done getting the posteriors under the alignment path for "
+              << num_done << " utterances. " << num_no_alignment << " with missing alignments.";
+
+  } catch(const std::exception &e) {
+    std::cerr << e.what();
+    return -1;
+  }
+}
+
+
+
+