Adding some internal code to the EventMap class, which was to be used for...

Adding some internal code to the EventMap class, which was to be used for mapping phone-sets -- this was created for something I now won't be doing, but the internal code may be useful in future so I'm committing it.

git-svn-id: https://svn.code.sf.net/p/kaldi/code/trunk@2495 5e6a8d80-dfce-4ca6-a32a-6e07a63d50c8

src/tree/context-dep.h

@@ -60,7 +60,8 @@ class ContextDependency: public ContextDependencyInterface {
 
 
   /// returns success or failure; outputs pdf to pdf_id
-  virtual bool Compute(const std::vector<int32> &phoneseq, int32 pdf_class, int32 *pdf_id) const;
+  virtual bool Compute(const std::vector<int32> &phoneseq,
+                       int32 pdf_class, int32 *pdf_id) const;
 
   virtual int32 NumPdfs() const {
     // this routine could be simplified to return to_pdf_->MaxResult()+1.  we're a

src/tree/event-map-test.cc

 // tree/event-map-test.cc
 
 // Copyright 2009-2011  Microsoft Corporation;  Haihua Xu;  Yanmin Qian
+//                2013  Johns Hopkins University (author: Daniel Povey)
 
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -135,10 +136,12 @@ void TestEventTypeIo(bool binary) {
   }
 }
 
+const int32 kMaxVal = 20;
+
 EventMap *RandomEventMap(const std::vector<EventKeyType> &keys) {
   // Do not mess with the probabilities inside this routine or there
   // is a danger this function will blow up.
-  int32 max_val = 20;
+  int32 max_val = kMaxVal;
   assert(keys.size() != 0);
   float f = RandUniform();
   if (f < 0.333) {  // w.p. 0.333, return ConstantEventMap.
@@ -200,6 +203,107 @@ void TestEventMapIo(bool binary) {
   }
 }
 
+void TestEventMapPrune() {
+  const EventAnswerType no_ans = -10;
+  std::vector<EventKeyType> keys;
+  keys.push_back(1); // these keys are 
+  keys.push_back(2); // hardwired into the code below, do not change
+  EventMap *em = RandomEventMap(keys);
+  EventType empty_event;
+  std::vector<EventAnswerType> all_answers;
+  em->MultiMap(empty_event, &all_answers);
+  SortAndUniq(&all_answers);
+  std::vector<EventMap*> new_leaves;
+  std::vector<EventAnswerType> mapping;
+  for (size_t i = 0; i < all_answers.size(); i++) {
+    EventAnswerType ans = all_answers[i];
+    KALDI_ASSERT(ans >= 0);
+    new_leaves.resize(ans + 1, NULL);
+    mapping.resize(ans + 1, no_ans);
+    EventAnswerType map_to;
+    if (rand() % 2 == 0) map_to = -1;
+    else map_to = rand() % 20;
+    new_leaves[ans] = new ConstantEventMap(map_to);
+    mapping[ans] = map_to;
+  }
+  EventMap *mapped_em = em->Copy(new_leaves),
+      *pruned_em = mapped_em->Prune();
+  for (size_t i = 0; i < new_leaves.size(); i++)
+    if (new_leaves[i]) delete new_leaves[i];
+  for (int32 i = 0; i < 10; i++) {
+    EventType event;
+    for (int32 key = 1; key <= 2; key++) {
+      if (rand() % 2 == 0) {
+        EventValueType value = rand() % 20;
+        event.push_back(std::make_pair(key, value));
+      }
+    }
+    EventAnswerType answer, answer2;
+    if (em->Map(event, &answer)) {
+      bool ret;
+      if (pruned_em == NULL) ret = false;
+      else ret = pruned_em->Map(event, &answer2);
+      KALDI_ASSERT(answer >= 0);
+      EventAnswerType mapped_ans = mapping[answer];
+      KALDI_ASSERT(mapped_ans != no_ans);
+      if (mapped_ans == -1) {
+        if (ret == false)
+          KALDI_LOG << "Answer was correctly pruned away.";
+        else
+          KALDI_LOG << "Answer was not pruned away [but this is not required]";
+      } else {
+        KALDI_ASSERT(ret == true);
+        KALDI_ASSERT(answer2 == mapped_ans);
+        KALDI_LOG << "Answers match " << answer << " -> " << answer2;
+      }
+    }
+  }
+  delete em;
+  delete mapped_em;
+  delete pruned_em;
+}
+
+void TestEventMapMapValues() {
+  std::vector<EventKeyType> keys;
+  keys.push_back(1); // these keys are 
+  keys.push_back(2); // hardwired into the code below, do not change
+  EventMap *em = RandomEventMap(keys);
+  EventType empty_event;
+
+  unordered_set<EventKeyType> mapped_keys;
+  unordered_map<EventKeyType,EventKeyType> value_map;
+  if (rand() % 2 == 0) mapped_keys.insert(1);
+  if (rand() % 2 == 0) mapped_keys.insert(2);
+
+  EventValueType v_offset = rand() % kMaxVal;
+  for (EventValueType v = 0; v < kMaxVal; v++)
+    value_map[v] = (v + v_offset) % kMaxVal;
+    
+  EventMap *mapped_em = em->MapValues(mapped_keys, value_map);
+  
+  for (int32 i = 0; i < 10; i++) {
+    EventType event, mapped_event;
+    for (int32 key = 1; key <= 2; key++) {
+      if (rand() % 2 == 0) {
+        EventValueType value = rand() % kMaxVal;
+        event.push_back(std::make_pair(key, value));
+        EventValueType mapped_value;
+        if (mapped_keys.count(key) == 0) mapped_value = value;
+        else mapped_value = value_map[value];
+        mapped_event.push_back(std::make_pair(key, mapped_value));
+      }
+    }
+    EventAnswerType answer, answer2;
+    if (em->Map(event, &answer)) {
+      bool ret = mapped_em->Map(mapped_event, &answer2);
+      KALDI_ASSERT(ret);
+      KALDI_ASSERT(answer == answer2);
+    }
+  }
+  delete em;
+  delete mapped_em;
+}
+
 
 
 } // end namespace kaldi
@@ -208,9 +312,14 @@ void TestEventMapIo(bool binary) {
 
 
 int main() {
-  kaldi::TestEventMap();
-  kaldi::TestEventTypeIo(false);
-  kaldi::TestEventTypeIo(true);
-  kaldi::TestEventMapIo(false);
-  kaldi::TestEventMapIo(true);
+  using namespace kaldi;
+  TestEventTypeIo(false);
+  TestEventTypeIo(true);
+  TestEventMapIo(false);
+  TestEventMapIo(true);
+  for (int32 i = 0; i <  10; i++) {
+    TestEventMap();
+    TestEventMapPrune();
+    TestEventMapMapValues();
+  }
 }

src/tree/event-map.cc

 // tree/event-map.cc
 
 // Copyright 2009-2011  Microsoft Corporation
+//                2013  Johns Hopkins University (author: Daniel Povey)
 
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -65,6 +66,56 @@ ConstantEventMap* ConstantEventMap::Read(std::istream &is, bool binary) {
   return new ConstantEventMap(answer);
 }
 
+EventMap* TableEventMap::Prune() const {
+  std::vector<EventMap*> table;
+  table.reserve(table_.size());
+  EventValueType size = table_.size();
+  for (EventKeyType value = 0; value < size; value++) {
+    if (table_[value] != NULL) {
+      EventMap *pruned_map = table_[value]->Prune();
+      if (pruned_map != NULL) {
+        table.resize(value + 1, NULL);
+        table[value] = pruned_map;
+      }
+    }
+  }
+  if (table.empty()) return NULL;
+  else return new TableEventMap(key_, table);
+}
+
+EventMap* TableEventMap::MapValues(
+    const unordered_set<EventKeyType> &keys_to_map,
+    const unordered_map<EventValueType,EventValueType> &value_map) const {
+  std::vector<EventMap*> table;
+  table.reserve(table_.size());
+  EventValueType size = table_.size();
+  for (EventValueType value = 0; value < size; value++) {
+    if (table_[value] != NULL) {
+      EventMap *this_map = table_[value]->MapValues(keys_to_map, value_map);
+      EventValueType mapped_value;
+
+      if (keys_to_map.count(key_) == 0) mapped_value = value;
+      else {
+        unordered_map<EventValueType,EventValueType>::const_iterator
+            iter = value_map.find(value);
+        if (iter == value_map.end()) {
+          KALDI_ERR << "Could not map value " << value
+                    << " for key " << key_;
+        }
+        mapped_value = iter->second;
+      }
+      KALDI_ASSERT(mapped_value >= 0);
+      if (static_cast<EventValueType>(table.size()) <= mapped_value)
+        table.resize(mapped_value + 1, NULL);
+      if (table[mapped_value] != NULL)
+        KALDI_ERR << "Multiple values map to the same point: this code cannot "
+                  << "handle this case.";
+      table[mapped_value] = this_map;
+    }
+  }
+  return new TableEventMap(key_, table);
+}
+
 
 void TableEventMap::Write(std::ostream &os, bool binary) {
   WriteToken(os, binary, "TE");
@@ -100,6 +151,42 @@ TableEventMap* TableEventMap::Read(std::istream &is, bool binary) {
   return new TableEventMap(key, table);
 }
 
+EventMap* SplitEventMap::Prune() const {
+  EventMap *yes = yes_->Prune(),
+      *no = no_->Prune();
+  if (yes == NULL && no == NULL) return NULL;
+  else if (yes == NULL) return no;
+  else if (no == NULL) return yes;
+  else return new SplitEventMap(key_, yes_set_, yes, no);
+}
+
+EventMap* SplitEventMap::MapValues(
+    const unordered_set<EventKeyType> &keys_to_map,
+    const unordered_map<EventValueType,EventValueType> &value_map) const {
+  EventMap *yes = yes_->MapValues(keys_to_map, value_map),
+      *no = no_->MapValues(keys_to_map, value_map);
+
+  if (keys_to_map.count(key_) == 0) {
+    return new SplitEventMap(key_, yes_set_, yes, no);
+  } else {
+    std::vector<EventValueType> yes_set;
+    for (ConstIntegerSet<EventValueType>::iterator iter = yes_set_.begin();
+         iter != yes_set_.end();
+         ++iter) {
+      EventValueType value = *iter;
+      unordered_map<EventValueType, EventValueType>::const_iterator
+          map_iter = value_map.find(value);
+      if (map_iter == value_map.end())
+        KALDI_ERR << "Value " << value << ", for key "
+                  << key_ << ", cannot be mapped.";
+      EventValueType mapped_value = map_iter->second;
+      yes_set.push_back(mapped_value);
+    }
+    SortAndUniq(&yes_set);
+    return new SplitEventMap(key_, yes_set, yes, no);
+  }  
+}
+
 void SplitEventMap::Write(std::ostream &os, bool binary) {
   WriteToken(os, binary, "SE");
   WriteBasicType(os, binary, key_);

src/tree/event-map.h

@@ -32,11 +32,11 @@ namespace kaldi {
 /// See \ref tree_internals for overview, and specifically \ref treei_event_map.
 
 
-// Note RE negative values: some of this code will not work if
-// things of type EventValueType and EventAnswerType are negative (kNoAnswer is a
-// special case).  In particular, TableEventMap can't be used if things of EventValueType
-// are negative, and additionally TableEventMap won't be efficient if things of EventValueType take
-// on extremely large values.  The EventKeyType can be negative though.
+// Note RE negative values: some of this code will not work if things of type
+// EventValueType are negative.  In particular, TableEventMap can't be used if
+// things of EventValueType are negative, and additionally TableEventMap won't
+// be efficient if things of EventValueType take on extremely large values.  The
+// EventKeyType can be negative though.
 
 /// Things of type EventKeyType can take any value.  The code does not assume they are contiguous.
 /// So values like -1, 1000000 and the like are acceptable.
@@ -54,7 +54,7 @@ typedef int32 EventValueType;
 typedef int32 EventAnswerType;
 
 typedef std::vector<std::pair<EventKeyType, EventValueType> > EventType;
-// It is required to be sorted and have unique names-- i.e. functions assume this when called
+// It is required to be sorted and have unique keys-- i.e. functions assume this when called
 // with this type.
 
 inline std::pair<EventKeyType, EventValueType> MakeEventPair (EventKeyType k, EventValueType v) {  
@@ -111,9 +111,32 @@ class EventMap {
   // Copy() does not take ownership of the pointers in new_leaves (it uses the Copy() function of those
   // EventMaps).
   virtual EventMap *Copy(const std::vector<EventMap*> &new_leaves) const = 0;
-
-  EventMap *Copy() const {  std::vector<EventMap*> new_leaves; return Copy(new_leaves); }
-
+  
+  EventMap *Copy() const { std::vector<EventMap*> new_leaves; return Copy(new_leaves); }
+
+  // The function MapValues() is intended to be used to map phone-sets between
+  // different integer representations.  For all the keys in the set
+  // "keys_to_map", it will map the corresponding values using the map
+  // "value_map".  Note: these values are the values in the key->value pairs of
+  // the EventMap, which really correspond to phones in the usual case; they are
+  // not the "answers" of the EventMap which correspond to clustered states.  In
+  // case multiple values are mapped to the same value, it will try to deal with
+  // it gracefully where it can, but will crash if, for example, this would
+  // cause problems with the TableEventMap.  It will also crash if any values
+  // used for keys in "keys_to_map" are not mapped by "value_map".  This
+  // function is not currently used.
+  virtual EventMap *MapValues(
+      const unordered_set<EventKeyType> &keys_to_map,
+      const unordered_map<EventValueType,EventValueType> &value_map) const = 0;
+
+  // The function Prune() is like Copy(), except it removes parts of the tree
+  // that return only -1 (it will return NULL if this EventMap returns only -1).
+  // This is a mechanism to remove parts of the tree-- you would first use the
+  // Copy() function with a vector of EventMap*, and for the parts you don't
+  // want, you'd put a ConstantEventMap with -1; you'd then call
+  // Prune() on the result.  This function is not currently used.
+  virtual EventMap *Prune() const = 0;
+  
   virtual EventAnswerType MaxResult() const {  // child classes may override this for efficiency; here is basic version.
     // returns -1 if nothing found.
     std::vector<EventAnswerType> tmp; EventType empty_event;
@@ -153,11 +176,21 @@ class ConstantEventMap: public EventMap {
   virtual void GetChildren(std::vector<EventMap*> *out) const { out->clear(); }
 
   virtual EventMap *Copy(const std::vector<EventMap*> &new_leaves) const {
-    if (answer_<0 || answer_>=(EventAnswerType)new_leaves.size() ||
+    if (answer_ < 0 || answer_ >= (EventAnswerType)new_leaves.size() ||
         new_leaves[answer_] == NULL)
       return new ConstantEventMap(answer_);
     else return new_leaves[answer_]->Copy();
   }
+
+  virtual EventMap *MapValues(
+      const unordered_set<EventKeyType> &keys_to_map,
+      const unordered_map<EventValueType,EventValueType> &value_map) const {
+    return new ConstantEventMap(answer_);
+  }
+
+  virtual EventMap *Prune() const {
+    return (answer_ == -1 ? NULL : new ConstantEventMap(answer_));
+  }
   
   explicit ConstantEventMap(EventAnswerType answer): answer_(answer) { }
   
@@ -198,6 +231,12 @@ class TableEventMap: public EventMap {
     }
   }
 
+  virtual EventMap *Prune() const;
+  
+  virtual EventMap *MapValues(
+      const unordered_set<EventKeyType> &keys_to_map,
+      const unordered_map<EventValueType,EventValueType> &value_map) const;
+  
   /// Takes ownership of pointers.
   explicit TableEventMap(EventKeyType key, const std::vector<EventMap*> &table): key_(key), table_(table) {}
   /// Takes ownership of pointers.
@@ -266,6 +305,12 @@ class SplitEventMap: public EventMap {  // A decision tree [non-leaf] node.
   virtual void Write(std::ostream &os, bool binary);
   static SplitEventMap *Read(std::istream &is, bool binary);
 
+  virtual EventMap *Prune() const;
+  
+  virtual EventMap *MapValues(
+      const unordered_set<EventKeyType> &keys_to_map,
+      const unordered_map<EventValueType,EventValueType> &value_map) const;
+  
   virtual ~SplitEventMap() { Destroy(); }
 
   /// This constructor takes ownership of the "yes" and "no" arguments.

src/tree/tree-renderer.h

@@ -15,8 +15,8 @@
 // See the Apache 2 License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef TREERENDERER_H
-#define TREERENDERER_H
+#ifndef KALDI_TREE_TREE_RENDERER_H_
+#define KALDI_TREE_TREE_RENDERER_H_
 
 #include "base/kaldi-common.h"
 #include "tree/event-map.h"
@@ -79,4 +79,4 @@ class TreeRenderer {
 
 } // namespace kaldi
 
-#endif // TREERENDERER_H
+#endif //  KALDI_TREE_TREE_RENDERER_H_