#include <set>

#include <queue>

#include "util/stl-utils.h"

#include "tree/build-tree-utils.h"

namespace kaldi {

void WriteBuildTreeStats(std::ostream &os, bool binary, const BuildTreeStatsType &stats) {

}

void ReadBuildTreeStats(std::istream &is, bool binary, const Clusterable &example, BuildTreeStatsType *stats) {

}

bool PossibleValues(EventKeyType key,

}

static void GetEventKeys(const EventType &vec, std::vector<EventKeyType> *keys) {

}

void FindAllKeys(const BuildTreeStatsType &stats, AllKeysType keys_type, std::vector<EventKeyType> *keys_out) {

}

EventMap *DoTableSplit(const EventMap &orig, EventKeyType key, const BuildTreeStatsType &stats,

int32 *num_leaves) {

// First-- map the stats to each leaf in the EventMap.

std::vector<BuildTreeStatsType> split_stats;

SplitStatsByMap(stats, orig, &split_stats);

// Now for each leaf that has stats in it, do the table split according to the given name.

std::vector<EventMap*> splits(split_stats.size(), NULL);

for (EventAnswerType leaf = 0; leaf < (EventAnswerType)split_stats.size(); leaf++) {

if (!split_stats[leaf].empty()) {

// first work out the possible values the name takes.

std::vector<EventValueType> vals; // vals are put here, sorted.

bool all_present = PossibleValues(key, split_stats[leaf], &vals);

KALDI_ASSERT(all_present); // currently do not support mapping undefined values.

KALDI_ASSERT(!vals.empty() && vals.front() >= 0); // don't support mapping negative values

// at present time-- would need different EventMap type, not TableEventMap.

std::vector<EventMap*> table(vals.back()+1, (EventMap*)NULL);

for (size_t idx = 0;idx < vals.size();idx++) {

EventValueType val = vals[idx];

if (idx == 0) table[val] = new ConstantEventMap(leaf); // reuse current leaf.

else table[val] = new ConstantEventMap( (*num_leaves)++ ); // else take new leaf id.

}

// takes ownershipof stats.

splits[leaf] = new TableEventMap(key, table);

}

}

EventMap *ans = orig.Copy(splits);

DeletePointers(&splits);

return ans;

}

EventMap *DoTableSplitMultiple(const EventMap &orig, const std::vector<EventKeyType> &keys, const BuildTreeStatsType &stats, int32 *num_leaves) {

if (keys.empty()) return orig.Copy();

else {

EventMap *cur = NULL; // would make it &orig, except for const issues.

for (size_t i = 0; i < keys.size(); i++) {

EventMap *next = DoTableSplit( (cur ? *cur : orig), keys[i], stats, num_leaves);

delete cur; // delete intermediate maps.

cur = next;

}

return cur;

}

}

void SplitStatsByMap(const BuildTreeStatsType &stats, const EventMap &e, std::vector<BuildTreeStatsType> *stats_out) {

}

void SplitStatsByKey(const BuildTreeStatsType &stats_in, EventKeyType key, std::vector<BuildTreeStatsType> *stats_out) {

}

void FilterStatsByKey(const BuildTreeStatsType &stats_in,

}

Clusterable *SumStats(const BuildTreeStatsType &stats_in) {

Clusterable *ans = NULL;

BuildTreeStatsType::const_iterator iter = stats_in.begin(), end = stats_in.end();

for (; iter != end; ++iter) {

Clusterable *cl = iter->second;

if (cl != NULL) {

if (!ans) ans = cl->Copy();

else ans->Add(*cl);

}

}

return ans;

}

BaseFloat SumNormalizer(const BuildTreeStatsType &stats_in) {

}

BaseFloat SumObjf(const BuildTreeStatsType &stats_in) {

}

void SumStatsVec(const std::vector<BuildTreeStatsType> &stats_in, std::vector<Clusterable*> *stats_out) {

}

BaseFloat ObjfGivenMap(const BuildTreeStatsType &stats_in, const EventMap &e) {

}

BaseFloat ComputeInitialSplit(const std::vector<Clusterable*> &summed_stats,

}

BaseFloat FindBestSplitForKey(const BuildTreeStatsType &stats,

}

class DecisionTreeSplitter {

};

EventMap *SplitDecisionTree(const EventMap &input_map,

const BuildTreeStatsType &stats,

Questions &q_opts,

BaseFloat thresh,

int32 max_leaves, // max_leaves<=0 -> no maximum.

int32 *num_leaves,

BaseFloat *obj_impr_out,

BaseFloat *smallest_split_change_out) {

KALDI_ASSERT(num_leaves != NULL && *num_leaves > 0); // can't be 0 or input_map would be empty.

int32 num_empty_leaves = 0;

BaseFloat like_impr = 0.0;

BaseFloat smallest_split_change = 1.0e+20;

std::vector<DecisionTreeSplitter*> builders;

{ // set up "builders" [one for each current leaf]. This array is never extended.

// the structures generated during splitting remain as trees at each array location.

std::vector<BuildTreeStatsType> split_stats;

SplitStatsByMap(stats, input_map, &split_stats);

KALDI_ASSERT(split_stats.size() != 0);

builders.resize(split_stats.size()); // size == #leaves.

for (size_t i = 0;i < split_stats.size();i++) {

EventAnswerType leaf = static_cast<EventAnswerType>(i);

if (split_stats[i].size() == 0) num_empty_leaves++;

builders[i] = new DecisionTreeSplitter(leaf, split_stats[i], q_opts);

}

}

{ // Do the splitting.

int32 count = 0;

std::priority_queue<std::pair<BaseFloat, size_t> > queue; // use size_t because logically these

// are just indexes into the array, not leaf-ids (after splitting they are no longer leaf id's).

// Initialize queue.

for (size_t i = 0; i < builders.size(); i++)

queue.push(std::make_pair(builders[i]->BestSplit(), i));

// Note-- queue's size never changes from now. All the alternatives leaves to split are

// inside the "DecisionTreeSplitter*" objects, in a tree structure.

while (queue.top().first > thresh

&& (max_leaves<=0 || *num_leaves < max_leaves)) {

smallest_split_change = std::min(smallest_split_change, queue.top().first);

size_t i = queue.top().second;

like_impr += queue.top().first;

builders[i]->DoSplit(num_leaves);

queue.pop();

queue.push(std::make_pair(builders[i]->BestSplit(), i));

count++;

}

KALDI_LOG << "DoDecisionTreeSplit: split "<< count << " times, #leaves now " << (*num_leaves);

}

if (smallest_split_change_out)

*smallest_split_change_out = smallest_split_change;

EventMap *answer = NULL;

{ // Create the output EventMap.

std::vector<EventMap*> sub_trees(builders.size());

for (size_t i = 0; i < sub_trees.size();i++) sub_trees[i] = builders[i]->GetMap();

answer = input_map.Copy(sub_trees);

for (size_t i = 0; i < sub_trees.size();i++) delete sub_trees[i];

}

// Free up memory.

for (size_t i = 0;i < builders.size();i++) delete builders[i];

if (obj_impr_out != NULL) *obj_impr_out = like_impr;

return answer;

}

int ClusterEventMapGetMapping(const EventMap &e_in,

}

EventMap *RenumberEventMap(const EventMap &e_in, int32 *num_leaves) {

EventType empty_vec;

std::vector<EventAnswerType> initial_leaves; // before renumbering.

e_in.MultiMap(empty_vec, &initial_leaves);

if (initial_leaves.empty()) {

KALDI_ASSERT(num_leaves);

if (num_leaves) *num_leaves = 0;

return e_in.Copy();

}

SortAndUniq(&initial_leaves);

EventAnswerType max_leaf_plus_one = initial_leaves.back() + 1; // will typically, but not always, equal *num_leaves.

std::vector<EventMap*> mapping(max_leaf_plus_one, (EventMap*)NULL);

std::vector<EventAnswerType>::iterator iter = initial_leaves.begin(), end = initial_leaves.end();

EventAnswerType cur_leaf = 0;

for (; iter != end; ++iter) {

KALDI_ASSERT(*iter >= 0 && *iter<max_leaf_plus_one);

mapping[*iter] = new ConstantEventMap(cur_leaf++);

}

EventMap *ans = e_in.Copy(mapping);

DeletePointers(&mapping);

KALDI_ASSERT((size_t)cur_leaf == initial_leaves.size());

if (num_leaves) *num_leaves = cur_leaf;

return ans;

}

EventMap *MapEventMapLeaves(const EventMap &e_in,

const std::vector<int32> &mapping_in) {

std::vector<EventMap*> mapping(mapping_in.size());

for (size_t i = 0; i < mapping_in.size(); i++)

mapping[i] = new ConstantEventMap(mapping_in[i]);

EventMap *ans = e_in.Copy(mapping);

DeletePointers(&mapping);

return ans;

}

EventMap *ClusterEventMap(const EventMap &e_in, const BuildTreeStatsType &stats,

BaseFloat thresh, int32 *num_removed_ptr) {

std::vector<EventMap*> mapping;

int32 num_removed = ClusterEventMapGetMapping(e_in, stats, thresh, &mapping);

EventMap *ans = e_in.Copy(mapping);

DeletePointers(&mapping);

if (num_removed_ptr != NULL) *num_removed_ptr = num_removed;

return ans;

}

EventMap *ShareEventMapLeaves(const EventMap &e_in, EventKeyType key,

std::vector<std::vector<EventValueType> > &values,

int32 *num_leaves) {

// the use of "pdfs" as the name of the next variable reflects the anticipated

// use of this function.

std::vector<std::vector<EventAnswerType> > pdfs(values.size());

for (size_t i = 0; i < values.size(); i++) {

EventType evec;

for (size_t j = 0; j < values[i].size(); j++) {

evec.push_back(MakeEventPair(key, values[i][j]));

size_t size_at_start = pdfs[i].size();

e_in.MultiMap(evec, &(pdfs[i])); // append any corresponding pdfs to pdfs[i].

if (pdfs[i].size() == size_at_start) { // Nothing added... unexpected.

KALDI_WARN << "ShareEventMapLeaves: had no leaves for key = " << key

<< ", value = " << (values[i][j]);

}

}

SortAndUniq(&(pdfs[i]));

}

std::vector<EventMap*> remapping;

for (size_t i = 0; i < values.size(); i++) {

if (pdfs[i].empty())

KALDI_WARN << "ShareEventMapLeaves: no leaves in one bucket."; // not expected.

else {

EventAnswerType map_to_this = pdfs[i][0]; // map all in this bucket

// to this value.

for (size_t j = 1; j < pdfs[i].size(); j++) {

EventAnswerType leaf = pdfs[i][j];

KALDI_ASSERT(leaf>=0);

if (remapping.size() <= static_cast<size_t>(leaf))

remapping.resize(leaf+1, NULL);

KALDI_ASSERT(remapping[leaf] == NULL);

remapping[leaf] = new ConstantEventMap(map_to_this);

}

}

}

EventMap *shared = e_in.Copy(remapping);

DeletePointers(&remapping);

EventMap *renumbered = RenumberEventMap(*shared, num_leaves);

delete shared;

return renumbered;

}

void DeleteBuildTreeStats(BuildTreeStatsType *stats) {

}

EventMap *GetToLengthMap(const BuildTreeStatsType &stats, int32 P,

const std::vector<EventValueType> *phones,

int32 default_length) {

std::vector<BuildTreeStatsType> stats_by_phone;

try {

SplitStatsByKey(stats, P, &stats_by_phone);

} catch(const KaldiFatalError &) {

KALDI_ERR <<

"You seem to have provided invalid stats [no central-phone key].";

}

std::map<EventValueType, EventAnswerType> phone_to_length;

for (size_t p = 0; p < stats_by_phone.size(); p++) {

if (! stats_by_phone[p].empty()) {

std::vector<BuildTreeStatsType> stats_by_length;

try {

SplitStatsByKey(stats_by_phone[p], kPdfClass, &stats_by_length);

} catch(const KaldiFatalError &) {

KALDI_ERR <<

"You seem to have provided invalid stats [no position key].";

}

size_t length = stats_by_length.size();

for (size_t i = 0; i < length; i++) {

if (stats_by_length[i].empty()) {

KALDI_ERR << "There are no stats available for position " << i

<< " of phone " << p;

}

}

phone_to_length[p] = length;

}

}

if (phones != NULL) { // set default length for unseen phones.

for (size_t i = 0; i < phones->size(); i++) {

if (phone_to_length.count( (*phones)[i] ) == 0) { // unseen.

phone_to_length[(*phones)[i]] = default_length;

}

}

}

EventMap *ans = new TableEventMap(P, phone_to_length);

return ans;

}

static int32 ClusterEventMapRestrictedHelper(const EventMap &e_in,

}

EventMap *ClusterEventMapRestrictedByKeys(const EventMap &e_in,

const BuildTreeStatsType &stats,

BaseFloat thresh,

const std::vector<EventKeyType> &keys,

int32 *num_removed) {

std::vector<EventMap*> leaf_mapping; // For output of ClusterEventMapGetMapping.

int32 nr = ClusterEventMapRestrictedHelper(e_in, stats, thresh, keys, &leaf_mapping);

if (num_removed != NULL) *num_removed = nr;

EventMap *ans = e_in.Copy(leaf_mapping);

DeletePointers(&leaf_mapping);

return ans;

}

EventMap *ClusterEventMapRestrictedByMap(const EventMap &e_in,

const BuildTreeStatsType &stats,

BaseFloat thresh,

const EventMap &e_restrict,

int32 *num_removed_ptr) {

std::vector<EventMap*> leaf_mapping;

std::vector<BuildTreeStatsType> split_stats;

int num_removed = 0;

SplitStatsByMap(stats, e_restrict, &split_stats);

for (size_t i = 0; i < split_stats.size(); i++) {

if (!split_stats[i].empty())

num_removed += ClusterEventMapGetMapping(e_in, split_stats[i], thresh,

&leaf_mapping);

}

if (num_removed_ptr != NULL) *num_removed_ptr = num_removed;

EventMap *ans = e_in.Copy(leaf_mapping);

DeletePointers(&leaf_mapping);

return ans;

}

EventMap *ClusterEventMapToNClustersRestrictedByMap(

const EventMap &e_in,

const BuildTreeStatsType &stats,

int32 num_clusters_required,

const EventMap &e_restrict,

int32 *num_removed_ptr) {

std::vector<BuildTreeStatsType> split_stats;

SplitStatsByMap(stats, e_restrict, &split_stats);

if (num_clusters_required < split_stats.size()) {

KALDI_WARN << "num-clusters-required is less than size of map. Not doing anything.";

if (num_removed_ptr) *num_removed_ptr = 0;

return e_in.Copy();

}

std::vector<std::vector<int32> > indexes(split_stats.size());

std::vector<std::vector<Clusterable*> > summed_stats_contiguous(split_stats.size());

BaseFloat normalizer = 0.0;

size_t max_index = 0;

int32 num_non_empty_clusters_required = num_clusters_required;

int32 num_non_empty_clusters_in_map = 0;

int32 num_non_empty_clusters = 0;

for (size_t i = 0; i < split_stats.size(); i++) {

if (!split_stats[i].empty()) {

num_non_empty_clusters_in_map++;

std::vector<BuildTreeStatsType> split_stats_i;

SplitStatsByMap(split_stats[i], e_in, &split_stats_i);

std::vector<Clusterable*> summed_stats_i;

SumStatsVec(split_stats_i, &summed_stats_i);

for (size_t j = 0; j < summed_stats_i.size(); j++) {

if (summed_stats_i[j] != NULL) {

num_non_empty_clusters++;

indexes[i].push_back(j);

summed_stats_contiguous[i].push_back(summed_stats_i[j]);

if (j > max_index) max_index = j;

}

}

normalizer += SumClusterableNormalizer(summed_stats_contiguous[i]);

} else {

// Even if split_stats[i] is empty, a cluster will be assigned to

// that. To compensate, we decrease the num-clusters required.

num_non_empty_clusters_required--;

}

}

KALDI_VLOG(1) << "Number of non-empty clusters in map = " << num_non_empty_clusters_in_map;

KALDI_VLOG(1) << "Number of non-empty clusters = " << num_non_empty_clusters;

if (num_non_empty_clusters_required > num_non_empty_clusters) {

KALDI_WARN << "Cannot get required num-clusters " << num_clusters_required

<< " as number of non-empty clusters required is larger than "

<< " number of non-empty clusters: " << num_non_empty_clusters_required

<< " > " << num_non_empty_clusters;

if (num_removed_ptr) *num_removed_ptr = 0;

return e_in.Copy();

}

std::vector<std::vector<int32> > assignments;

BaseFloat change = ClusterBottomUpCompartmentalized(

summed_stats_contiguous,

std::numeric_limits<BaseFloat>::infinity(),

num_non_empty_clusters_required,

NULL, // don't need clusters out.

&assignments); // this algorithm is quadratic, so might be quite slow.

KALDI_ASSERT(assignments.size() == split_stats.size());

int32 num_combined = 0;

for (size_t i = 0; i < split_stats.size(); i++) {

KALDI_ASSERT(assignments[i].size() == summed_stats_contiguous[i].size());

if (assignments[i].size() == 0) continue;

size_t num_clust_i = *std::max_element(assignments[i].begin(),

assignments[i].end()) + 1;

num_combined += summed_stats_contiguous[i].size() - num_clust_i;

}

KALDI_VLOG(2) << "ClusterBottomUpCompartmentalized combined " << num_combined

<< " leaves and gave a likelihood change of " << change

<< ", normalized = " << (change / normalizer)

<< ", normalizer = " << normalizer;

KALDI_ASSERT(change < 0.0001); // should be negative or zero.

std::vector<EventMap*> leaf_mapping(max_index + 1, NULL);

for (size_t i = 0; i < split_stats.size(); i++) {

for (size_t j = 0; j < summed_stats_contiguous[i].size(); j++) {

size_t index = indexes[i][j];

size_t new_index = indexes[i][assignments[i][j]];

// index assigned by clusterig-- map to existing indices in the map,

// that we clustered from, so we don't conflict with indices in other parts

// of the tree.

KALDI_ASSERT(leaf_mapping[index] == NULL || "Error: Cluster seems to have been "

"called for different parts of the tree with overlapping sets of "

"indices.");

leaf_mapping[index] = new ConstantEventMap(new_index);

}

DeletePointers(&summed_stats_contiguous[i]);

}

if (num_removed_ptr) *num_removed_ptr = num_combined;

EventMap *ans = e_in.Copy(leaf_mapping);

DeletePointers(&leaf_mapping);

return ans;

}

EventMap *GetStubMap(int32 P,

const std::vector<std::vector<int32> > &phone_sets,

const std::vector<int32> &phone2num_pdf_classes,

const std::vector<bool> &share_roots,

int32 *num_leaves_out) {

{ // Checking inputs.

KALDI_ASSERT(!phone_sets.empty() && share_roots.size() == phone_sets.size());

std::set<int32> all_phones;

for (size_t i = 0; i < phone_sets.size(); i++) {

KALDI_ASSERT(IsSortedAndUniq(phone_sets[i]));

KALDI_ASSERT(!phone_sets[i].empty());

for (size_t j = 0; j < phone_sets[i].size(); j++) {

KALDI_ASSERT(all_phones.count(phone_sets[i][j]) == 0); // check not present.

all_phones.insert(phone_sets[i][j]);

}

}

}

// Initially create a single leaf for each phone set.

size_t max_set_size = 0;

int32 highest_numbered_phone = 0;

for (size_t i = 0; i < phone_sets.size(); i++) {

max_set_size = std::max(max_set_size, phone_sets[i].size());

highest_numbered_phone =