# Get data

train_data, train_table = load_wikisql_data(path_wikisql, mode='train', toy_model=toy_model, toy_size=toy_size, no_hs_tok=no_hs_tok, aug=aug)

dev_data, dev_table = load_wikisql_data(path_wikisql, mode='dev', toy_model=toy_model, toy_size=toy_size, no_hs_tok=no_hs_tok)

# Get word vector

if no_w2i:

w2i, wemb = None, None

else:

w2i, wemb = load_w2i_wemb(path_wikisql, bert)

""" Load training sets

if aug:

mode = f"aug.{mode}"

print('Augmented data is loaded!')

path_sql = os.path.join(path_wikisql, mode+'_knowledge.jsonl')

if no_hs_tok:

path_table = os.path.join(path_wikisql, mode + '.tables.jsonl')

else:

path_table = os.path.join(path_wikisql, mode+'_tok.tables.jsonl')

data = []

table = {}

with open(path_sql) as f:

for idx, line in enumerate(f):

if toy_model and idx >= toy_size:

break

t1 = json.loads(line.strip())

data.append(t1)

with open(path_table) as f:

for idx, line in enumerate(f):

if toy_model and idx > toy_size:

break

t1 = json.loads(line.strip())

table[t1['id']] = t1

""" Load pre-made subset of TAPI.

if bert:

with open(os.path.join(path_wikisql, 'w2i_bert.json'), 'r') as f_w2i:

w2i = json.load(f_w2i)

wemb = load(os.path.join(path_wikisql, 'wemb_bert.npy'), )

else:

with open(os.path.join(path_wikisql, 'w2i.json'), 'r') as f_w2i:

w2i = json.load(f_w2i)

wemb = load(os.path.join(path_wikisql, 'wemb.npy'), )

train_loader = torch.utils.data.DataLoader(

batch_size=bS,

dataset=data_train,

shuffle=shuffle_train,

num_workers=4,

collate_fn=lambda x: x # now dictionary values are not merged!

)

dev_loader = torch.utils.data.DataLoader(

batch_size=bS,

dataset=data_dev,

shuffle=shuffle_dev,

num_workers=4,

collate_fn=lambda x: x # now dictionary values are not merged!

)

nlu1 = t1['question']

nlu_t1 = t1['question_tok']

tid1 = t1['table_id']

sql_i1 = t1['sql']

sql_q1 = t1['query']

if no_sql_t:

sql_t1 = None

else:

sql_t1 = t1['query_tok']

tb1 = tables[tid1]

if not no_hs_t:

hs_t1 = tb1['header_tok']

else:

hs_t1 = []

hs1 = tb1['header']

nlu, nlu_t, tid, sql_i, sql_q, sql_t, tb, hs_t, hs = [], [], [], [], [], [], [], [], []

for t1 in t1s:

if no_hs_t:

nlu1, nlu_t1, tid1, sql_i1, sql_q1, sql_t1, tb1, hs_t1, hs1 = get_fields_1(t1, tables, no_hs_t, no_sql_t)

else:

nlu1, nlu_t1, tid1, sql_i1, sql_q1, sql_t1, tb1, hs_t1, hs1 = get_fields_1(t1, tables, no_hs_t, no_sql_t)

nlu.append(nlu1)

nlu_t.append(nlu_t1)

tid.append(tid1)

sql_i.append(sql_i1)

sql_q.append(sql_q1)

sql_t.append(sql_t1)

tb.append(tb1)

hs_t.append(hs_t1)

hs.append(hs1)

w2i_l1 = []

l1 = len(words1) # +2 because of <BEG>, <END>

for w in words1:

idx = w2i.get(w, 0)

w2i_l1.append(idx)

if not no_BE:

l1 += 2

w2i_l1 = [1] + w2i_l1 + [2]

"""

bS = len(words)

l = torch.zeros(bS, dtype=torch.long).to(device) # length of the seq. of words.

w2i_l_list = [] # shall be replaced to arr

# wemb_NLq_batch = []

for i, words1 in enumerate(words):

w2i_l1, l1 = word_to_idx1(words1, w2i, no_BE)

w2i_l_list.append(w2i_l1)

l[i] = l1

# Prepare tensor of wemb

# overwrite w2i_l

w2i_l = torch.zeros([bS, int(max(l))], dtype=torch.long).to(device)

for b in range(bS):

w2i_l[b, :l[b]] = torch.LongTensor(w2i_l_list[b]).to(device)

""" Zero-padded when word is not available (teated as <UNK>)

bS = len(hs_t) # now, B = B_NLq

hpu_t = [] # header pseudo-utterance

l_hs = []

for hs_t1 in hs_t:

hpu_t += hs_t1

l_hs1 = len(hs_t1)

l_hs.append(l_hs1)

w2i_hpu, l_hpu = words_to_idx(hpu_t, w2i, no_BE=no_BE)

""" [batch_size, max token length, dim_emb]

bS, mL, eS = wemb_l.shape

# sort before packking

l = array(l)

perm_idx = argsort(-l)

perm_idx_inv = generate_perm_inv(perm_idx)

# pack sequence

packed_wemb_l = nn.utils.rnn.pack_padded_sequence(wemb_l[perm_idx, :, :],

l[perm_idx],

batch_first=True)

# Time to encode

if hc0 is not None:

hc0 = (hc0[0][:, perm_idx], hc0[1][:, perm_idx])

# ipdb.set_trace()

packed_wemb_l = packed_wemb_l.float() # I don't know why..

packed_wenc, hc_out = lstm(packed_wemb_l, hc0)

hout, cout = hc_out

# unpack

wenc, _l = nn.utils.rnn.pad_packed_sequence(packed_wenc, batch_first=True)

if last_only:

# Take only final outputs for each columns.

wenc = wenc[tuple(range(bS)), l[perm_idx] - 1] # [batch_size, dim_emb]

wenc.unsqueeze_(1) # [batch_size, 1, dim_emb]

wenc = wenc[perm_idx_inv]

if return_hidden:

# hout.shape = [number_of_directoin * num_of_layer, seq_len(=batch size), dim * number_of_direction ] w/ batch_first.. w/o batch_first? I need to see.

hout = hout[:, perm_idx_inv].to(device)

cout = cout[:, perm_idx_inv].to(device) # Is this correct operation?

return wenc, hout, cout

else:

wenc_hpu, hout, cout = encode( lstm,

wemb_hpu,

l_hpu,

return_hidden=True,

hc0=None,

last_only=True )

wenc_hpu = wenc_hpu.squeeze(1)

bS_hpu, mL_hpu, eS = wemb_hpu.shape

hS = wenc_hpu.size(-1)

wenc_hs = wenc_hpu.new_zeros(len(l_hs), max(l_hs), hS)

wenc_hs = wenc_hs.to(device)

# Re-pack according to batch.

# ret = [B_NLq, max_len_headers_all, dim_lstm]

st = 0

for i, l_hs1 in enumerate(l_hs):

wenc_hs[i, :l_hs1] = wenc_hpu[st:(st + l_hs1)]

st += l_hs1

"""

wc1 = []

for cond in conds:

wc1.append(cond[0])

"""

wo1 = []

for cond in conds:

wo1.append(cond[1])

"""

wv1 = []

for cond in conds:

wv1.append(cond[2])

""" for backward compatibility, separated with get_g"""

g_sc = []

g_sa = []

g_wn = []

g_wc = []

g_wo = []

g_wv = []

for b, psql_i1 in enumerate(sql_i):

g_sc.append( psql_i1["sel"] )

g_sa.append( psql_i1["agg"])

conds = psql_i1['conds']

if not psql_i1["agg"] < 0:

g_wn.append( len( conds ) )

g_wc.append( get_wc1(conds) )

g_wo.append( get_wo1(conds) )

g_wv.append( get_wv1(conds) )

else:

raise EnvironmentError

g_wvi_corenlp = []

for t1 in t:

g_wvi_corenlp.append( t1['wvi_corenlp'] )

""" Follow same approach from SQLNet author's code.

# global idx_w2i, w2i, wemb # idx, word2vec, word to idx dictionary, list of embedding vec, n_total: total number of words

if (word in wv) and (word not in w2i):

idx_w2i += 1

w2i[word] = idx_w2i

wemb.append(wv[word])

n_total += 1

w2i = {'<UNK>': 0, '<BEG>': 1, '<END>': 2} # to use it when embeds NL query.

idx_w2i = 2

n_total = 3

wemb = [np.zeros(300, dtype=np.float32) for _ in range(3)] # 128 is of TAPI vector.

idx_w2i, n_total = generate_w2i_wemb(data_train, wv, idx_w2i, n_total, w2i, wemb)

idx_w2i, n_total = generate_w2i_wemb_table(table_train, wv, idx_w2i, n_total, w2i, wemb)

idx_w2i, n_total = generate_w2i_wemb(data_dev, wv, idx_w2i, n_total, w2i, wemb)

idx_w2i, n_total = generate_w2i_wemb_table(table_dev, wv, idx_w2i, n_total, w2i, wemb)

idx_w2i, n_total = generate_w2i_wemb(data_test, wv, idx_w2i, n_total, w2i, wemb)

idx_w2i, n_total = generate_w2i_wemb_table(table_test, wv, idx_w2i, n_total, w2i, wemb)

path_w2i = os.path.join(path_save_w2i_wemb, 'w2i.json')

path_wemb = os.path.join(path_save_w2i_wemb, 'wemb.npy')

wemb = np.stack(wemb, axis=0)

with open(path_w2i, 'w') as f_w2i:

json.dump(w2i, f_w2i)

np.save(path_wemb, wemb)

""" Generate subset of GloVe

# word_set from NL query

for table_id, table_contents in tables.items():

# NLq = t1['question']

# word_tokens = NLq.rstrip().replace('?', '').split(' ')

headers = table_contents['header_tok'] # [ ['state/terriotry'], ['current', 'slogan'], [],

for header_tokens in headers:

for token in header_tokens:

idx_w2i, n_total = update_w2i_wemb(token, wv, idx_w2i, n_total, w2i, wemb)

# WikiSQL generaets unbelivable query... using state/territory in the NLq. Unnatural.. but as is

# when there is slash, unlike original SQLNet which treats them as single token, we use

# both tokens. e.g. 'state/terriotry' -> 'state'

# token_spl = token.split('/')

# for token_spl1 in token_spl:

# idx_w2i, n_total = update_w2i_wemb(token_spl1, wv, idx_w2i, n_total, w2i, wemb)

""" Generate subset of GloVe

# word_set from NL query

for i, t1 in enumerate(train_data):

# NLq = t1['question']

# word_tokens = NLq.rstrip().replace('?', '').split(' ')

word_tokens = t1['question_tok']

# Currently, TAPI does not use "?". So, it is removed.

for word in word_tokens:

idx_w2i, n_total = update_w2i_wemb(word, wv, idx_w2i, n_total, w2i, wemb)

n_total += 1

""" Generate subset of TAPI from english-to-korean dict of table headers etc..

# word_set from NL query

for table_name, e2k_dict in e2k_dicts.items():

word_tokens_list = list(e2k_dict.values())

# Currently, TAPI does not use "?". So, it is removed.

for word_tokens in word_tokens_list:

for word in word_tokens:

idx_w2i, n_total = update_w2i_wemb(word, wv, idx_w2i, n_total, w2i, wemb)

n_total += 1

hds_all_tok = []

for hds11 in hds1:

sub_tok = tokenizer.tokenize(hds11)

tokens = []

segment_ids = []

tokens.append("[CLS]")

i_st_nlu = len(tokens) # to use it later

segment_ids.append(0)

for token in nlu1_tok:

tokens.append(token)

segment_ids.append(0)

i_ed_nlu = len(tokens)

tokens.append("[SEP]")

segment_ids.append(0)

i_hds = []

# for doc

# for i, hds11 in enumerate(hds1):

# i_st_hd = len(tokens)

# sub_tok = tokenizer.tokenize(hds11)

# tokens += sub_tok

# i_ed_hd = len(tokens)

# i_hds.append((i_st_hd, i_ed_hd))

# segment_ids += [1] * len(sub_tok)

# if i < len(hds1)-1:

# tokens.append("[SEP]")

# segment_ids.append(0)

# elif i == len(hds1)-1:

# tokens.append("[SEP]")

# segment_ids.append(1)

# else:

# raise EnvironmentError

i_nlu = (i_st_nlu, i_ed_nlu)

tokens = []

segment_ids = []

tokens.append("[CLS]")

i_st_nlu = len(tokens) # to use it later

segment_ids.append(0)

for token in nlu1_tok:

tokens.append(token)

segment_ids.append(0)

i_ed_nlu = len(tokens)

tokens.append("[SEP]")

segment_ids.append(0)

i_hds = []

# for doc

for i, hds11 in enumerate(hds1):

i_st_hd = len(tokens)

sub_tok = tokenizer.tokenize(hds11)

tokens += sub_tok

i_ed_hd = len(tokens)

i_hds.append((i_st_hd, i_ed_hd))

segment_ids += [1] * len(sub_tok)

if i < len(hds1)-1:

tokens.append("[SEP]")

segment_ids.append(0)

elif i == len(hds1)-1:

tokens.append("[SEP]")

segment_ids.append(1)

else:

raise EnvironmentError

i_nlu = (i_st_nlu, i_ed_nlu)

"""

l_hpu = []

for i_hds1 in i_hds:

for i_hds11 in i_hds1:

l_hpu.append(i_hds11[1] - i_hds11[0])

"""

l_n = []

l_hs = [] # The length of columns for each batch

l_input = []

input_ids = []

tokens = []

segment_ids = []

input_mask = []

i_nlu = [] # index to retreive the position of contextual vector later.

i_hds = []

i_sql_vocab = []

doc_tokens = []

nlu_tt = []

t_to_tt_idx = []

tt_to_t_idx = []

for b, nlu_t1 in enumerate(nlu_t):

hds1 = hds[b]

l_hs.append(len(hds1))

# 1. 2nd tokenization using WordPiece

tt_to_t_idx1 = [] # number indicates where sub-token belongs to in 1st-level-tokens (here, CoreNLP).

t_to_tt_idx1 = [] # orig_to_tok_idx[i] = start index of i-th-1st-level-token in all_tokens.

nlu_tt1 = [] # all_doc_tokens[ orig_to_tok_idx[i] ] returns first sub-token segement of i-th-1st-level-token

for (i, token) in enumerate(nlu_t1):

t_to_tt_idx1.append(

len(nlu_tt1)) # all_doc_tokens[ indicate the start position of original 'white-space' tokens.

sub_tokens = tokenizer.tokenize(token)

for sub_token in sub_tokens:

tt_to_t_idx1.append(i)

nlu_tt1.append(sub_token) # all_doc_tokens are further tokenized using WordPiece tokenizer

nlu_tt.append(nlu_tt1)

tt_to_t_idx.append(tt_to_t_idx1)

t_to_tt_idx.append(t_to_tt_idx1)

l_n.append(len(nlu_tt1))

# hds1_all_tok = tokenize_hds1(tokenizer, hds1)

# [CLS] nlu [SEP] col1 [SEP] col2 [SEP] ...col-n [SEP]

# 2. Generate BERT inputs & indices.

# Combine hds1 and sql_vocab

tokens1, segment_ids1, i_sql_vocab1, i_nlu1, i_hds1 = generate_inputs_s2s(tokenizer, nlu_tt1, hds1, sql_vocab)

# i_hds1

input_ids1 = tokenizer.convert_tokens_to_ids(tokens1)

# Input masks

# The mask has 1 for real tokens and 0 for padding tokens. Only real

# tokens are attended to.

input_mask1 = [1] * len(input_ids1)

# 3. Zero-pad up to the sequence length.

l_input.append( len(input_ids1) )

while len(input_ids1) < max_seq_length:

input_ids1.append(0)

input_mask1.append(0)

segment_ids1.append(0)

assert len(input_ids1) == max_seq_length

assert len(input_mask1) == max_seq_length

assert len(segment_ids1) == max_seq_length

input_ids.append(input_ids1)

tokens.append(tokens1)

segment_ids.append(segment_ids1)

input_mask.append(input_mask1)

i_nlu.append(i_nlu1)

i_hds.append(i_hds1)

i_sql_vocab.append(i_sql_vocab1)

# Convert to tensor

all_input_ids = torch.tensor(input_ids, dtype=torch.long).to(device)

all_input_mask = torch.tensor(input_mask, dtype=torch.long).to(device)

all_segment_ids = torch.tensor(segment_ids, dtype=torch.long).to(device)

# 4. Generate BERT output.

all_encoder_layer, pooled_output = model_bert(all_input_ids, all_segment_ids, all_input_mask)

# 5. generate l_hpu from i_hds

l_hpu = gen_l_hpu(i_hds)

return all_encoder_layer, pooled_output, tokens, i_nlu, i_hds, i_sql_vocab, \

l_n, l_hpu, l_hs, l_input, \

"""

l_n = []

l_hs = [] # The length of columns for each batch

input_ids = []

tokens = []

segment_ids = []

input_mask = []

i_nlu = [] # index to retreive the position of contextual vector later.

i_hds = []

doc_tokens = []

nlu_tt = []

t_to_tt_idx = []

tt_to_t_idx = []

for b, nlu_t1 in enumerate(nlu_t):

hds1 = hds[b]

l_hs.append(len(hds1))

# 1. 2nd tokenization using WordPiece

tt_to_t_idx1 = [] # number indicates where sub-token belongs to in 1st-level-tokens (here, CoreNLP).

t_to_tt_idx1 = [] # orig_to_tok_idx[i] = start index of i-th-1st-level-token in all_tokens.

nlu_tt1 = [] # all_doc_tokens[ orig_to_tok_idx[i] ] returns first sub-token segement of i-th-1st-level-token

for (i, token) in enumerate(nlu_t1):

t_to_tt_idx1.append(

len(nlu_tt1)) # all_doc_tokens[ indicate the start position of original 'white-space' tokens.

sub_tokens = tokenizer.tokenize(token)

for sub_token in sub_tokens:

tt_to_t_idx1.append(i)

nlu_tt1.append(sub_token) # all_doc_tokens are further tokenized using WordPiece tokenizer

nlu_tt.append(nlu_tt1)

tt_to_t_idx.append(tt_to_t_idx1)

t_to_tt_idx.append(t_to_tt_idx1)

l_n.append(len(nlu_tt1))

# hds1_all_tok = tokenize_hds1(tokenizer, hds1)

# [CLS] nlu [SEP] col1 [SEP] col2 [SEP] ...col-n [SEP]

# 2. Generate BERT inputs & indices.

tokens1, segment_ids1, i_nlu1, i_hds1 = generate_inputs(tokenizer, nlu_tt1, hds1)

input_ids1 = tokenizer.convert_tokens_to_ids(tokens1)

# Input masks

# The mask has 1 for real tokens and 0 for padding tokens. Only real

# tokens are attended to.

input_mask1 = [1] * len(input_ids1)

# 3. Zero-pad up to the sequence length.

while len(input_ids1) < max_seq_length:

input_ids1.append(0)

input_mask1.append(0)

segment_ids1.append(0)

assert len(input_ids1) == max_seq_length

assert len(input_mask1) == max_seq_length

assert len(segment_ids1) == max_seq_length

input_ids.append(input_ids1)

tokens.append(tokens1)

segment_ids.append(segment_ids1)

input_mask.append(input_mask1)

i_nlu.append(i_nlu1)

i_hds.append(i_hds1)

# Convert to tensor

all_input_ids = torch.tensor(input_ids, dtype=torch.long).to(device)

all_input_mask = torch.tensor(input_mask, dtype=torch.long).to(device)

all_segment_ids = torch.tensor(segment_ids, dtype=torch.long).to(device)

# 4. Generate BERT output.

all_encoder_layer, pooled_output = model_bert(all_input_ids, all_segment_ids, all_input_mask)

# 5. generate l_hpu from i_hds

l_hpu = gen_l_hpu(i_hds)

return all_encoder_layer, pooled_output, tokens, i_nlu, i_hds, \

l_n, l_hpu, l_hs, \

"""

l_n = []

l_hs = [] # The length of columns for each batch

input_ids = []

tokens = []

segment_ids = []

input_mask = []

i_nlu = [] # index to retreive the position of contextual vector later.

i_hds = []

doc_tokens = []

nlu_tt = []

t_to_tt_idx = []

tt_to_t_idx = []

for b, nlu_t1 in enumerate(nlu_t):

hds1 = hds[b]

l_hs.append(len(hds1))

# 1. 2nd tokenization using WordPiece

tt_to_t_idx1 = [] # number indicates where sub-token belongs to in 1st-level-tokens (here, CoreNLP).

t_to_tt_idx1 = [] # orig_to_tok_idx[i] = start index of i-th-1st-level-token in all_tokens.

nlu_tt1 = [] # all_doc_tokens[ orig_to_tok_idx[i] ] returns first sub-token segement of i-th-1st-level-token

for (i, token) in enumerate(nlu_t1):

t_to_tt_idx1.append(

len(nlu_tt1)) # all_doc_tokens[ indicate the start position of original 'white-space' tokens.

sub_tokens = tokenizer.tokenize(token)

for sub_token in sub_tokens:

tt_to_t_idx1.append(i)

nlu_tt1.append(sub_token) # all_doc_tokens are further tokenized using WordPiece tokenizer

nlu_tt.append(nlu_tt1)

tt_to_t_idx.append(tt_to_t_idx1)

t_to_tt_idx.append(t_to_tt_idx1)

l_n.append(len(nlu_tt1))

# hds1_all_tok = tokenize_hds1(tokenizer, hds1)

# [CLS] nlu [SEP]

# 2. Generate BERT inputs & indices.

tokens1, segment_ids1, i_nlu1, i_hds1 = generate_inputs_agg(tokenizer, nlu_tt1, hds1)

input_ids1 = tokenizer.convert_tokens_to_ids(tokens1)

# Input masks

# The mask has 1 for real tokens and 0 for padding tokens. Only real

# tokens are attended to.

input_mask1 = [1] * len(input_ids1)

# 3. Zero-pad up to the sequence length.

while len(input_ids1) < max_seq_length:

input_ids1.append(0)

input_mask1.append(0)

segment_ids1.append(0)

assert len(input_ids1) == max_seq_length

assert len(input_mask1) == max_seq_length

assert len(segment_ids1) == max_seq_length

input_ids.append(input_ids1)

tokens.append(tokens1)

segment_ids.append(segment_ids1)

input_mask.append(input_mask1)

i_nlu.append(i_nlu1)

i_hds.append(i_hds1)

# Convert to tensor

all_input_ids = torch.tensor(input_ids, dtype=torch.long).to(device)

all_input_mask = torch.tensor(input_mask, dtype=torch.long).to(device)

all_segment_ids = torch.tensor(segment_ids, dtype=torch.long).to(device)

# 4. Generate BERT output.

all_encoder_layer, pooled_output = model_bert(all_input_ids, all_segment_ids, all_input_mask)

# 5. generate l_hpu from i_hds

l_hpu = gen_l_hpu(i_hds)

return all_encoder_layer, pooled_output, tokens, i_nlu, i_hds, \

l_n, l_hpu, l_hs, \

"""

bS = len(l_n)

l_n_max = max(l_n)

wemb_n = torch.zeros([bS, l_n_max, hS * num_out_layers_n]).to(device)

for b in range(bS):

# [B, max_len, dim]

# Fill zero for non-exist part.

l_n1 = l_n[b]

i_nlu1 = i_nlu[b]

for i_noln in range(num_out_layers_n):

i_layer = num_hidden_layers - 1 - i_noln

st = i_noln * hS

ed = (i_noln + 1) * hS

wemb_n[b, 0:(i_nlu1[1] - i_nlu1[0]), st:ed] = all_encoder_layer[i_layer][b, i_nlu1[0]:i_nlu1[1], :]

return wemb_n

"""

bS = len(l_hs)

l_hpu_max = max(l_hpu)

num_of_all_hds = sum(l_hs)

wemb_h = torch.zeros([num_of_all_hds, l_hpu_max, hS * num_out_layers_h]).to(device)

b_pu = -1

for b, i_hds1 in enumerate(i_hds):

for b1, i_hds11 in enumerate(i_hds1):

b_pu += 1

for i_nolh in range(num_out_layers_h):

i_layer = num_hidden_layers - 1 - i_nolh

st = i_nolh * hS

ed = (i_nolh + 1) * hS

wemb_h[b_pu, 0:(i_hds11[1] - i_hds11[0]), st:ed] \

= all_encoder_layer[i_layer][b, i_hds11[0]:i_hds11[1],:]

# get contextual output of all tokens from bert

all_encoder_layer, pooled_output, tokens, i_nlu, i_hds,\

l_n, l_hpu, l_hs, \

nlu_tt, t_to_tt_idx, tt_to_t_idx = get_bert_output_agg(model_bert, tokenizer, nlu_t, hds, max_seq_length)

# all_encoder_layer: BERT outputs from all layers.

# pooled_output: output of [CLS] vec.

# tokens: BERT intput tokens

# i_nlu: start and end indices of question in tokens

# i_hds: start and end indices of headers

# get the wemb

wemb_n = get_wemb_n(i_nlu, l_n, bert_config.hidden_size, bert_config.num_hidden_layers, all_encoder_layer,

num_out_layers_n)

# wemb_h = get_wemb_h(i_hds, l_hpu, l_hs, bert_config.hidden_size, bert_config.num_hidden_layers, all_encoder_layer,

# num_out_layers_h)

return wemb_n, None, l_n, l_hpu, l_hs, \

# get contextual output of all tokens from bert

all_encoder_layer, pooled_output, tokens, i_nlu, i_hds,\

l_n, l_hpu, l_hs, \

nlu_tt, t_to_tt_idx, tt_to_t_idx = get_bert_output(model_bert, tokenizer, nlu_t, hds, max_seq_length)

# all_encoder_layer: BERT outputs from all layers.

# pooled_output: output of [CLS] vec.

# tokens: BERT intput tokens

# i_nlu: start and end indices of question in tokens

# i_hds: start and end indices of headers

# get the wemb

wemb_n = get_wemb_n(i_nlu, l_n, bert_config.hidden_size, bert_config.num_hidden_layers, all_encoder_layer,

num_out_layers_n)

wemb_h = get_wemb_h(i_hds, l_hpu, l_hs, bert_config.hidden_size, bert_config.num_hidden_layers, all_encoder_layer,

num_out_layers_h)

return wemb_n, wemb_h, l_n, l_hpu, l_hs, \