From 8ab37387f306ba76337fa8abb976839dae13b615 Mon Sep 17 00:00:00 2001 From: Zishen Chen <32064967+jasonchan117@users.noreply.github.com> Date: Sun, 28 Mar 2021 08:04:58 +0800 Subject: [PATCH 01/19] Testing --- train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train.py b/train.py index d4e87b0..55ad09a 100644 --- a/train.py +++ b/train.py @@ -12,7 +12,7 @@ from utils import DataProcessor parser = argparse.ArgumentParser(allow_abbrev=False) - +## #Network parser.add_argument("--num_units", type=int, default=64, help="Network size.", dest='layer_size') parser.add_argument("--model_type", type=str, default='full', help="""full(default) | intent_only From 4ef358f0d671e82da3330f2c7671382b8dac6f11 Mon Sep 17 00:00:00 2001 From: jasonchan_co <18933818702@163.com> Date: Sun, 28 Mar 2021 00:51:06 +0000 Subject: [PATCH 02/19] Adaptation --- train.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/train.py b/train.py index 55ad09a..4b8c0b8 100644 --- a/train.py +++ b/train.py @@ -5,7 +5,7 @@ import tensorflow as tf import numpy as np from tensorflow.python.ops import rnn_cell_impl - +from tensorflow.contrib.rnn.python.ops.core_rnn_cell import _linear from utils import createVocabulary from utils import loadVocabulary from utils import computeF1Score @@ -114,7 +114,7 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, slot_inputs_shape = tf.shape(slot_inputs) slot_inputs = tf.reshape(slot_inputs, [-1, attn_size]) - y = rnn_cell_impl._linear(slot_inputs, attn_size, True) + y = _linear(slot_inputs, attn_size, True) y = tf.reshape(y, slot_inputs_shape) y = tf.expand_dims(y, 2) s = tf.reduce_sum(v * tf.tanh(hidden_features + y), [3]) @@ -134,7 +134,7 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, hidden_features = tf.nn.conv2d(hidden, k, [1, 1, 1, 1], "SAME") v = tf.get_variable("AttnV", [attn_size]) - y = rnn_cell_impl._linear(intent_input, attn_size, True) + y = _linear(intent_input, attn_size, True) y = tf.reshape(y, [-1, 1, 1, attn_size]) s = tf.reduce_sum(v*tf.tanh(hidden_features + y), [2,3]) a = tf.nn.softmax(s) @@ -148,7 +148,7 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, intent_output = d with tf.variable_scope('slot_gated'): - intent_gate = rnn_cell_impl._linear(intent_output, attn_size, True) + intent_gate = _linear(intent_output, attn_size, True) intent_gate = tf.reshape(intent_gate, [-1, 1, intent_gate.get_shape()[1].value]) v1 = tf.get_variable("gateV", [attn_size]) if remove_slot_attn == False: @@ -165,10 +165,10 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, slot_output = tf.concat([slot_gate, slot_inputs], 1) with tf.variable_scope('intent_proj'): - intent = rnn_cell_impl._linear(intent_output, intent_size, True) + intent = _linear(intent_output, intent_size, True) with tf.variable_scope('slot_proj'): - slot = rnn_cell_impl._linear(slot_output, slot_size, True) + slot = _linear(slot_output, slot_size, True) outputs = [slot, intent] return outputs From c6030fba318e9fe651375762f2cec00216acde75 Mon Sep 17 00:00:00 2001 From: jasonchan_co <18933818702@163.com> Date: Sun, 28 Mar 2021 19:30:20 +0000 Subject: [PATCH 03/19] Add debug message --- train.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/train.py b/train.py index 4b8c0b8..97f6f8f 100644 --- a/train.py +++ b/train.py @@ -6,6 +6,7 @@ import numpy as np from tensorflow.python.ops import rnn_cell_impl from tensorflow.contrib.rnn.python.ops.core_rnn_cell import _linear +from tensorflow.python import debug as tfdbg from utils import createVocabulary from utils import loadVocabulary from utils import computeF1Score @@ -92,7 +93,7 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, inputs = tf.nn.embedding_lookup(embedding, input_data) state_outputs, final_state = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, inputs, sequence_length=sequence_length, dtype=tf.float32) - + sa = tf.shape(state_outputs) final_state = tf.concat([final_state[0][0], final_state[0][1], final_state[1][0], final_state[1][1]], 1) state_outputs = tf.concat([state_outputs[0], state_outputs[1]], 2) state_shape = state_outputs.get_shape() @@ -170,7 +171,7 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, with tf.variable_scope('slot_proj'): slot = _linear(slot_output, slot_size, True) - outputs = [slot, intent] + outputs = [slot, intent, sa] return outputs # Create Training Model @@ -186,7 +187,8 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, slots_shape = tf.shape(slots) slots_reshape = tf.reshape(slots, [-1]) - +# Debug print +sa = training_outputs[2] slot_outputs = training_outputs[0] with tf.variable_scope('slot_loss'): crossent = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=slots_reshape, logits=slot_outputs) @@ -222,8 +224,8 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, gradient_norm_intent = norm_intent update_slot = opt.apply_gradients(zip(clipped_gradients_slot, slot_params)) update_intent = opt.apply_gradients(zip(clipped_gradients_intent, intent_params), global_step=global_step) - -training_outputs = [global_step, slot_loss, update_intent, update_slot, gradient_norm_intent, gradient_norm_slot] +# Debug output +training_outputs = [global_step, slot_loss, update_intent, update_slot, gradient_norm_intent, gradient_norm_slot, sa] inputs = [input_data, sequence_length, slots, slot_weights, intent] # Create Inference Model @@ -242,6 +244,7 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, # Start Training with tf.Session() as sess: + sess.run(tf.global_variables_initializer()) logging.info('Training Start') @@ -280,6 +283,7 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, epochs += 1 logging.info('Step: ' + str(step)) logging.info('Epochs: ' + str(epochs)) + logging.info('Shape: '+ str(ret[6])) logging.info('Loss: ' + str(loss/num_loss)) num_loss = 0 loss = 0.0 From 2d458b0074378b9ab9e0d9b6733dbca65a223b96 Mon Sep 17 00:00:00 2001 From: jasonchan_co <18933818702@163.com> Date: Sun, 28 Mar 2021 19:57:02 +0000 Subject: [PATCH 04/19] Add Annotations. --- train.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/train.py b/train.py index 97f6f8f..8174b66 100644 --- a/train.py +++ b/train.py @@ -91,9 +91,10 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, embedding = tf.get_variable('embedding', [input_size, layer_size]) inputs = tf.nn.embedding_lookup(embedding, input_data) - + # State_outputs: ontains forward and backwards sequence. Shape: 2 x batchsize x len x dim + # Final_state: The final states of both forward and backwards LSTM. Shape: 2 x batchsize x dim state_outputs, final_state = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, inputs, sequence_length=sequence_length, dtype=tf.float32) - sa = tf.shape(state_outputs) + sa = final_state final_state = tf.concat([final_state[0][0], final_state[0][1], final_state[1][0], final_state[1][1]], 1) state_outputs = tf.concat([state_outputs[0], state_outputs[1]], 2) state_shape = state_outputs.get_shape() From 2bd8fc80f3943df24f8730c45ec33bb65e9266db Mon Sep 17 00:00:00 2001 From: jasonchan_co <18933818702@163.com> Date: Sun, 28 Mar 2021 20:31:11 +0000 Subject: [PATCH 05/19] Add Annotations. --- train.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/train.py b/train.py index 8174b66..fe1a2bf 100644 --- a/train.py +++ b/train.py @@ -92,13 +92,14 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, embedding = tf.get_variable('embedding', [input_size, layer_size]) inputs = tf.nn.embedding_lookup(embedding, input_data) # State_outputs: ontains forward and backwards sequence. Shape: 2 x batchsize x len x dim - # Final_state: The final states of both forward and backwards LSTM. Shape: 2 x batchsize x dim + # Final_state: The final states of both forward and backwards LSTM. Shape: 2 x 2(cell and hidden) x batchsize x dim state_outputs, final_state = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, inputs, sequence_length=sequence_length, dtype=tf.float32) sa = final_state + # concatenate in the last dim, so final will become batch_size x dim(256) final_state = tf.concat([final_state[0][0], final_state[0][1], final_state[1][0], final_state[1][1]], 1) - state_outputs = tf.concat([state_outputs[0], state_outputs[1]], 2) + state_outputs = tf.concat([state_outputs[0], state_outputs[1]], 2) # Shape batchsize x len x dim(128) state_shape = state_outputs.get_shape() - + with tf.variable_scope('attention'): slot_inputs = state_outputs if remove_slot_attn == False: From e735b92def336c09c9acf528b5cb468ee23cc7d3 Mon Sep 17 00:00:00 2001 From: jasonchan_co <18933818702@163.com> Date: Sun, 28 Mar 2021 21:28:23 +0000 Subject: [PATCH 06/19] Add Annotations. --- train.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/train.py b/train.py index fe1a2bf..912bd38 100644 --- a/train.py +++ b/train.py @@ -94,7 +94,7 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, # State_outputs: ontains forward and backwards sequence. Shape: 2 x batchsize x len x dim # Final_state: The final states of both forward and backwards LSTM. Shape: 2 x 2(cell and hidden) x batchsize x dim state_outputs, final_state = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, inputs, sequence_length=sequence_length, dtype=tf.float32) - sa = final_state + sa = tf.state_outputs # concatenate in the last dim, so final will become batch_size x dim(256) final_state = tf.concat([final_state[0][0], final_state[0][1], final_state[1][0], final_state[1][1]], 1) state_outputs = tf.concat([state_outputs[0], state_outputs[1]], 2) # Shape batchsize x len x dim(128) @@ -104,12 +104,12 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, slot_inputs = state_outputs if remove_slot_attn == False: with tf.variable_scope('slot_attn'): - attn_size = state_shape[2].value + attn_size = state_shape[2].value # dim(128) origin_shape = tf.shape(state_outputs) - hidden = tf.expand_dims(state_outputs, 1) - hidden_conv = tf.expand_dims(state_outputs, 2) + hidden = tf.expand_dims(state_outputs, 1) # Shape: batchsize x 1 x len x dim(128) + hidden_conv = tf.expand_dims(state_outputs, 2) # Shape: batchsize x len x 1 x dim(128) # hidden shape = [batch, sentence length, 1, hidden size] - k = tf.get_variable("AttnW", [1, 1, attn_size, attn_size]) + k = tf.get_variable("AttnW", [1, 1, attn_size, attn_size]) # 1 x 1 x 128 x 128 hidden_features = tf.nn.conv2d(hidden_conv, k, [1, 1, 1, 1], "SAME") hidden_features = tf.reshape(hidden_features, origin_shape) hidden_features = tf.expand_dims(hidden_features, 1) From 60c4e44bcf078a5c6a32e90a6f8dce887ac541f3 Mon Sep 17 00:00:00 2001 From: jasonchan_co <18933818702@163.com> Date: Sun, 28 Mar 2021 23:27:01 +0000 Subject: [PATCH 07/19] Add Annotations. --- train.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/train.py b/train.py index 912bd38..31d925a 100644 --- a/train.py +++ b/train.py @@ -94,10 +94,10 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, # State_outputs: ontains forward and backwards sequence. Shape: 2 x batchsize x len x dim # Final_state: The final states of both forward and backwards LSTM. Shape: 2 x 2(cell and hidden) x batchsize x dim state_outputs, final_state = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, inputs, sequence_length=sequence_length, dtype=tf.float32) - sa = tf.state_outputs + # concatenate in the last dim, so final will become batch_size x dim(256) final_state = tf.concat([final_state[0][0], final_state[0][1], final_state[1][0], final_state[1][1]], 1) - state_outputs = tf.concat([state_outputs[0], state_outputs[1]], 2) # Shape batchsize x len x dim(128) + state_outputs = tf.concat([state_outputs[0], state_outputs[1]], 2) # Shape: batchsize x len x dim(128) state_shape = state_outputs.get_shape() with tf.variable_scope('attention'): @@ -110,17 +110,19 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, hidden_conv = tf.expand_dims(state_outputs, 2) # Shape: batchsize x len x 1 x dim(128) # hidden shape = [batch, sentence length, 1, hidden size] k = tf.get_variable("AttnW", [1, 1, attn_size, attn_size]) # 1 x 1 x 128 x 128 + # Convolutional: Attention weights hidden_features = tf.nn.conv2d(hidden_conv, k, [1, 1, 1, 1], "SAME") hidden_features = tf.reshape(hidden_features, origin_shape) hidden_features = tf.expand_dims(hidden_features, 1) - v = tf.get_variable("AttnV", [attn_size]) + # Derive the hidden states weighted from attention + v = tf.get_variable("AttnV", [attn_size])# 128 slot_inputs_shape = tf.shape(slot_inputs) - slot_inputs = tf.reshape(slot_inputs, [-1, attn_size]) - y = _linear(slot_inputs, attn_size, True) + slot_inputs = tf.reshape(slot_inputs, [-1, attn_size])# Shape: (batchsize x len) x dim(128) + y = _linear(slot_inputs, attn_size, True)# The y here is the origin hidden states. y = tf.reshape(y, slot_inputs_shape) y = tf.expand_dims(y, 2) - s = tf.reduce_sum(v * tf.tanh(hidden_features + y), [3]) + s = tf.reduce_sum(v * tf.tanh(hidden_features + y), [3])# Sum the origin hidden states and weighted hidden states up a = tf.nn.softmax(s) # a shape = [batch, input size, sentence length, 1] a = tf.expand_dims(a, -1) From dc25c8a6526250fc73dfefe925ab01471fb1ed99 Mon Sep 17 00:00:00 2001 From: jasonchan_co <18933818702@163.com> Date: Mon, 29 Mar 2021 00:20:04 +0000 Subject: [PATCH 08/19] Add Annotations. --- train.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/train.py b/train.py index 31d925a..b6e44a7 100644 --- a/train.py +++ b/train.py @@ -114,7 +114,7 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, hidden_features = tf.nn.conv2d(hidden_conv, k, [1, 1, 1, 1], "SAME") hidden_features = tf.reshape(hidden_features, origin_shape) hidden_features = tf.expand_dims(hidden_features, 1) - # Derive the hidden states weighted from attention + # Derive the hidden states weighted from attention (Content vector) v = tf.get_variable("AttnV", [attn_size])# 128 slot_inputs_shape = tf.shape(slot_inputs) @@ -131,11 +131,12 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, attn_size = state_shape[2].value slot_inputs = tf.reshape(slot_inputs, [-1, attn_size]) - intent_input = final_state + intent_input = final_state # Shape: 2 x 2(cell and hidden) x batchsize x dim with tf.variable_scope('intent_attn'): - attn_size = state_shape[2].value - hidden = tf.expand_dims(state_outputs, 2) - k = tf.get_variable("AttnW", [1, 1, attn_size, attn_size]) + attn_size = state_shape[2].value # dim(128) + hidden = tf.expand_dims(state_outputs, 2) # Shape: batchsize x len x 1 x dim(128) + k = tf.get_variable("AttnW", [1, 1, attn_size, attn_size]) # 1 x 1 128 x 128 + # Attention weighted hidden_features = tf.nn.conv2d(hidden, k, [1, 1, 1, 1], "SAME") v = tf.get_variable("AttnV", [attn_size]) @@ -153,7 +154,7 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, intent_output = d with tf.variable_scope('slot_gated'): - intent_gate = _linear(intent_output, attn_size, True) + intent_gate = _linear(intent_output, attn_size, True) intent_gate = tf.reshape(intent_gate, [-1, 1, intent_gate.get_shape()[1].value]) v1 = tf.get_variable("gateV", [attn_size]) if remove_slot_attn == False: From 89a6e43bb1ac52cb60e8a36930ef1303bc605f07 Mon Sep 17 00:00:00 2001 From: jasonchan_co <18933818702@163.com> Date: Thu, 1 Apr 2021 08:50:05 +0000 Subject: [PATCH 09/19] Add Annotations. --- train.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/train.py b/train.py index b6e44a7..6b54649 100644 --- a/train.py +++ b/train.py @@ -91,21 +91,21 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, embedding = tf.get_variable('embedding', [input_size, layer_size]) inputs = tf.nn.embedding_lookup(embedding, input_data) - # State_outputs: ontains forward and backwards sequence. Shape: 2 x batchsize x len x dim - # Final_state: The final states of both forward and backwards LSTM. Shape: 2 x 2(cell and hidden) x batchsize x dim + # State_outputs: ontains forward and backwards sequence. Shape: 2 x batchsize x len x dim [2 12 23 64] + # Final_state: The final states of both forward and backwards LSTM. Shape: 2 x 2(cell and hidden) x batchsize x dim [2 2 12 64] state_outputs, final_state = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, inputs, sequence_length=sequence_length, dtype=tf.float32) - # concatenate in the last dim, so final will become batch_size x dim(256) + # concatenate in the last dim, so final will become batch_size x dim(256) [12 256] final_state = tf.concat([final_state[0][0], final_state[0][1], final_state[1][0], final_state[1][1]], 1) - state_outputs = tf.concat([state_outputs[0], state_outputs[1]], 2) # Shape: batchsize x len x dim(128) + state_outputs = tf.concat([state_outputs[0], state_outputs[1]], 2) # Shape: batchsize x len x dim(128) [12 23 128] state_shape = state_outputs.get_shape() with tf.variable_scope('attention'): - slot_inputs = state_outputs + slot_inputs = state_outputs #[12 23 128] if remove_slot_attn == False: with tf.variable_scope('slot_attn'): - attn_size = state_shape[2].value # dim(128) - origin_shape = tf.shape(state_outputs) + attn_size = state_shape[2].value #128 + origin_shape = tf.shape(state_outputs) #[12 23 128] hidden = tf.expand_dims(state_outputs, 1) # Shape: batchsize x 1 x len x dim(128) hidden_conv = tf.expand_dims(state_outputs, 2) # Shape: batchsize x len x 1 x dim(128) # hidden shape = [batch, sentence length, 1, hidden size] @@ -118,7 +118,8 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, v = tf.get_variable("AttnV", [attn_size])# 128 slot_inputs_shape = tf.shape(slot_inputs) - slot_inputs = tf.reshape(slot_inputs, [-1, attn_size])# Shape: (batchsize x len) x dim(128) + slot_inputs = tf.reshape(slot_inputs, [-1, attn_size])# Shape: (batchsize x len) x dim(128) [276 128] + y = _linear(slot_inputs, attn_size, True)# The y here is the origin hidden states. y = tf.reshape(y, slot_inputs_shape) y = tf.expand_dims(y, 2) @@ -126,7 +127,9 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, a = tf.nn.softmax(s) # a shape = [batch, input size, sentence length, 1] a = tf.expand_dims(a, -1) - slot_d = tf.reduce_sum(a * hidden, [2]) + slot_d = tf.reduce_sum(a * hidden, [2]) #[12 23 128] + + else: attn_size = state_shape[2].value slot_inputs = tf.reshape(slot_inputs, [-1, attn_size]) @@ -149,7 +152,8 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, d = tf.reduce_sum(a * hidden, [1, 2]) if add_final_state_to_intent == True: - intent_output = tf.concat([d, intent_input], 1) + intent_output = tf.concat([d, intent_input], 1) #[12 384] + sa = tf.shape(intent_output) else: intent_output = d @@ -172,9 +176,10 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, with tf.variable_scope('intent_proj'): intent = _linear(intent_output, intent_size, True) - + with tf.variable_scope('slot_proj'): - slot = _linear(slot_output, slot_size, True) + slot = _linear(slot_output, slot_size, True) # slot_output: [ 12 384] + outputs = [slot, intent, sa] return outputs From 5d3dbdbb36533a86054f41dcff5efa3b88fc7f85 Mon Sep 17 00:00:00 2001 From: jasonchan_co <18933818702@163.com> Date: Thu, 1 Apr 2021 11:08:36 +0000 Subject: [PATCH 10/19] Add Annotations. --- train.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/train.py b/train.py index 6b54649..78e795c 100644 --- a/train.py +++ b/train.py @@ -79,7 +79,7 @@ slot_vocab = loadVocabulary(os.path.join(arg.vocab_path, 'slot_vocab')) intent_vocab = loadVocabulary(os.path.join(arg.vocab_path, 'intent_vocab')) -def createModel(input_data, input_size, sequence_length, slot_size, intent_size, layer_size = 128, isTraining = True): +def createModel(input_data, input_size, sequence_length, slot_size, intent_size, layer_size = 128, interplay = False, isTraining = True): cell_fw = tf.contrib.rnn.BasicLSTMCell(layer_size) cell_bw = tf.contrib.rnn.BasicLSTMCell(layer_size) @@ -153,7 +153,7 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, if add_final_state_to_intent == True: intent_output = tf.concat([d, intent_input], 1) #[12 384] - sa = tf.shape(intent_output) + else: intent_output = d @@ -165,20 +165,23 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, slot_gate = v1 * tf.tanh(slot_d + intent_gate) else: slot_gate = v1 * tf.tanh(state_outputs + intent_gate) - slot_gate = tf.reduce_sum(slot_gate, [2]) + # Slot_gete Before sum: [ 12 23 128] + slot_gate = tf.reduce_sum(slot_gate, [2]) # Slot_gate after sum: [12 23] slot_gate = tf.expand_dims(slot_gate, -1) + if remove_slot_attn == False: - slot_gate = slot_d * slot_gate + slot_gate = slot_d * slot_gate # slot_d : [12 23 128] else: slot_gate = state_outputs * slot_gate - slot_gate = tf.reshape(slot_gate, [-1, attn_size]) - slot_output = tf.concat([slot_gate, slot_inputs], 1) - + slot_gate = tf.reshape(slot_gate, [-1, attn_size]) # [276 128] + slot_output = tf.concat([slot_gate, slot_inputs], 1) #[276 256] + with tf.variable_scope('intent_proj'): - intent = _linear(intent_output, intent_size, True) + intent = _linear(intent_output, intent_size, True) # intent shape: [12 9] with tf.variable_scope('slot_proj'): - slot = _linear(slot_output, slot_size, True) # slot_output: [ 12 384] + # Slot :[276 74] + slot = _linear(slot_output, slot_size, True) # slot_output: [276 256] outputs = [slot, intent, sa] From 6b5ab8a2470fc80acfa32ce56028f910abe934e0 Mon Sep 17 00:00:00 2001 From: jasonchan_co <18933818702@163.com> Date: Thu, 1 Apr 2021 11:13:32 +0000 Subject: [PATCH 11/19] Add interplay from slot to intent --- train.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/train.py b/train.py index 78e795c..1869f9a 100644 --- a/train.py +++ b/train.py @@ -153,7 +153,9 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, if add_final_state_to_intent == True: intent_output = tf.concat([d, intent_input], 1) #[12 384] - + if interplay == True: + slot_t = tf.reshape(slot_d, [12, -1]) + intent_output = tf.concat([d, slot_t], 1) else: intent_output = d From e7fc2a5af1ff28789b7ca13fb2217cd9aae37bf2 Mon Sep 17 00:00:00 2001 From: jasonchan_co <18933818702@163.com> Date: Thu, 1 Apr 2021 12:18:24 +0000 Subject: [PATCH 12/19] Add for extra experiments --- train.py | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/train.py b/train.py index 1869f9a..9101cc7 100644 --- a/train.py +++ b/train.py @@ -40,6 +40,10 @@ parser.add_argument("--slot_file", type=str, default='seq.out', help="Slot file name.") parser.add_argument("--intent_file", type=str, default='label', help="Intent file name.") +parser.add_argument("--interplay", action='store_true') +parser.add_argument("--remove_intent_attn", action='store_true') +parser.add_argument("--remove_gate", action='store_true') + arg=parser.parse_args() #Print arguments @@ -79,7 +83,7 @@ slot_vocab = loadVocabulary(os.path.join(arg.vocab_path, 'slot_vocab')) intent_vocab = loadVocabulary(os.path.join(arg.vocab_path, 'intent_vocab')) -def createModel(input_data, input_size, sequence_length, slot_size, intent_size, layer_size = 128, interplay = False, isTraining = True): +def createModel(input_data, input_size, sequence_length, slot_size, intent_size, layer_size = 128, interplay = False, remove_intent_attn = False, remove_gate = False, isTraining = True): cell_fw = tf.contrib.rnn.BasicLSTMCell(layer_size) cell_bw = tf.contrib.rnn.BasicLSTMCell(layer_size) @@ -134,9 +138,9 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, attn_size = state_shape[2].value slot_inputs = tf.reshape(slot_inputs, [-1, attn_size]) - intent_input = final_state # Shape: 2 x 2(cell and hidden) x batchsize x dim + intent_input = final_state # [12 256]] with tf.variable_scope('intent_attn'): - attn_size = state_shape[2].value # dim(128) + attn_size = state_shape[2].value # dim(128) state_outputs : [12 23 128] hidden = tf.expand_dims(state_outputs, 2) # Shape: batchsize x len x 1 x dim(128) k = tf.get_variable("AttnW", [1, 1, attn_size, attn_size]) # 1 x 1 128 x 128 # Attention weighted @@ -152,14 +156,22 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, d = tf.reduce_sum(a * hidden, [1, 2]) if add_final_state_to_intent == True: - intent_output = tf.concat([d, intent_input], 1) #[12 384] + if remove_intent_attn == True: + + intent_output = tf.tf.concat([tf.reshape(state_outputs, [state_outputs.get_shape()[0].value, -1]), intent_input], 1) + else: + intent_output = tf.concat([d, intent_input], 1) #[12 384] if interplay == True: - slot_t = tf.reshape(slot_d, [12, -1]) + temp = slot_d.get_shape() + slot_t = tf.reshape(slot_d, [temp[0].value, -1]) intent_output = tf.concat([d, slot_t], 1) else: intent_output = d with tf.variable_scope('slot_gated'): + slot_without_gate = tf.reshape( slot_d, [-1, attn_size]) + slot_without_gate = tf.concat([slot_without_gate, slot_inputs], 1) + intent_gate = _linear(intent_output, attn_size, True) intent_gate = tf.reshape(intent_gate, [-1, 1, intent_gate.get_shape()[1].value]) v1 = tf.get_variable("gateV", [attn_size]) @@ -183,10 +195,13 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, with tf.variable_scope('slot_proj'): # Slot :[276 74] - slot = _linear(slot_output, slot_size, True) # slot_output: [276 256] + if remove_gate == True: + slot = _linear(slot_without_gate, slot_size, True) + else: + slot = _linear(slot_output, slot_size, True) # slot_output: [276 256] - outputs = [slot, intent, sa] + outputs = [slot, intent] return outputs # Create Training Model @@ -198,7 +213,7 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, intent = tf.placeholder(tf.int32, [None], name='intent') with tf.variable_scope('model'): - training_outputs = createModel(input_data, len(in_vocab['vocab']), sequence_length, len(slot_vocab['vocab']), len(intent_vocab['vocab']), layer_size=arg.layer_size) + training_outputs = createModel(input_data, len(in_vocab['vocab']), sequence_length, len(slot_vocab['vocab']), len(intent_vocab['vocab']), layer_size=arg.layer_size, interplay = arg.interplay, remove_intent_attn = arg.remove_intent_attn, remove_gate = arg.remove_gate) slots_shape = tf.shape(slots) slots_reshape = tf.reshape(slots, [-1]) @@ -240,12 +255,12 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, update_slot = opt.apply_gradients(zip(clipped_gradients_slot, slot_params)) update_intent = opt.apply_gradients(zip(clipped_gradients_intent, intent_params), global_step=global_step) # Debug output -training_outputs = [global_step, slot_loss, update_intent, update_slot, gradient_norm_intent, gradient_norm_slot, sa] +training_outputs = [global_step, slot_loss, update_intent, update_slot, gradient_norm_intent, gradient_norm_slot] inputs = [input_data, sequence_length, slots, slot_weights, intent] # Create Inference Model with tf.variable_scope('model', reuse=True): - inference_outputs = createModel(input_data, len(in_vocab['vocab']), sequence_length, len(slot_vocab['vocab']), len(intent_vocab['vocab']), layer_size=arg.layer_size, isTraining=False) + inference_outputs = createModel(input_data, len(in_vocab['vocab']), sequence_length, len(slot_vocab['vocab']), len(intent_vocab['vocab']), layer_size=arg.layer_size, interplay = arg.interplay, remove_intent_attn = arg.remove_intent_attn, remove_gate = arg.remove_gate, isTraining=False) inference_slot_output = tf.nn.softmax(inference_outputs[0], name='slot_output') inference_intent_output = tf.nn.softmax(inference_outputs[1], name='intent_output') @@ -298,7 +313,7 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, epochs += 1 logging.info('Step: ' + str(step)) logging.info('Epochs: ' + str(epochs)) - logging.info('Shape: '+ str(ret[6])) + # logging.info('Shape: '+ str(ret[6])) logging.info('Loss: ' + str(loss/num_loss)) num_loss = 0 loss = 0.0 From b9a82ebec87f4f2af03cc85604be80b9e2faa40c Mon Sep 17 00:00:00 2001 From: jasonchan_co <18933818702@163.com> Date: Fri, 2 Apr 2021 22:54:27 +0000 Subject: [PATCH 13/19] Debug --- train.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/train.py b/train.py index 9101cc7..7d94218 100644 --- a/train.py +++ b/train.py @@ -104,7 +104,10 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, state_outputs = tf.concat([state_outputs[0], state_outputs[1]], 2) # Shape: batchsize x len x dim(128) [12 23 128] state_shape = state_outputs.get_shape() + with tf.variable_scope('attention'): + bs = state_shape[0].value + sa = tf.shape(state_outputs) slot_inputs = state_outputs #[12 23 128] if remove_slot_attn == False: with tf.variable_scope('slot_attn'): @@ -154,17 +157,17 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, a = tf.expand_dims(a, -1) a = tf.expand_dims(a, -1) d = tf.reduce_sum(a * hidden, [1, 2]) - + if add_final_state_to_intent == True: if remove_intent_attn == True: - intent_output = tf.tf.concat([tf.reshape(state_outputs, [state_outputs.get_shape()[0].value, -1]), intent_input], 1) + intent_output = tf.tf.concat([tf.reshape(state_outputs, [bs, -1]), intent_input], 1) + elif interplay == True: + slot_t = tf.reshape(slot_d, [tf.shape(slot_d).eval()[0], -1]) + intent_output = tf.concat([d, slot_t], 1) else: intent_output = tf.concat([d, intent_input], 1) #[12 384] - if interplay == True: - temp = slot_d.get_shape() - slot_t = tf.reshape(slot_d, [temp[0].value, -1]) - intent_output = tf.concat([d, slot_t], 1) + else: intent_output = d @@ -201,7 +204,7 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, slot = _linear(slot_output, slot_size, True) # slot_output: [276 256] - outputs = [slot, intent] + outputs = [slot, intent, sa] return outputs # Create Training Model @@ -255,7 +258,7 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, update_slot = opt.apply_gradients(zip(clipped_gradients_slot, slot_params)) update_intent = opt.apply_gradients(zip(clipped_gradients_intent, intent_params), global_step=global_step) # Debug output -training_outputs = [global_step, slot_loss, update_intent, update_slot, gradient_norm_intent, gradient_norm_slot] +training_outputs = [global_step, slot_loss, update_intent, update_slot, gradient_norm_intent, gradient_norm_slot, sa] inputs = [input_data, sequence_length, slots, slot_weights, intent] # Create Inference Model @@ -313,7 +316,7 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, epochs += 1 logging.info('Step: ' + str(step)) logging.info('Epochs: ' + str(epochs)) - # logging.info('Shape: '+ str(ret[6])) + logging.info('Shape: '+ str(ret[6])) logging.info('Loss: ' + str(loss/num_loss)) num_loss = 0 loss = 0.0 From be845c64419e5d5978171f41d29a736b8768c281 Mon Sep 17 00:00:00 2001 From: jasonchan_co <18933818702@163.com> Date: Sat, 3 Apr 2021 00:08:16 +0000 Subject: [PATCH 14/19] Debug --- train.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/train.py b/train.py index 7d94218..5f2a32e 100644 --- a/train.py +++ b/train.py @@ -107,7 +107,7 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, with tf.variable_scope('attention'): bs = state_shape[0].value - sa = tf.shape(state_outputs) + slot_inputs = state_outputs #[12 23 128] if remove_slot_attn == False: with tf.variable_scope('slot_attn'): @@ -156,15 +156,19 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, a = tf.nn.softmax(s) a = tf.expand_dims(a, -1) a = tf.expand_dims(a, -1) - d = tf.reduce_sum(a * hidden, [1, 2]) + d = tf.reduce_sum(a * hidden, [1, 2]) # a * hidden shape:[ 12 23 1 128] + sa = tf.shape(d) if add_final_state_to_intent == True: if remove_intent_attn == True: - intent_output = tf.tf.concat([tf.reshape(state_outputs, [bs, -1]), intent_input], 1) + intent_output = tf.concat([tf.reshape(state_outputs, [bs, -1]), intent_input], 1) elif interplay == True: - slot_t = tf.reshape(slot_d, [tf.shape(slot_d).eval()[0], -1]) - intent_output = tf.concat([d, slot_t], 1) + slot_t = tf.reduce_sum(slot_d, 2) + # slot_t = tf.layers.Flatten()(slot_d) + #slot_t = tf.reshape(slot_d, [d.get_shape()[0].value, -1]) + intent_output = tf.concat([d, intent_input], 1) + intent_output = tf.concat([intent_input, slot_t], 1) else: intent_output = tf.concat([d, intent_input], 1) #[12 384] From 6eece3360bc65e4268387dfd30692069c0fa2ea9 Mon Sep 17 00:00:00 2001 From: jasonchan_co <18933818702@163.com> Date: Sat, 3 Apr 2021 09:26:10 +0000 Subject: [PATCH 15/19] Debug --- train.py | 61 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/train.py b/train.py index 5f2a32e..a727150 100644 --- a/train.py +++ b/train.py @@ -143,37 +143,38 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, intent_input = final_state # [12 256]] with tf.variable_scope('intent_attn'): - attn_size = state_shape[2].value # dim(128) state_outputs : [12 23 128] - hidden = tf.expand_dims(state_outputs, 2) # Shape: batchsize x len x 1 x dim(128) - k = tf.get_variable("AttnW", [1, 1, attn_size, attn_size]) # 1 x 1 128 x 128 - # Attention weighted - hidden_features = tf.nn.conv2d(hidden, k, [1, 1, 1, 1], "SAME") - v = tf.get_variable("AttnV", [attn_size]) - - y = _linear(intent_input, attn_size, True) - y = tf.reshape(y, [-1, 1, 1, attn_size]) - s = tf.reduce_sum(v*tf.tanh(hidden_features + y), [2,3]) - a = tf.nn.softmax(s) - a = tf.expand_dims(a, -1) - a = tf.expand_dims(a, -1) - - d = tf.reduce_sum(a * hidden, [1, 2]) # a * hidden shape:[ 12 23 1 128] - sa = tf.shape(d) - if add_final_state_to_intent == True: - if remove_intent_attn == True: - - intent_output = tf.concat([tf.reshape(state_outputs, [bs, -1]), intent_input], 1) - elif interplay == True: - slot_t = tf.reduce_sum(slot_d, 2) - # slot_t = tf.layers.Flatten()(slot_d) - #slot_t = tf.reshape(slot_d, [d.get_shape()[0].value, -1]) - intent_output = tf.concat([d, intent_input], 1) - intent_output = tf.concat([intent_input, slot_t], 1) - else: - intent_output = tf.concat([d, intent_input], 1) #[12 384] + if remove_intent_attn == True: + intent_output = tf.concat([tf.reduce_sum(state_outputs, 1), intent_input], 1) + else: + attn_size = state_shape[2].value # dim(128) state_outputs : [12 23 128] + hidden = tf.expand_dims(state_outputs, 2) # Shape: batchsize x len x 1 x dim(128) + k = tf.get_variable("AttnW", [1, 1, attn_size, attn_size]) # 1 x 1 128 x 128 + # Attention weighted + hidden_features = tf.nn.conv2d(hidden, k, [1, 1, 1, 1], "SAME") + v = tf.get_variable("AttnV", [attn_size]) + + y = _linear(intent_input, attn_size, True) + y = tf.reshape(y, [-1, 1, 1, attn_size]) + s = tf.reduce_sum(v*tf.tanh(hidden_features + y), [2,3]) + a = tf.nn.softmax(s) + a = tf.expand_dims(a, -1) + a = tf.expand_dims(a, -1) + + d = tf.reduce_sum(a * hidden, [1, 2]) # a * hidden shape:[ 12 23 1 128] + # d: Shape [12 128] + sa = tf.shape(d) + if add_final_state_to_intent == True: + if interplay == True: + slot_t = tf.reduce_sum(slot_d, 1) + # slot_t = tf.reduce_sum(slot_d, 2) + # slot_t = tf.layers.Flatten()(slot_d) + #slot_t = tf.reshape(slot_d, [d.get_shape()[0].value, -1]) + intent_output = tf.concat([slot_t, d, intent_input], 1) + else: + intent_output = tf.concat([d, intent_input], 1) #[12 384] - else: - intent_output = d + else: + intent_output = d with tf.variable_scope('slot_gated'): slot_without_gate = tf.reshape( slot_d, [-1, attn_size]) From 82bd25e9bce7be5d69f5e389350dfb41e4b95d4e Mon Sep 17 00:00:00 2001 From: jasonchan_co <18933818702@163.com> Date: Sat, 3 Apr 2021 09:32:37 +0000 Subject: [PATCH 16/19] Debug --- train.py | 1 + 1 file changed, 1 insertion(+) diff --git a/train.py b/train.py index a727150..d8ce910 100644 --- a/train.py +++ b/train.py @@ -143,6 +143,7 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, intent_input = final_state # [12 256]] with tf.variable_scope('intent_attn'): + sa = tf.shape(final_state) if remove_intent_attn == True: intent_output = tf.concat([tf.reduce_sum(state_outputs, 1), intent_input], 1) else: From 70d883ebd27fc02ce17910e586a9ebff795341c3 Mon Sep 17 00:00:00 2001 From: jasonchan_co <18933818702@163.com> Date: Sat, 3 Apr 2021 10:24:47 +0000 Subject: [PATCH 17/19] Add extra parameters --- train.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/train.py b/train.py index d8ce910..2f0a1d2 100644 --- a/train.py +++ b/train.py @@ -29,7 +29,7 @@ #Model and Vocab parser.add_argument("--dataset", type=str, default=None, help="""Type 'atis' or 'snips' to use dataset provided by us or enter what ever you named your own dataset. Note, if you don't want to use this part, enter --dataset=''. It can not be None""") -parser.add_argument("--model_path", type=str, default='./model', help="Path to save model.") +parser.add_argument("--model_path", type=str, default='/content/drive/MyDrive/Data/SlotGate/interplay/snips', help="Path to save model.") parser.add_argument("--vocab_path", type=str, default='./vocab', help="Path to vocabulary files.") #Data @@ -39,6 +39,8 @@ parser.add_argument("--input_file", type=str, default='seq.in', help="Input file name.") parser.add_argument("--slot_file", type=str, default='seq.out', help="Slot file name.") parser.add_argument("--intent_file", type=str, default='label', help="Intent file name.") +parser.add_argument("--ckpt", default='/content/drive/MyDrive/Data/SlotGate/interplay/snips', help='The path to the model file') + parser.add_argument("--interplay", action='store_true') parser.add_argument("--remove_intent_attn", action='store_true') From 2554e6be12429e950a7dd888085fc47d2c183c86 Mon Sep 17 00:00:00 2001 From: jasonchan_co <18933818702@163.com> Date: Sun, 4 Apr 2021 03:43:38 +0000 Subject: [PATCH 18/19] Add extra parameters --- train.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/train.py b/train.py index 2f0a1d2..b13b9dc 100644 --- a/train.py +++ b/train.py @@ -180,8 +180,9 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, intent_output = d with tf.variable_scope('slot_gated'): - slot_without_gate = tf.reshape( slot_d, [-1, attn_size]) - slot_without_gate = tf.concat([slot_without_gate, slot_inputs], 1) + if remove_gate == True: + slot_without_gate = tf.reshape( slot_d, [-1, attn_size]) + slot_without_gate = tf.concat([slot_without_gate, slot_inputs], 1) intent_gate = _linear(intent_output, attn_size, True) intent_gate = tf.reshape(intent_gate, [-1, 1, intent_gate.get_shape()[1].value]) From a0a6fd839d1ee27d3d68d85e9dffbba084237447 Mon Sep 17 00:00:00 2001 From: Zishen Chen <32064967+jasonchan117@users.noreply.github.com> Date: Tue, 6 Apr 2021 15:49:09 +0800 Subject: [PATCH 19/19] Remove extra information --- train.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/train.py b/train.py index b13b9dc..a8ddd57 100644 --- a/train.py +++ b/train.py @@ -42,9 +42,9 @@ parser.add_argument("--ckpt", default='/content/drive/MyDrive/Data/SlotGate/interplay/snips', help='The path to the model file') -parser.add_argument("--interplay", action='store_true') -parser.add_argument("--remove_intent_attn", action='store_true') -parser.add_argument("--remove_gate", action='store_true') +parser.add_argument("--interplay", action='store_true', help='Use the interplay between slot filling and intent detection or not.') +parser.add_argument("--remove_intent_attn", action='store_true', help='Remove the intent attention.') +parser.add_argument("--remove_gate", action='store_true', help='Remove the gate.') arg=parser.parse_args() @@ -145,7 +145,7 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, intent_input = final_state # [12 256]] with tf.variable_scope('intent_attn'): - sa = tf.shape(final_state) + if remove_intent_attn == True: intent_output = tf.concat([tf.reduce_sum(state_outputs, 1), intent_input], 1) else: @@ -213,7 +213,7 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, slot = _linear(slot_output, slot_size, True) # slot_output: [276 256] - outputs = [slot, intent, sa] + outputs = [slot, intent] return outputs # Create Training Model @@ -230,7 +230,7 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, slots_shape = tf.shape(slots) slots_reshape = tf.reshape(slots, [-1]) # Debug print -sa = training_outputs[2] +#sa = training_outputs[2] slot_outputs = training_outputs[0] with tf.variable_scope('slot_loss'): crossent = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=slots_reshape, logits=slot_outputs) @@ -267,7 +267,7 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, update_slot = opt.apply_gradients(zip(clipped_gradients_slot, slot_params)) update_intent = opt.apply_gradients(zip(clipped_gradients_intent, intent_params), global_step=global_step) # Debug output -training_outputs = [global_step, slot_loss, update_intent, update_slot, gradient_norm_intent, gradient_norm_slot, sa] +training_outputs = [global_step, slot_loss, update_intent, update_slot, gradient_norm_intent, gradient_norm_slot] inputs = [input_data, sequence_length, slots, slot_weights, intent] # Create Inference Model @@ -325,7 +325,7 @@ def createModel(input_data, input_size, sequence_length, slot_size, intent_size, epochs += 1 logging.info('Step: ' + str(step)) logging.info('Epochs: ' + str(epochs)) - logging.info('Shape: '+ str(ret[6])) + # logging.info('Shape: '+ str(ret[6])) logging.info('Loss: ' + str(loss/num_loss)) num_loss = 0 loss = 0.0