forked from lzy1732008/GaussionTransformer
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathModel.py
More file actions
122 lines (103 loc) · 8.08 KB
/
Copy pathModel.py
File metadata and controls
122 lines (103 loc) · 8.08 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
import hyperparams as param
import modules
import tensorflow as tf
class GaussionTransformer:
def __init__(self):
self.is_training = True
self.inputX_word = tf.placeholder(name="inputX_word", dtype=tf.float64,
shape=[None, param.Hyperparams.X_maxlen, param.Hyperparams.word_dimension])
self.inputX_char = tf.placeholder(name="inputX_char", dtype=tf.float64,
shape=[None, param.Hyperparams.X_maxlen, param.Hyperparams.char_dimension])
self.inputY_word = tf.placeholder(name="inputY_word", dtype=tf.float64,
shape=[None, param.Hyperparams.Y_maxLen, param.Hyperparams.word_dimension])
self.inputY_char = tf.placeholder(name="inputY_char", dtype=tf.float64,
shape=[None, param.Hyperparams.Y_maxLen, param.Hyperparams.char_dimension])
self.y = tf.placeholder(name="target_y", dtype=tf.int32, shape=[None, 2])
self.dropout_rate = tf.placeholder(tf.float64, name='keep_prob')
self.run_GaussionTransformer()
def run_GaussionTransformer(self):
embeddingScope = "embeddingBlock"
encodingBlock = "encodingBlcok"
interactionBlock = "interactionBlock"
comparisonBlock = "comparisonBlock"
self.positionEncoding1 = modules.positional_encoding(inputs=self.inputX_word,
num_units=param.Hyperparams.postion_dimension) #(N, L, postion_dimension)
self.positionEncoding2 = modules.positional_encoding(inputs=self.inputY_word,
num_units=param.Hyperparams.postion_dimension)
self.shift = tf.Variable(tf.abs(tf.random_normal([1],stddev=0, seed=0,dtype=tf.float64)) + 0.001, trainable=True, name='shift', dtype=tf.float64)
self.bias = tf.Variable(-tf.abs(tf.random_normal([1],stddev=0, seed=0,dtype=tf.float64)), trainable=True, name='bias', dtype=tf.float64)
with tf.variable_scope(embeddingScope,reuse=False):
self.embedding_1 = modules.embedding_block(self.inputX_word,
self.inputX_char,
self.positionEncoding1,
scope="embedding_1")
self.embedding_2 = modules.embedding_block(self.inputY_word,
self.inputY_char,
self.positionEncoding2,
scope="embedding_2")
self.embedding_1 = tf.check_numerics(self.embedding_1, "nan happend!!!!")
self.embedding_2 = tf.check_numerics(self.embedding_2, "nan happend!!!!")
with tf.variable_scope(encodingBlock, reuse=False):
self.encoding_1 = self.embedding_1
self.encoding_2 = self.embedding_2
for i in range(param.Hyperparams.encoder_num_blocks):
with tf.variable_scope("multihead-atttention_{0}".format(i), reuse=False):#这里添加scope， {}.format
self.encoding_1 = modules.multihead_attention(self.encoding_1,
self.shift,
self.bias,
num_heads=param.Hyperparams.num_heads,
dropout_rate=self.dropout_rate,
is_training=self.is_training)
self.encoding_1 = tf.check_numerics(self.encoding_1, "encoding nan happend!!!! multihead-atttention_{0}".format(i))
with tf.variable_scope("multihead-atttention_{0}".format(i), reuse=True): # 这里添加scope， {}.format
self.encoding_2 = modules.multihead_attention(self.encoding_2,
self.shift,
self.bias,
num_heads=param.Hyperparams.num_heads,
dropout_rate=self.dropout_rate,
is_training=self.is_training)
self.encoding_1 += self.positionEncoding1
self.encoding_2 += self.positionEncoding2
with tf.variable_scope(interactionBlock, reuse=None):
self.interaction_1 = self.encoding_1
self.interaction_2 = self.encoding_2
for i in range(param.Hyperparams.inter_num_blocks):
with tf.variable_scope("interaction_{0}".format(i), reuse=False):
self.interaction_1 = modules.InteractionBlock(queries=self.interaction_1,
keys=self.interaction_2,
shift=self.shift,
bias=self.bias,
num_heads=param.Hyperparams.num_heads,
dropout_rate=self.dropout_rate,
is_training=self.is_training)
self.interaction_1 = tf.check_numerics(self.interaction_1, "nan happend!!!!")
with tf.variable_scope("interaction_{0}".format(i), reuse=True):
self.interaction_2 = modules.InteractionBlock(queries=self.interaction_2,
keys=self.interaction_1,
shift=self.shift,
bias=self.bias,
num_heads=param.Hyperparams.num_heads,
dropout_rate=self.dropout_rate,
is_training=self.is_training)
self.encoding_1 = tf.check_numerics(self.encoding_1,"encoding_1 is nan")
self.encoding_2 = tf.check_numerics(self.encoding_2,"encoding_2 is nan")
self.interaction_1 = tf.check_numerics(self.interaction_1,"interaction_ 1 is nan")
self.interaction_2 = tf.check_numerics(self.interaction_2,"interaction_2 is nan")
with tf.variable_scope(comparisonBlock, reuse=None):
self.logit = modules.ComparisonBlock(input1_Encoding=self.encoding_1,
input1_Interaction=self.interaction_1,
input2_Encoding=self.encoding_2,
input2_Interaction=self.interaction_2)
self.pred_y = tf.argmax(tf.nn.softmax(self.logit),1)
if self.is_training:
with tf.name_scope("optimize"):
# 损失函数，交叉熵
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=self.logit,
labels=self.y) # 对logits进行softmax操作后，做交叉墒，输出的是一个向量
self.loss = tf.reduce_mean(cross_entropy) # 将交叉熵向量求和，即可得到交叉熵
# 优化器
self.optim = tf.train.AdamOptimizer(learning_rate=param.Hyperparams.lr).minimize(self.loss)
with tf.name_scope("accuracy"):
# 准确率
correct_pred = tf.equal(tf.argmax(self.y, 1), self.pred_y) # 由于input_y也是onehot编码，因此，调用tf.argmax(self.input_y)得到的是1所在的下表
self.acc = tf.reduce_mean(tf.cast(correct_pred, tf.float32))