add layer_dropout

Author: ewrfcas

Attention/context2query_attention.py KerasLayer/context2query_attention.py

@@ -0,0 +1,55 @@
+# ! -*- coding: utf-8 -*-
+from keras.engine.topology import Layer
+from keras.regularizers import *
+import tensorflow as tf
+import keras.backend as K
+
+class GateAttention(Layer):
+    def __init__(self, filters, dropout=0.0, regularizer=l2(3e-7), **kwargs):
+        self.filters = filters
+        self.dropout = dropout
+        self.regularizer = regularizer
+        super(GateAttention, self).__init__(**kwargs)
+
+    def build(self, input_shape):
+        self.WC = self.add_weight(name='WC',
+                                  shape=(input_shape[0][-1], self.filters),
+                                  regularizer=self.regularizer,
+                                  initializer='glorot_uniform',
+                                  trainable=True)
+        self.WQ = self.add_weight(name='WQ',
+                                  shape=(input_shape[1][-1], self.filters),
+                                  regularizer=self.regularizer,
+                                  initializer='glorot_uniform',
+                                  trainable=True)
+        self.V = self.add_weight(name='V',
+                                 shape=(2 * input_shape[1][-1], self.filters),
+                                 regularizer=self.regularizer,
+                                 initializer='glorot_uniform',
+                                 trainable=True)
+        super(GateAttention, self).build(input_shape)
+
+    def mask_logits(self, inputs, mask, clen, mask_value=-1e12):
+        shapes = [x if x != None else -1 for x in inputs.shape.as_list()]
+        mask = K.cast(mask, tf.int32)
+        mask = K.one_hot(mask[:, 0], shapes[-1])
+        mask = 1 - K.cumsum(mask, 1)
+        mask = tf.cast(mask, tf.float32)
+        mask = tf.tile(tf.expand_dims(mask, axis=1), [1, clen, 1])
+        return inputs + mask_value * (1 - mask)
+
+    def call(self, x, mask=None):
+        x_cont, x_ques, ques_len = x
+        input_shape_ = x_cont.shape.as_list()
+        x_cont_ = tf.nn.relu(K.dot(x_cont, self.WC))
+        x_ques_ = tf.nn.relu(K.dot(x_ques, self.WQ))
+        logits = tf.matmul(x_cont_, x_ques_, transpose_b=True) / (self.filters ** 0.5)
+        logits = self.mask_logits(logits, ques_len, clen=input_shape_[1])
+        logits = tf.nn.softmax(logits)
+        C = tf.matmul(logits, x_ques)
+        res = tf.concat([x_cont, C], axis=2)
+        gate = tf.nn.sigmoid(K.dot(res, self.V))
+        return gate
+
+    def compute_output_shape(self, input_shape):
+        return input_shape[0]

KerasLayer/gate_attention.py

@@ -36,15 +36,15 @@ def mask_logits(self, inputs, mask, mask_value=-1e12):
         mask = tf.reshape(mask, [shapes[0], 1, 1, shapes[-1]])
         return inputs + mask_value * (1 - mask)
 
-    def dot_product_attention(self, x, seq_len=None, dropout=0.1):
+    def dot_product_attention(self, x, seq_len=None, dropout=0.1, training=None):
         q, k, v = x
         logits = tf.matmul(q, k, transpose_b=True)
         if self.bias:
             logits += self.b
         if seq_len is not None:
             logits = self.mask_logits(logits, seq_len)
         weights = tf.nn.softmax(logits, name="attention_weights")
-        weights = tf.nn.dropout(weights, 1.0 - dropout)
+        weights = K.in_train_phase(K.dropout(weights, dropout), weights, training=training)
         x = tf.matmul(weights, v)
         return x
 
@@ -56,7 +56,7 @@ def combine_last_two_dimensions(self, x):
         ret.set_shape(new_shape)
         return ret
 
-    def call(self, x, mask=None):
+    def call(self, x, mask=None, training=None):
         memory, query, seq_len = x
         Q = self.split_last_dimension(query, self.num_heads)
         memory = tf.split(memory, 2, axis=2)
@@ -65,7 +65,7 @@ def call(self, x, mask=None):
 
         key_depth_per_head = self.units // self.num_heads
         Q *= (key_depth_per_head ** -0.5)
-        x = self.dot_product_attention([Q, K, V], seq_len, dropout=self.dropout)
+        x = self.dot_product_attention([Q, K, V], seq_len, dropout=self.dropout, training=training)
         x = self.combine_last_two_dimensions(tf.transpose(x, [0,2,1,3]))
 
         return x

Attention/multihead_attention.py KerasLayer/multihead_attention.py

@@ -1,12 +1,13 @@
 from keras.layers import *
 from keras.regularizers import *
 from keras.models import *
-from Attention.context2query_attention import context2query_attention
-from Attention.multihead_attention import Attention as MultiHeadAttention
-from Attention.position_embedding import Position_Embedding as PositionEmbedding
+from KerasLayer.context2query_attention import context2query_attention
+from KerasLayer.multihead_attention import Attention as MultiHeadAttention
+from KerasLayer.position_embedding import Position_Embedding as PositionEmbedding
 from keras import layers
 from keras.optimizers import *
 from keras.callbacks import *
+from KerasLayer.layer_dropout import LayerDropout
 from keras.initializers import *
 
 regularizer = l2(3e-7)
@@ -21,12 +22,6 @@ def mask_logits(inputs, mask, mask_value=-1e12, axis=1, time_dim=1):
         mask = tf.expand_dims(mask, axis)
     return inputs + mask_value * (1 - mask)
 
-def layer_dropout(x, residual, dropout):
-    pred = tf.random_uniform([]) < dropout
-    x = Dropout(dropout)(x)
-    x = layers.add([x, residual])
-    return Lambda(lambda x: tf.cond(pred, lambda: x[1], lambda: x[0]))([x, residual])
-
 def highway(highway_layers, x, num_layers=2, dropout=0.0):
     # reduce dim
     x = highway_layers[0](x)
@@ -45,7 +40,7 @@ def conv_block(conv_layers, x, num_conv=4, dropout=0.0, l=1., L=1.):
         x = Dropout(dropout)(x)
         x = conv_layers[i][0](x)
         x = conv_layers[i][1](x)
-        x = layer_dropout(x, residual, dropout * (l / L))
+        x = LayerDropout(dropout * (l / L))([x, residual])
     x = Lambda(lambda v: tf.squeeze(v, axis=2))(x)
     return x
 
@@ -56,7 +51,7 @@ def attention_block(attention_layer, x, seq_len, dropout=0.0, l=1., L=1.):
     x1 = attention_layer[0](x)
     x2 = attention_layer[1](x)
     x = attention_layer[2]([x1,x2,seq_len])
-    x = layer_dropout(x, residual, dropout * (l / L))
+    x = LayerDropout(dropout * (l / L))([x, residual])
     return x
 
 def feed_forward_block(FeedForward_layers, x, dropout=0.0, l=1., L=1.):
@@ -65,7 +60,7 @@ def feed_forward_block(FeedForward_layers, x, dropout=0.0, l=1., L=1.):
     x = Dropout(dropout)(x)
     x = FeedForward_layers[0](x)
     x = FeedForward_layers[1](x)
-    x = layer_dropout(x, residual, dropout * (l / L))
+    x = LayerDropout(dropout * (l / L))([x, residual])
     return x
 
 def output_block(x1, x2, ans_limit=50):

Attention/multihead_attention_bysu.py KerasLayer/multihead_attention_bysu.py

@@ -1,5 +1,5 @@
 import tensorflow as tf
-from layers import regularizer, residual_block, highway, conv, mask_logits, optimized_trilinear_for_attention, total_params
+from NetModel.QANet_tensorflow.layers import regularizer, residual_block, highway, conv, mask_logits, optimized_trilinear_for_attention, total_params
 
 class Model(object):
     def __init__(self, config, word_mat=None, char_mat=None, test=False):
@@ -12,12 +12,12 @@ def __init__(self, config, word_mat=None, char_mat=None, test=False):
         self.ans_limit = config['ans_limit']
         self.filters = config['filters']
         self.num_heads = config['num_heads']
-        self.dropout = config['dropout']
         self.batch_size = config['batch_size']
         self.l2_norm = config['l2_norm']
         self.decay = config['decay']
         self.learning_rate = config['learning_rate']
         self.grad_clip = config['grad_clip']
+        self.dropout = tf.placeholder_with_default(0.0, (), name="dropout")
 
         # embedding layer
         self.word_mat = tf.get_variable("word_mat", initializer=tf.constant(word_mat, dtype=tf.float32), trainable=False)

Attention/position_embedding.py KerasLayer/position_embedding.py

@@ -289,7 +289,6 @@ def dot_product_attention(q,
             # shapes = [x  if x != None else -1 for x in logits.shape.as_list()]
             mask = tf.expand_dims(mask, axis=1)
             mask = tf.expand_dims(mask, axis=1)
-            # mask = tf.reshape(mask, [shapes[0],1,1,shapes[-1]])
             logits = mask_logits(logits, mask)
         weights = tf.nn.softmax(logits, name="attention_weights")
         # dropping out the attention links for each of the heads

Attention/position_embedding_bysu.py KerasLayer/position_embedding_bysu.py

@@ -1,8 +1,8 @@
 import numpy as np
 import pandas as pd
-import QANet_model
+from NetModel.QANet_tensorflow import QANet_model
 import tensorflow as tf
-import util
+from NetModel.QANet_tensorflow import util
 import json
 import os
 import time
@@ -115,7 +115,8 @@ def cal_ETA(t_start, i, n_batch):
             loss_value, _ = sess.run([model.loss, model.train_op],
                                      feed_dict={model.contw_input_: contw_input, model.quesw_input_: quesw_input,
                                                 model.contc_input_: contc_input, model.quesc_input_: quesc_input,
-                                                model.y_start_: y_start, model.y_end_: y_end})
+                                                model.y_start_: y_start, model.y_end_: y_end,
+                                                model.dropout: config['dropout']})
             sum_loss += loss_value
             last_train_str = "\r[epoch:%d/%d, steps:%d/%d] -ETA: %ds -loss: %.4f" % (
             epoch + 1, config['epoch'], i + 1, n_batch, cal_ETA(t_start, i, n_batch), sum_loss / (i + 1))