Python tensorflow 模块,name_scope() 实例源码
我们从Python开源项目中,提取了以下50个代码示例,用于说明如何使用tensorflow.name_scope()。
def calculate_loss_mix2(self, predictions, predictions_class, predictions_encoder, labels, **unused_params):
with tf.name_scope("loss_mix2"):
float_labels = tf.cast(labels, tf.float32)
float_encoders = float_labels
for i in range(FLAGS.encoder_layers):
var_i = np.loadtxt(FLAGS.autoencoder_dir+'autoencoder_layer%d.model' % i)
weight_i = tf.constant(var_i[:-1,:],dtype=tf.float32)
bias_i = tf.reshape(tf.constant(var_i[-1,dtype=tf.float32),[-1])
float_encoders = tf.nn.xw_plus_b(float_encoders,weight_i,bias_i)
if i<FLAGS.encoder_layers-1:
float_encoders = tf.nn.relu(float_encoders)
else:
hidden_mean = tf.reduce_mean(float_encoders,axis=1,keep_dims=True)
hidden_std = tf.sqrt(tf.reduce_mean(tf.square(float_encoders-hidden_mean),keep_dims=True))
float_encoders = (float_encoders-hidden_mean)/(hidden_std+1e-6)
#float_encoders = tf.nn.sigmoid(float_encoders)
cross_entropy_encoder = 0.1*self.calculate_mseloss(predictions_encoder,float_encoders)
cross_entropy_loss = self.calculate_loss(predictions,labels)
return cross_entropy_encoder+cross_entropy_loss, float_encoders
#return cross_entropy_encoder,float_encoders
def inference(self):
"""main computation graph here: 1. embeddding layer,2.Bi-LSTM layer,3.concat,4.FC layer 5.softmax """
#1.get emebedding of words in the sentence
self.embedded_words = tf.nn.embedding_lookup(self.Embedding,self.input_x) #shape:[None,sentence_length,embed_size]
#2. Bi-lstm layer
# define lstm cess:get lstm cell output
lstm_fw_cell=rnn.BasicLSTMCell(self.hidden_size) #forward direction cell
lstm_bw_cell=rnn.BasicLSTMCell(self.hidden_size) #backward direction cell
if self.dropout_keep_prob is not None:
lstm_fw_cell=rnn.DropoutWrapper(lstm_fw_cell,output_keep_prob=self.dropout_keep_prob)
lstm_bw_cell=rnn.DropoutWrapper(lstm_bw_cell,output_keep_prob=self.dropout_keep_prob)
# bidirectional_dynamic_rnn: input: [batch_size,max_time,input_size]
# output: A tuple (outputs,output_states)
# where:outputs: A tuple (output_fw,output_bw) containing the forward and the backward rnn output `Tensor`.
outputs,_=tf.nn.bidirectional_dynamic_rnn(lstm_fw_cell,lstm_bw_cell,self.embedded_words,dtype=tf.float32) #[batch_size,sequence_length,hidden_size] #creates a dynamic bidirectional recurrent neural network
print("outputs:===>",outputs) #outputs:(<tf.Tensor 'bidirectional_rnn/fw/fw/transpose:0' shape=(?,5,100) dtype=float32>,<tf.Tensor 'ReverseV2:0' shape=(?,100) dtype=float32>))
#3. concat output
output_rnn=tf.concat(outputs,axis=2) #[batch_size,hidden_size*2]
self.output_rnn_last=tf.reduce_mean(output_rnn,axis=1) #[batch_size,hidden_size*2] #output_rnn_last=output_rnn[:,-1,:] ##[batch_size,hidden_size*2] #Todo
print("output_rnn_last:", self.output_rnn_last) # <tf.Tensor 'strided_slice:0' shape=(?,200) dtype=float32>
#4. logits(use linear layer)
with tf.name_scope("output"): #inputs: A `Tensor` of shape `[batch_size,dim]`. The forward activations of the input network.
logits = tf.matmul(self.output_rnn_last, self.W_projection) + self.b_projection # [batch_size,num_classes]
return logits
def value_transition(self, curr_state, next_symbols, batch_size):
first_value_token = self.num_functions + self.num_begin_tokens + self.num_control_tokens
num_value_tokens = self.output_size - first_value_token
with tf.name_scope('grammar_transition'):
adjusted_next_symbols = tf.where(next_symbols >= self.num_control_tokens, next_symbols + (first_value_token - self.num_control_tokens), next_symbols)
assert1 = tf.Assert(tf.reduce_all(tf.logical_and(next_symbols < num_value_tokens, next_symbols >= 0)), [curr_state, next_symbols])
with tf.control_dependencies([assert1]):
transitions = tf.gather(tf.constant(self.transition_matrix), curr_state)
assert transitions.get_shape()[1:] == (self.output_size,)
indices = tf.stack((tf.range(0, batch_size), adjusted_next_symbols), axis=1)
next_state = tf.gather_nd(transitions, indices)
assert2 = tf.Assert(tf.reduce_all(next_state >= 0), adjusted_next_symbols, next_state])
with tf.control_dependencies([assert2]):
return tf.identity(next_state)
def finalize(self, outputs : BeamSearchOptimizationDecoderOutput, final_state : BeamSearchOptimizationDecoderState, sequence_lengths):
# all output fields are [max_time,batch_size,...]
predicted_ids = tf.contrib.seq2seq.gather_tree(
outputs.predicted_ids, outputs.parent_ids,
sequence_length=sequence_lengths, name='predicted_ids')
total_loss = tf.reduce_sum(outputs.loss, axis=0, name='violation_loss')
predicted_time = tf.shape(predicted_ids)[0]
last_score = predicted_time-1
with tf.name_scope('gold_score'):
gold_score = outputs.gold_score[last_score]
with tf.name_scope('sequence_scores'):
sequence_scores = outputs.scores[last_score]
return FinalBeamSearchOptimizationDecoderOutput(beam_search_decoder_output=outputs,
predicted_ids=predicted_ids,
scores=sequence_scores,
gold_score=gold_score,
gold_beam_id=final_state.gold_beam_id,
num_available_beams=final_state.num_available_beams,
total_violation_loss=total_loss), final_state
def step(self, time, inputs, state, name=None):
with tf.name_scope(name, "GrammarDecodingStep", (time, state)):
decoder_state, grammar_state = state
cell_outputs, cell_state = self._cell(inputs, decoder_state)
if self._output_layer is not None:
cell_outputs = self._output_layer(cell_outputs)
grammar_cell_outputs = self._grammar_helper.constrain_logits(cell_outputs, grammar_state)
cell_outputs = grammar_cell_outputs
sample_ids = self._helper.sample(time=time, outputs=grammar_cell_outputs, state=cell_state)
(finished, next_inputs, next_decoder_state) = self._helper.next_inputs(
time=time,
outputs=cell_outputs,
state=cell_state,
sample_ids=sample_ids)
if self._fixed_outputs is not None:
next_grammar_state = self._grammar_helper.transition(grammar_state, self._fixed_outputs.read(time), self.batch_size)
else:
next_grammar_state = self._grammar_helper.transition(grammar_state, sample_ids, self.batch_size)
next_state = (next_decoder_state, next_grammar_state)
outputs = BasicDecoderOutput(cell_outputs, sample_ids)
return (outputs, next_state, finished)
def create_optimizer():
optimizer = tf.train.AdamOptimizer(learning_rate=FLAGS.learning_rate,
beta1=FLAGS.beta1,
beta2=FLAGS.beta2,
epsilon=FLAGS.epsilon)
return optimizer
# Towers
# ======
# In order to properly make use of multiple GPU's,one must introduce new abstractions,
# not present when using a single GPU,that facilitate the multi-GPU use case.
# In particular,one must introduce a means to isolate the inference and gradient
# calculations on the varIoUs GPU's.
# The abstraction we intoduce for this purpose is called a 'tower'.
# A tower is specified by two properties:
# * **Scope** - A scope,as provided by `tf.name_scope()`,
# is a means to isolate the operations within a tower.
# For example,all operations within 'tower 0' Could have their name prefixed with `tower_0/`.
# * **Device** - A hardware device,as provided by `tf.device()`,
# on which all operations within the tower execute.
# For example,all operations of 'tower 0' Could execute on the first GPU `tf.device('/gpu:0')`.
def highway(self, input_1, input_2, size_1, size_2, l2_penalty=1e-8, layer_size=1):
output = input_2
for idx in range(layer_size):
with tf.name_scope('output_lin_%d' % idx):
W = tf.Variable(tf.truncated_normal([size_2,size_1], stddev=0.1), name="W")
b = tf.Variable(tf.constant(0.1, shape=[size_1]), name="b")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(W))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(b))
output = tf.nn.relu(tf.nn.xw_plus_b(output,W,b))
with tf.name_scope('transform_lin_%d' % idx):
W = tf.Variable(tf.truncated_normal([size_1, value=l2_penalty*tf.nn.l2_loss(b))
transform_gate = tf.sigmoid(tf.nn.xw_plus_b(input_1,b))
carry_gate = tf.constant(1.0) - transform_gate
output = transform_gate * output + carry_gate * input_1
return output
def calculate_loss_distill_boost(self, labels_distill, **unused_params):
with tf.name_scope("loss_distill_boost"):
print("loss_distill_boost")
epsilon = 10e-6
float_labels = tf.cast(labels, tf.float32)
batch_size = tf.shape(float_labels)[0]
float_labels_distill = tf.cast(labels_distill, tf.float32)
error = tf.negative(float_labels * tf.log(float_labels_distill + epsilon) + (
1 - float_labels) * tf.log(1 - float_labels_distill + epsilon))
error = tf.reduce_sum(error,keep_dims=True)
alpha = error / tf.reduce_sum(error) * tf.cast(batch_size,dtype=tf.float32)
alpha = tf.clip_by_value(alpha, 0.5, 5)
alpha = alpha / tf.reduce_sum(alpha) * tf.cast(batch_size,dtype=tf.float32)
cross_entropy_loss = float_labels * tf.log(predictions + epsilon) + (
1 - float_labels) * tf.log(1 - predictions + epsilon)
cross_entropy_loss = tf.negative(cross_entropy_loss * alpha)
return tf.reduce_mean(tf.reduce_sum(cross_entropy_loss, 1))
def calculate_loss_distill_relabel(self, **unused_params):
with tf.name_scope("loss_distill_relabel"):
print("loss_distill_relabel")
epsilon = 10e-6
float_labels = tf.cast(labels, tf.float32)
sum_labels = tf.cast(tf.reduce_sum(float_labels),dtype=tf.int32)
pos_distill, _ = tf.nn.top_k(tf.reshape(labels_distill,[-1]), k=sum_labels)
labels_true = tf.ones(tf.shape(labels))
labels_false = tf.zeros(tf.shape(labels))
labels_add = tf.where(tf.greater_equal(labels_distill, pos_distill[-1]), labels_true, labels_false)
print(labels_add.get_shape().as_list())
float_labels = float_labels+labels_add*(1.0-float_labels)
cross_entropy_loss = float_labels * tf.log(predictions + epsilon) + (
1 - float_labels) * tf.log(1 - predictions + epsilon)
cross_entropy_loss = tf.negative(cross_entropy_loss)
return tf.reduce_mean(tf.reduce_sum(cross_entropy_loss, 1))
def calculate_loss(self, **unused_params):
bound = FLAGS.softmax_bound
vocab_size_1 = bound
with tf.name_scope("loss_softmax"):
epsilon = 10e-8
float_labels = tf.cast(labels, tf.float32)
labels_1 = float_labels[:,:vocab_size_1]
predictions_1 = predictions[:,:vocab_size_1]
cross_entropy_loss = CrossEntropyLoss().calculate_loss(predictions_1,labels_1)
lables_2 = float_labels[:,vocab_size_1:]
predictions_2 = predictions[:,vocab_size_1:]
# l1 normalization (labels are no less than 0)
label_rowsum = tf.maximum(
tf.reduce_sum(lables_2, 1, keep_dims=True),
epsilon)
label_append = 1.0-tf.reduce_max(lables_2, keep_dims=True)
norm_float_labels = tf.concat((tf.div(lables_2, label_rowsum),label_append),axis=1)
predictions_append = 1.0-tf.reduce_sum(predictions_2, keep_dims=True)
softmax_outputs = tf.concat((predictions_2,predictions_append),axis=1)
softmax_loss = norm_float_labels * tf.log(softmax_outputs + epsilon) + (
1 - norm_float_labels) * tf.log(1 - softmax_outputs + epsilon)
softmax_loss = tf.negative(tf.reduce_sum(softmax_loss, 1))
return tf.reduce_mean(softmax_loss) + cross_entropy_loss
def calculate_loss(self, **unused_params):
bound = FLAGS.softmax_bound
vocab_size_1 = bound
with tf.name_scope("loss_softmax"):
epsilon = 10e-8
float_labels = tf.cast(labels, tf.float32)
labels_1 = float_labels[:,:vocab_size_1]
predictions_1 = predictions[:,:vocab_size_1]
cross_entropy_loss = CrossEntropyLoss().calculate_loss(predictions_1,labels_1)
lables_2 = float_labels[:,vocab_size_1:]
predictions_2 = predictions[:,vocab_size_1:]
# l1 normalization (labels are no less than 0)
label_rowsum = tf.maximum(
tf.reduce_sum(lables_2,
epsilon)
label_append = 1.0-tf.reduce_max(lables_2, keep_dims=True)
norm_float_labels = tf.concat((tf.div(lables_2,axis=1)
predictions_append = 1.0-tf.reduce_sum(predictions_2, keep_dims=True)
softmax_outputs = tf.concat((predictions_2,axis=1)
softmax_loss = norm_float_labels * tf.log(softmax_outputs + epsilon) + (
1 - norm_float_labels) * tf.log(1 - softmax_outputs + epsilon)
softmax_loss = tf.negative(tf.reduce_sum(softmax_loss, 1))
return tf.reduce_mean(softmax_loss) + cross_entropy_loss
def get_input_evaluation_tensors(reader,
data_pattern,
batch_size=1024):
logging.info("Using batch size of " + str(batch_size) + " for evaluation.")
with tf.name_scope("eval_input"):
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find the evaluation files.")
logging.info("number of evaluation files: " + str(len(files)))
files.sort()
filename_queue = tf.train.string_input_producer(
files, shuffle=False, num_epochs=1)
eval_data = reader.prepare_reader(filename_queue)
return tf.train.batch(
eval_data,
batch_size=batch_size,
capacity=3 * batch_size,
allow_smaller_final_batch=True,
enqueue_many=True)
def calculate_loss(self, weights=None, **unused_params):
with tf.name_scope("loss_xent"):
epsilon = 10e-6
if FLAGS.label_smoothing:
float_labels = smoothing(labels)
else:
float_labels = tf.cast(labels, tf.float32)
cross_entropy_loss = float_labels * tf.log(predictions + epsilon) + (
1 - float_labels) * tf.log(1 - predictions + epsilon)
cross_entropy_loss = tf.negative(cross_entropy_loss)
if weights is not None:
print cross_entropy_loss, weights
weighted_loss = tf.einsum("ij,i->ij", cross_entropy_loss, weights)
print "create weighted_loss", weighted_loss
return tf.reduce_mean(tf.reduce_sum(weighted_loss, 1))
else:
return tf.reduce_mean(tf.reduce_sum(cross_entropy_loss, 1))
def get_input_evaluation_tensors(reader,
batch_size=256):
logging.info("Using batch size of " + str(batch_size) + " for evaluation.")
with tf.name_scope("eval_input"):
files = gfile.Glob(data_pattern)
if not files:
print data_pattern, files
raise IOError("Unable to find the evaluation files.")
logging.info("number of evaluation files: " + str(len(files)))
files.sort()
filename_queue = tf.train.string_input_producer(
files,
enqueue_many=True)
def get_input_evaluation_tensors(reader,
capacity=3 * FLAGS.batch_size,
enqueue_many=True)
def get_input_data_tensors(reader,
data_pattern,
batch_size=256):
logging.info("Using batch size of " + str(batch_size) + " for evaluation.")
with tf.name_scope("eval_input"):
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find the evaluation files.")
logging.info("number of evaluation files: " + str(len(files)))
files.sort()
filename_queue = tf.train.string_input_producer(
files,
capacity=4 * batch_size,
enqueue_many=True)
def get_input_data_tensors(reader,
batch_size=256,
num_epochs=None):
logging.info("Using batch size of " + str(batch_size) + " for training.")
with tf.name_scope("train_input"):
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find training files. data_pattern='" +
data_pattern + "'.")
logging.info("Number of training files: %s.", str(len(files)))
files.sort()
filename_queue = tf.train.string_input_producer(
files, num_epochs=num_epochs, shuffle=False)
training_data = reader.prepare_reader(filename_queue)
return tf.train.batch(
training_data,
capacity=FLAGS.batch_size * 4,
enqueue_many=True)
def get_input_data_tensors(reader,
batch_size=256):
logging.info("Using batch size of " + str(batch_size) + " for evaluation.")
with tf.name_scope("eval_input"):
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find the evaluation files.")
logging.info("number of evaluation files: " + str(len(files)))
files.sort()
filename_queue = tf.train.string_input_producer(
files,
enqueue_many=True)
def get_input_data_tensors(reader,
capacity=batch_size,
enqueue_many=True)
def baseline_forward(self, X, size, n_class):
shape = X.get_shape()
_X = tf.transpose(X, [1, 0, 2]) # batch_size x sentence_length x word_length -> batch_size x sentence_length x word_length
_X = tf.reshape(_X, [-1, int(shape[2])]) # (batch_size x sentence_length) x word_length
seq = tf.split(0, int(shape[1]), _X) # sentence_length x (batch_size x word_length)
with tf.name_scope("LSTM"):
lstm_cell = rnn_cell.BasicLSTMCell(size, forget_bias=1.0)
outputs, states = rnn.rnn(lstm_cell, seq, dtype=tf.float32)
with tf.name_scope("LSTM-Classifier"):
W = tf.Variable(tf.random_normal([size, n_class]), name="W")
b = tf.Variable(tf.random_normal([n_class]), name="b")
output = tf.matmul(outputs[-1], W) + b
return output
def cross_entropy_sequence_loss(logits, targets, sequence_length):
"""Calculates the per-example cross-entropy loss for a sequence of logits and
masks out all losses passed the sequence length.
Args:
logits: Logits of shape `[T,B,vocab_size]`
targets: Target classes of shape `[T,B]`
sequence_length: An int32 tensor of shape `[B]` corresponding
to the length of each input
Returns:
A tensor of shape [T,B] that contains the loss per example,per time step.
"""
with tf.name_scope("cross_entropy_sequence_loss"):
losses = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=targets)
# Mask out the losses we don't care about
loss_mask = tf.sequence_mask(
tf.to_int32(sequence_length), tf.to_int32(tf.shape(targets)[0]))
losses = losses * tf.transpose(tf.to_float(loss_mask), 0])
return losses
def _build_graph(self):
config = self.config
config.fast_test = False
eval_config = Config(clone=config)
eval_config.batch_size = 1
initializer = self.model_initializer
with tf.name_scope("Train"):
with tf.variable_scope("Model", reuse=False, initializer=initializer):
self.train_model = self.Model(config=config, is_training=True, loss_fct=self.loss_fct)
tf.summary.scalar("Training Loss", self.train_model.cost)
tf.summary.scalar("Learning Rate", self.train_model.lr)
with tf.name_scope("Valid"):
with tf.variable_scope("Model", reuse=True, initializer=initializer):
self.validation_model = self.Model(config=config, is_training=False, loss_fct="softmax")
tf.summary.scalar("Validation Loss", self.validation_model.cost)
with tf.name_scope("Test"):
with tf.variable_scope("Model", initializer=initializer):
self.test_model = self.Model(config=eval_config, is_training=False)
def get_training_tensors(self, learning_rate = 0.001, grad_clip = 5):
#-----------------------------------------------------------------------
# Build a loss function
#-----------------------------------------------------------------------
with tf.name_scope('targets-encode'):
y_one_hot = tf.one_hot(self.targets, self.n_classes)
y_reshaped = tf.reshape(y_one_hot, self.logits.get_shape())
with tf.name_scope('loss'):
loss = tf.nn.softmax_cross_entropy_with_logits(logits=self.logits,
labels=y_reshaped)
loss = tf.reduce_mean(loss)
tf.summary.scalar('loss', loss)
#-----------------------------------------------------------------------
# Build the optimizer
#-----------------------------------------------------------------------
with tf.name_scope('optimizer'):
tvars = tf.trainable_variables()
grads, _ = tf.clip_by_global_norm(tf.gradients(loss, tvars),
grad_clip)
train_op = tf.train.AdamOptimizer(learning_rate)
optimizer = train_op.apply_gradients(zip(grads, tvars))
return loss, optimizer
def get_optimizer(self, learning_rate = 0.001):
with tf.name_scope('loss'):
input_shape = tf.shape(self.inputs)
ones = tf.ones([input_shape[0], input_shape[1]])
loss = tf.contrib.seq2seq.sequence_loss(self.logits, self.targets,
ones)
#-----------------------------------------------------------------------
# Build the optimizer
#-----------------------------------------------------------------------
with tf.name_scope('optimizer'):
optimizer = tf.train.AdamOptimizer(learning_rate)
gradients = optimizer.compute_gradients(loss)
capped_gradients = [(tf.clip_by_value(grad, -1., 1.), var) \
for grad, var in gradients if grad is not None]
optimizer_op = optimizer.apply_gradients(capped_gradients)
return optimizer_op, loss
def unpack_grad_tuple(gv, gpt):
"""Unpack a prevIoUsly packed collection of gradient tensors.
Args:
gv: A (grad,var) pair to be unpacked.
gpt: A GradPackTuple describing the packing operation that produced gv.
Returns:
A list of (grad,var) pairs corresponding to the values that were
originally packed into gv,maybe following subsequent operations like
reduction.
"""
elt_widths = [x.num_elements() for x in gpt.shapes]
with tf.device(gv[0][0].device):
with tf.name_scope('unpack'):
splits = tf.split(gv[0], elt_widths)
unpacked_gv = []
for idx, s in enumerate(splits):
unpacked_gv.append((tf.reshape(s, gpt.shapes[idx]), gpt.vars[idx]))
return unpacked_gv
def decode_jpeg(image_buffer, scope=None): #,dtype=tf.float32):
"""Decode a JPEG string into one 3-D float image Tensor.
Args:
image_buffer: scalar string Tensor.
scope: Optional scope for op_scope.
Returns:
3-D float Tensor with values ranging from [0,1).
"""
# with tf.op_scope([image_buffer],scope,'decode_jpeg'):
# with tf.name_scope(scope,'decode_jpeg',[image_buffer]):
with tf.name_scope(scope or 'decode_jpeg'):
# Decode the string as an RGB JPEG.
# Note that the resulting image contains an unkNown height and width
# that is set dynamically by decode_jpeg. In other words,the height
# and width of image is unkNown at compile-time.
image = tf.image.decode_jpeg(image_buffer, channels=3,
fancy_upscaling=False,
dct_method='INTEGER_FAST')
# image = tf.Print(image,[tf.shape(image)],'Image shape: ')
return image
def make_png_thumbnail(x, n):
'''
Input:
`x`: Tensor,value range=[-1,1),shape=[n*n,h,w,c]
`n`: sqrt of the number of images
Return:
`tf.string` (bytes) of the PNG.
(write these binary directly into a file)
'''
with tf.name_scope('MakeThumbnail'):
_, h, w, c = x.get_shape().as_list()
x = tf.reshape(x, [n, n, c])
x = tf.transpose(x, [0, 2, 3, 4])
x = tf.reshape(x, [n * h, n * w, c])
x = x / 2. + .5
x = tf.image.convert_image_dtype(x, tf.uint8, saturate=True)
x = tf.image.encode_png(x)
return x
def make_png_jet_thumbnail(x, c])
x = x / 2. + .5
x = gray2jet(x)
x = tf.image.convert_image_dtype(x, saturate=True)
x = tf.image.encode_png(x)
return x
def _optimize(self):
'''
NOTE: The author said that there was no need for 100 d_iter per 100 iters.
https://github.com/igul222/improved_wgan_training/issues/3
'''
global_step = tf.Variable(0, name='global_step')
lr = self.arch['training']['lr']
b1 = self.arch['training']['beta1']
b2 = self.arch['training']['beta2']
optimizer = tf.train.AdamOptimizer(lr, b1, b2)
g_vars = tf.trainable_variables()
with tf.name_scope('Update'):
opt_g = optimizer.minimize(self.loss['G'], var_list=g_vars, global_step=global_step)
return {
'g': opt_g,
'global_step': global_step
}
def _optimize(self):
'''
NOTE: The author said that there was no need for 100 d_iter per 100 iters.
https://github.com/igul222/improved_wgan_training/issues/3
'''
global_step = tf.Variable(0, name='global_step')
lr = self.arch['training']['lr']
b1 = self.arch['training']['beta1']
b2 = self.arch['training']['beta2']
optimizer = tf.train.AdamOptimizer(lr, b2)
trainables = tf.trainable_variables()
g_vars = trainables
# g_vars = [v for v in trainables if 'Generator' in v.name or 'y_emb' in v.name]
with tf.name_scope('Update'):
opt_g = optimizer.minimize(self.loss['G'],
'global_step': global_step
}
def bilateral_slice(grid, guide, name=None):
"""Slices into a bilateral grid using the guide map.
Args:
grid: (Tensor) [batch_size,grid_h,grid_w,depth,n_outputs]
grid to slice from.
guide: (Tensor) [batch_size,w ] guide map to slice along.
name: (string) name for the operation.
Returns:
sliced: (Tensor) [batch_size,n_outputs] sliced output.
"""
with tf.name_scope(name):
gridshape = grid.get_shape().as_list()
if len(gridshape) == 6:
_, _, n_out, n_in = gridshape
grid = tf.concat(tf.unstack(grid, None, axis=5), 4)
sliced = hdrnet_ops.bilateral_slice(grid, guide)
if len(gridshape) == 6:
sliced = tf.stack(tf.split(sliced, n_in, axis=3), axis=4)
return sliced
# pylint: enable=redefined-builtin
def get_label_queue(self,batch_size):
tf_labels = tf.convert_to_tensor(self.attr.values, dtype=tf.uint8)#0,1
with tf.name_scope('label_queue'):
uint_label=tf.train.slice_input_producer([tf_labels])[0]
label=tf.to_float(uint_label)
#All labels,not just those in causal_model
dict_data={sl:tl for sl,tl in
zip(self.label_names,tf.split(label,len(self.label_names)))}
num_preprocess_threads = max(self.num_worker-3,1)
data_batch = tf.train.shuffle_batch(
dict_data,
batch_size=batch_size,
num_threads=num_preprocess_threads,
capacity=self.min_queue_examples + 3 * batch_size,
min_after_dequeue=self.min_queue_examples,
)
return data_batch
def _get_loss(self,labels):
with tf.name_scope("Loss"):
"""
with tf.name_scope("logloss"):
logit = tf.squeeze(tf.nn.sigmoid(self.logit))
self.loss = tf.reduce_mean(self._logloss(labels,logit))
"""
with tf.name_scope("L2_loss"):
if self.flags.lambdax:
lambdax = self.flags.lambdax
else:
lambdax = 0
self.l2loss = lambdax*tf.add_n(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
with tf.name_scope("dice_coef"):
#yp_label = tf.cast(logit>self.flags.threshold,tf.float32)
logit = tf.squeeze(self.logit)
self.acc = tf.reduce_mean(self._dice_coef(labels,logit))
self.metric = "dice_coef"
self.loss = -self.acc
with tf.name_scope("summary"):
if self.flags.visualize:
tf.summary.scalar(name='dice coef', tensor=self.acc, collections=[tf.GraphKeys.SCAlars])
def smoothing_cross_entropy(self,logits, vocab_size, confidence=0.9): #confidence = 1.0 - label_smoothing. where label_smooth=0.1. from http://github.com/tensorflow/tensor2tensor
"""Cross entropy with label smoothing to limit over-confidence."""
with tf.name_scope("smoothing_cross_entropy", [logits, labels]):
# Low confidence is given to all non-true labels,uniformly.
low_confidence = (1.0 - confidence) / tf.to_float(vocab_size - 1)
# normalizing constant is the best cross-entropy value with soft targets.
# We subtract it just for readability,makes no difference on learning.
normalizing = -(confidence * tf.log(confidence) + tf.to_float(vocab_size - 1) * low_confidence * tf.log(low_confidence + 1e-20))
# Soft targets.
soft_targets = tf.one_hot(
tf.cast(labels, tf.int32),
depth=vocab_size,
on_value=confidence,
off_value=low_confidence)
xentropy = tf.nn.softmax_cross_entropy_with_logits(
logits=logits, labels=soft_targets)
return xentropy - normalizing
def smoothing_cross_entropy(self, labels=soft_targets)
return xentropy - normalizing
def loss(self, l2_lambda=0.0001): # 0.001
with tf.name_scope("loss"):
# input: `logits`:[batch_size,num_classes],and `labels`:[batch_size]
# output: A 1-D `Tensor` of length `batch_size` of the same type as `logits` with the softmax cross entropy loss.
losses = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=self.input_y_label,logits=self.logits); # sigmoid_cross_entropy_with_logits.#losses=tf.nn.softmax_cross_entropy_with_logits(labels=self.input_y,logits=self.logits)
# print("1.sparse_softmax_cross_entropy_with_logits.losses:",losses) # shape=(?,)
loss = tf.reduce_mean(losses) # print("2.loss.loss:",loss) #shape=()
l2_losses = tf.add_n([tf.nn.l2_loss(v) for v in tf.trainable_variables() if ('bias' not in v.name ) and ('alpha' not in v.name)]) * l2_lambda
loss = loss + l2_losses
return loss
#def loss_seq2seq(self):
# with tf.variable_scope("loss"):
# losses = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=self.input_y_label,logits=self.logits);#losses:[batch_size,self.decoder_sent_length]
# loss_batch=tf.reduce_sum(losses,axis=1)/self.decoder_sent_length #loss_batch:[batch_size]
# loss=tf.reduce_mean(loss_batch)
# l2_losses = tf.add_n([tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'bias' not in v.name]) * self.l2_lambda
# loss = loss + l2_losses
# return loss
def inference(self):
"""main computation graph here: 1. embeddding layer,3.max pooling,embed_size]
#2. Bi-lstm layer
output_conv=self.conv_layer_with_recurrent_structure() #shape:[None,embed_size*3]
#3. max pooling
#print("output_conv:",output_conv) #(3,8,100)
output_pooling=tf.reduce_max(output_conv,axis=1) #shape:[None,embed_size*3]
#print("output_pooling:",output_pooling) #(3,100)
#4. logits(use linear layer)
with tf.name_scope("dropout"):
h_drop=tf.nn.dropout(output_pooling,keep_prob=self.dropout_keep_prob) #[None,num_filters_total]
with tf.name_scope("output"): #inputs: A `Tensor` of shape `[batch_size,dim]`. The forward activations of the input network.
logits = tf.matmul(h_drop,num_classes]
return logits
def instantiate_weights(self):
"""define all weights here"""
with tf.name_scope("weights"): # embedding matrix
self.Embedding = tf.get_variable("Embedding",shape=[self.vocab_size, self.embed_size],initializer=self.initializer) #[vocab_size,embed_size] tf.random_uniform([self.vocab_size,self.embed_size],-1.0,1.0)
self.left_side_first_word= tf.get_variable("left_side_first_word",shape=[self.batch_size,initializer=self.initializer) #Todo removed. replaced with zero vector
self.right_side_last_word = tf.get_variable("right_side_last_word",initializer=self.initializer) #Todo removed. replaced with zero vector
#self.left_side_context_first= tf.get_variable("left_side_context_first",shape=[self.batch_size,initializer=self.initializer) #Todo removed. replaced with zero vector
#self.right_side_context_last=tf.get_variable("right_side_context_last",initializer=self.initializer) #Todo removed. replaced with zero vector
self.W_l=tf.get_variable("W_l",shape=[self.embed_size,initializer=self.initializer)
self.W_r=tf.get_variable("W_r",initializer=self.initializer)
self.W_sl=tf.get_variable("W_sl",initializer=self.initializer)
self.W_sr=tf.get_variable("W_sr",initializer=self.initializer)
self.b = tf.get_variable("b", [self.embed_size])
self.W_projection = tf.get_variable("W_projection",shape=[self.hidden_size*3, self.num_classes],initializer=self.initializer) #[embed_size,label_size]
self.b_projection = tf.get_variable("b_projection",shape=[self.num_classes]) #[label_size]
#b = tf.get_variable("b",[self.embed_size*3])
#h = tf.nn.relu(tf.nn.bias_add(output_conv,b),"relu")
def inference(self):
"""main computation graph here: 1. embeddding layer,embed_size*3]
#2.1 apply nolinearity
#b = tf.get_variable("b","relu")
#3. max pooling
output_pooling=tf.reduce_max(output_conv,embed_size*3]
#4. logits(use linear layer)
with tf.name_scope("dropout"):
h_drop=tf.nn.dropout(output_pooling,embed_size*3]
with tf.name_scope("output"): #inputs: A `Tensor` of shape `[batch_size, self.W_projection) + self.b_projection #shape:[batch_size,num_classes]<-----h_drop:[None,embed_size*3];b_projection:[hidden_size*3,self.num_classes]
return logits
def inference2(self):
"""main computation graph here: 1. embeddding layer,100)
#4. logits(use linear layer)
with tf.name_scope("dropout_rcnn"):
h_drop=tf.nn.dropout(output_pooling,embed_size*3]
#with tf.name_scope("output"): #inputs: A `Tensor` of shape `[batch_size, self.W_projection_rcnn) + self.b_projection_rcnn # [batch_size,num_classes]
return logits
def inference(self):
"""main computation graph here: 1. embeddding layers,2.convolutional layer,3.max-pooling,4.softmax layer."""
# 1.=====>get emebedding of words in the sentence
self.embedded_words1 = tf.nn.embedding_lookup(self.Embedding,self.input_x)#[None,embed_size]
self.sentence_embeddings_expanded1=tf.expand_dims(self.embedded_words1,-1) #[None,embed_size,1). expand dimension so meet input requirement of 2d-conv
self.embedded_words2 = tf.nn.embedding_lookup(self.Embedding,self.input_x2)#[None,embed_size]
self.sentence_embeddings_expanded2=tf.expand_dims(self.embedded_words2,1). expand dimension so meet input requirement of 2d-conv
#2.1 get features of sentence1
h1=self.conv_relu_pool_dropout(self.sentence_embeddings_expanded1,name_scope_prefix="s1") #[None,num_filters_total]
#2.2 get features of sentence2
h2 =self.conv_relu_pool_dropout(self.sentence_embeddings_expanded2,name_scope_prefix="s2") # [None,num_filters_total]
#3. concat features
h=tf.concat([h1,h2],axis=1) #[None,num_filters_total*2]
#4. logits(use linear layer)and predictions(argmax)
with tf.name_scope("output"):
logits = tf.matmul(h,self.W_projection) + self.b_projection #shape:[None,self.num_classes]==tf.matmul([None,self.num_filters_total*2],[self.num_filters_total*2,self.num_classes])
return logits
def preprocess_for_eval(image, height, width,
central_fraction=0.875, scope=None):
"""Prepare one image for evaluation.
If height and width are specified it would output an image with that size by
applying resize_bilinear.
If central_fraction is specified it would cropt the central fraction of the
input image.
Args:
image: 3-D Tensor of image. If dtype is tf.float32 then the range should be
[0,1],otherwise it would converted to tf.float32 assuming that the range
is [0,MAX],where MAX is largest positive representable number for
int(8/16/32) data type (see `tf.image.convert_image_dtype` for details)
height: integer
width: integer
central_fraction: Optional Float,fraction of the image to crop.
scope: Optional scope for name_scope.
Returns:
3-D float Tensor of prepared image.
"""
with tf.name_scope(scope, 'eval_image', [image, width]):
if image.dtype != tf.float32:
image = tf.image.convert_image_dtype(image, dtype=tf.float32)
# Crop the central region of the image with an area containing 87.5% of
# the original image.
if central_fraction:
image = tf.image.central_crop(image, central_fraction=central_fraction)
if height and width:
# Resize the image to the specified height and width.
image = tf.expand_dims(image, 0)
image = tf.image.resize_bilinear(image, [height, width],
align_corners=False)
image = tf.squeeze(image, [0])
image = tf.subtract(image, 0.5)
image = tf.multiply(image, 2.0)
return image
def constrain_value_logits(self, logits, curr_state):
first_value_token = self.num_functions + self.num_begin_tokens + self.num_control_tokens
num_value_tokens = self.output_size - first_value_token
value_allowed_token_matrix = np.concatenate((self.allowed_token_matrix[:,:self.num_control_tokens], self.allowed_token_matrix[:,first_value_token:]), axis=1)
with tf.name_scope('constrain_logits'):
allowed_tokens = tf.gather(tf.constant(value_allowed_token_matrix), curr_state)
assert allowed_tokens.get_shape()[1:] == (self.num_control_tokens + num_value_tokens,)
constrained_logits = logits - tf.to_float(tf.logical_not(allowed_tokens)) * 1e+10
return constrained_logits
def constrain_logits(self, curr_state):
with tf.name_scope('constrain_logits'):
allowed_tokens = tf.gather(tf.constant(self.allowed_token_matrix), curr_state)
assert allowed_tokens.get_shape()[1:] == (self.output_size,)
constrained_logits = tf.where(allowed_tokens, tf.fill(tf.shape(allowed_tokens), -1e+10))
return constrained_logits
版权声明:本文内容由互联网用户自发贡献,该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容, 请发送邮件至 dio@foxmail.com 举报,一经查实,本站将立刻删除。