Python tensorflow 模块,constant() 实例源码
我们从Python开源项目中,提取了以下50个代码示例,用于说明如何使用tensorflow.constant()。
def get_weight_variable(shape, name=None, type='xavier_uniform', regularize=True, **kwargs):
initialise_from_constant = False
if type == 'xavier_uniform':
initial = xavier_initializer(uniform=True, dtype=tf.float32)
elif type == 'xavier_normal':
initial = xavier_initializer(uniform=False, dtype=tf.float32)
elif type == 'he_normal':
initial = variance_scaling_initializer(uniform=False, factor=2.0, mode='FAN_IN', dtype=tf.float32)
elif type == 'he_uniform':
initial = variance_scaling_initializer(uniform=True, dtype=tf.float32)
elif type == 'caffe_uniform':
initial = variance_scaling_initializer(uniform=True, factor=1.0, dtype=tf.float32)
elif type == 'simple':
stddev = kwargs.get('stddev', 0.02)
initial = tf.truncated_normal(shape, stddev=stddev, dtype=tf.float32)
initialise_from_constant = True
elif type == 'bilinear':
weights = _bilinear_upsample_weights(shape)
initial = tf.constant(weights, shape=shape, dtype=tf.float32)
initialise_from_constant = True
else:
raise ValueError('UnkNown initialisation requested: %s' % type)
if name is None: # This keeps to option open to use unnamed Variables
weight = tf.Variable(initial)
else:
if initialise_from_constant:
weight = tf.get_variable(name, initializer=initial)
else:
weight = tf.get_variable(name, initializer=initial)
if regularize:
tf.add_to_collection('weight_variables', weight)
return weight
def calculate_loss_mix2(self, predictions, predictions_class, predictions_encoder, labels, **unused_params):
with tf.name_scope("loss_mix2"):
float_labels = tf.cast(labels, tf.float32)
float_encoders = float_labels
for i in range(FLAGS.encoder_layers):
var_i = np.loadtxt(FLAGS.autoencoder_dir+'autoencoder_layer%d.model' % i)
weight_i = tf.constant(var_i[:-1,:],dtype=tf.float32)
bias_i = tf.reshape(tf.constant(var_i[-1,dtype=tf.float32),[-1])
float_encoders = tf.nn.xw_plus_b(float_encoders,weight_i,bias_i)
if i<FLAGS.encoder_layers-1:
float_encoders = tf.nn.relu(float_encoders)
else:
hidden_mean = tf.reduce_mean(float_encoders,axis=1,keep_dims=True)
hidden_std = tf.sqrt(tf.reduce_mean(tf.square(float_encoders-hidden_mean),keep_dims=True))
float_encoders = (float_encoders-hidden_mean)/(hidden_std+1e-6)
#float_encoders = tf.nn.sigmoid(float_encoders)
cross_entropy_encoder = 0.1*self.calculate_mseloss(predictions_encoder,float_encoders)
cross_entropy_loss = self.calculate_loss(predictions,labels)
return cross_entropy_encoder+cross_entropy_loss, float_encoders
#return cross_entropy_encoder,float_encoders
def _conv_layer(self, bottom, filter_size, filter_num, scope_name, bottom_channel=None, padding='SAME'):
if not bottom_channel:
_, _, bottom_channel = bottom.get_shape().as_list()
with tf.variable_scope(scope_name):
kernel = tf.Variable(
tf.truncated_normal([*filter_size, bottom_channel, filter_num], dtype=tf.float32, stddev=1e-1),
trainable=False,
name='weights'
)
conv = tf.nn.conv2d(bottom, kernel, [1, 1, 1], padding=padding)
biases = tf.Variable(
tf.constant(0.0, shape=[filter_num], dtype=tf.float32),
trainable=True,
name='bias'
)
out = tf.nn.bias_add(conv, biases)
return out
def omniglot():
sess = tf.InteractiveSession()
""" def wrapper(v):
return tf.Print(v,[v],message="Printing v")
v = tf.Variable(initial_value=np.arange(0,36).reshape((6,6)),dtype=tf.float32,name='Matrix')
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
temp = tf.Variable(initial_value=np.arange(0,name='temp')
temp = wrapper(v)
#with tf.control_dependencies([temp]):
temp.eval()
print 'Hello'"""
def update_tensor(V, dim2, val): # Update tensor V,with index(:,dim2[:]) by val[:]
val = tf.cast(val, V.dtype)
def body(_, (v, d2, chg)):
d2_int = tf.cast(d2, tf.int32)
return tf.slice(tf.concat_v2([v[:d2_int],[chg] ,v[d2_int+1:]], axis=0), [0], [v.get_shape().as_list()[0]])
Z = tf.scan(body, elems=(V, val), initializer=tf.constant(1, shape=V.get_shape().as_list()[1:], name="Scan_Update")
return Z
def bag_of_tokens(config, label_lengths):
if config.train_output_embeddings:
with tf.variable_scope('embed', reuse=True):
output_embeddings = tf.get_variable('output_embedding')
else:
output_embeddings = tf.constant(config.output_embedding_matrix)
#everything_label_placeholder = tf.placeholder(shape=(None,config.max_length,),dtype=tf.int32)
#everything_label_length_placeholder = tf.placeholder(shape=(None,dtype=tf.int32)
labels = tf.constant(np.array(labels))
embedded_output = tf.gather(output_embeddings, labels)
print('embedded_output before', embedded_output)
#mask = tf.sequence_mask(label_lengths,maxlen=config.max_length,dtype=tf.float32)
# note: this multiplication will broadcast the mask along all elements of the depth dimension
# (which is why we run the expand_dims to choose how to broadcast)
#embedded_output = embedded_output * tf.expand_dims(mask,axis=2)
#print('embedded_output after',embedded_output)
return tf.reduce_sum(embedded_output, axis=1)
def highway(self, input_1, input_2, size_1, size_2, l2_penalty=1e-8, layer_size=1):
output = input_2
for idx in range(layer_size):
with tf.name_scope('output_lin_%d' % idx):
W = tf.Variable(tf.truncated_normal([size_2,size_1], stddev=0.1), name="W")
b = tf.Variable(tf.constant(0.1, shape=[size_1]), name="b")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(W))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(b))
output = tf.nn.relu(tf.nn.xw_plus_b(output,W,b))
with tf.name_scope('transform_lin_%d' % idx):
W = tf.Variable(tf.truncated_normal([size_1, value=l2_penalty*tf.nn.l2_loss(b))
transform_gate = tf.sigmoid(tf.nn.xw_plus_b(input_1,b))
carry_gate = tf.constant(1.0) - transform_gate
output = transform_gate * output + carry_gate * input_1
return output
def conv_block(self, input, out_size, layer, kernalsize=3, shortcut=False):
in_shape = input.get_shape().as_list()
if layer>0:
filter_shape = [kernalsize, in_shape[3], out_size]
else:
filter_shape = [kernalsize, in_shape[2], out_size]
W = tf.Variable(tf.truncated_normal(filter_shape, name="W-%s" % layer)
b = tf.Variable(tf.constant(0.1, shape=[out_size]), name="b-%s" % layer)
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(W))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(b))
if layer>0:
conv = tf.nn.conv2d(input, W, strides=[1, padding="SAME", name="conv-%s" % layer)
else:
conv = tf.nn.conv2d(input, padding="VALID", name="conv-%s" % layer)
if shortcut:
shortshape = [1,1,in_shape[3], out_size]
Ws = tf.Variable(tf.truncated_normal(shortshape, stddev=0.05), name="Ws-%s" % layer)
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Ws))
conv = conv + tf.nn.conv2d(input, Ws, name="conv-shortcut-%s" % layer)
h = tf.nn.bias_add(conv, b)
h2 = tf.nn.relu(tf.contrib.layers.batch_norm(h, center=True, scale=True, epsilon=1e-5, decay=0.9), name="relu-%s" % layer)
return h2
def test_without_residuals(self):
inputs = tf.constant(np.random.randn(1, 2))
state = (tf.constant(np.random.randn(1, 2)),
tf.constant(np.random.randn(1, 2)))
with tf.variable_scope("root", initializer=tf.constant_initializer(0.5)):
standard_cell = tf.contrib.rnn.MultiRNNCell(
[tf.contrib.rnn.GRUCell(2) for _ in range(2)], state_is_tuple=True)
res_standard = standard_cell(inputs, state, scope="standard")
test_cell = rnn_cell.ExtendedMultiRNNCell(
[tf.contrib.rnn.GRUCell(2) for _ in range(2)])
res_test = test_cell(inputs, scope="test")
with self.test_session() as sess:
sess.run([tf.global_variables_initializer()])
res_standard_, res_test_, = sess.run([res_standard, res_test])
# Make sure it produces the same results as the standard cell
self.assertAllClose(res_standard_[0], res_test_[0])
self.assertAllClose(res_standard_[1][0], res_test_[1][0])
self.assertAllClose(res_standard_[1][1], res_test_[1][1])
def _test_with_residuals(self, inputs, **kwargs):
"""Runs the cell in a session"""
inputs = tf.convert_to_tensor(inputs)
state = (tf.constant(np.random.randn(1, initializer=tf.constant_initializer(0.5)):
test_cell = rnn_cell.ExtendedMultiRNNCell(
[tf.contrib.rnn.GRUCell(2) for _ in range(2)],
residual_connections=True,
**kwargs)
res_test = test_cell(inputs, scope="test")
with self.test_session() as sess:
sess.run([tf.global_variables_initializer()])
return sess.run(res_test)
def test_eos_masking(self):
probs = tf.constant([[-.2, -.2, -.2], [-.3, -.3, 3, 0],
[5, 6, 0, 0]])
eos_token = 0
prevIoUsly_finished = tf.constant([0, dtype=tf.float32)
masked = beam_search.mask_probs(probs, eos_token, prevIoUsly_finished)
with self.test_session() as sess:
probs = sess.run(probs)
masked = sess.run(masked)
np.testing.assert_array_equal(probs[0], masked[0])
np.testing.assert_array_equal(probs[2], masked[2])
np.testing.assert_equal(masked[1][0], 0)
np.testing.assert_approx_equal(masked[1][1], np.finfo('float32').min)
np.testing.assert_approx_equal(masked[1][2], np.finfo('float32').min)
np.testing.assert_approx_equal(masked[1][3], np.finfo('float32').min)
np.testing.assert_approx_equal(masked[1][4], np.finfo('float32').min)
def get_loss(pred, label, end_points, reg_weight=0.001):
""" pred: B*NUM_CLASSES,
label: B,"""
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=pred, labels=label)
classify_loss = tf.reduce_mean(loss)
tf.summary.scalar('classify loss', classify_loss)
# Enforce the transformation as orthogonal matrix
transform = end_points['transform'] # BxKxK
K = transform.get_shape()[1].value
mat_diff = tf.matmul(transform, tf.transpose(transform, perm=[0,2,1]))
mat_diff -= tf.constant(np.eye(K), dtype=tf.float32)
mat_diff_loss = tf.nn.l2_loss(mat_diff)
tf.summary.scalar('mat loss', mat_diff_loss)
return classify_loss + mat_diff_loss * reg_weight
def get_loss(pred, reg_weight=0.001):
""" pred: BxNxC,
label: BxN, labels=label)
classify_loss = tf.reduce_mean(loss)
tf.scalar_summary('classify loss', dtype=tf.float32)
mat_diff_loss = tf.nn.l2_loss(mat_diff)
tf.scalar_summary('mat_loss', mat_diff_loss)
return classify_loss + mat_diff_loss * reg_weight
def get_loss(l_pred, seg_pred, seg, weight, end_points):
per_instance_label_loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=l_pred, labels=label)
label_loss = tf.reduce_mean(per_instance_label_loss)
# size of seg_pred is batch_size x point_num x part_cat_num
# size of seg is batch_size x point_num
per_instance_seg_loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=seg_pred, labels=seg), axis=1)
seg_loss = tf.reduce_mean(per_instance_seg_loss)
per_instance_seg_pred_res = tf.argmax(seg_pred, 2)
# Enforce the transformation as orthogonal matrix
transform = end_points['transform'] # BxKxK
K = transform.get_shape()[1].value
mat_diff = tf.matmul(transform,1])) - tf.constant(np.eye(K), dtype=tf.float32)
mat_diff_loss = tf.nn.l2_loss(mat_diff)
total_loss = weight * seg_loss + (1 - weight) * label_loss + mat_diff_loss * 1e-3
return total_loss, label_loss, per_instance_label_loss, seg_loss, per_instance_seg_loss, per_instance_seg_pred_res
def mnist_batcher_in_tanh_vector(
batch_size,
capacity=256,
min_after_dequeue=128,
):
(x, y), (_, _) = keras.datasets.mnist.load_data()
x = tf.constant(x)
x = tf.cast(x, tf.float32)
x = keras.layers.Flatten()(x) / 127.5 - 1.
y = tf.cast(y, tf.int64)
return tf.train.shuffle_batch(
[x, y],
batch_size=batch_size,
capacity=capacity,
min_after_dequeue=min_after_dequeue,
enqueue_many=True
)
def _validate(self, machine, n=10):
N = n * n
z = np.random.normal(0., 1., size=[n, self.arch['z_dim']])
z = np.concatenate([z] * n, axis=1)
z = np.reshape(z, [N, -1]).astype(np.float32) # consecutive rows
y = np.asarray(
[[5, 0, 0 ],
[9,
[12, 0,
[17,
[19,
[161,
[170, 16, 0 ], 9, 4 ], 24, 50]],
dtype=np.int64)
y = np.concatenate([y] * n, axis=0)
Z = tf.constant(z)
Y = tf.constant(y)
Xh = machine.generate(Z, Y) # 100,64,3
Xh = make_png_thumbnail(Xh, n)
return Xh
def batch_norm_layer(self, to_be_normalized, is_training):
if is_training:
train_phase = tf.constant(1)
else:
train_phase = tf.constant(-1)
beta = tf.Variable(tf.constant(0.0, shape=[to_be_normalized.shape[-1]]), name='beta', trainable=True)
gamma = tf.Variable(tf.constant(1.0, name='gamma', trainable=True)
# axises = np.arange(len(to_be_normalized.shape) - 1) # change to apply tensorflow 1.3
axises = [0,2]
print("start nn.moments")
print("axises : " + str(axises))
batch_mean, batch_var = tf.nn.moments(to_be_normalized, axises, name='moments')
print("nn.moments successful")
ema = tf.train.ExponentialMovingAverage(decay=0.5)
def mean_var_with_update():
ema_apply_op = ema.apply([batch_mean, batch_var])
with tf.control_dependencies([ema_apply_op]):
return tf.identity(batch_mean), tf.identity(batch_var)
mean, var = tf.cond(train_phase > 0, mean_var_with_update, lambda: (ema.average(batch_mean), ema.average(batch_var))) # if is training --> update
normed = tf.nn.batch_normalization(to_be_normalized, mean, var, beta, gamma, 1e-3)
return normed
def repeat(tensor: tf.Tensor, repeats: int, axis: int) -> tf.Tensor:
"""
Repeat elements of the input tensor in the specified axis ``repeats``-times.
.. note::
Chaining of this op may produce TF warnings although the performance seems to be unaffected.
:param tensor: TF tensor to be repeated
:param repeats: number of repeats
:param axis: axis to repeat
:return: tensor with repeated elements
"""
shape = tensor.get_shape().as_list()
dims = np.arange(len(tensor.shape))
prepare_perm = np.hstack(([axis], np.delete(dims, axis)))
restore_perm = np.hstack((dims[1:axis+1], dims[axis+1:]))
indices = tf.cast(tf.floor(tf.range(0, shape[axis]*repeats)/tf.constant(repeats)), 'int32')
shuffled = tf.transpose(tensor, prepare_perm)
repeated = tf.gather(shuffled, indices)
return tf.transpose(repeated, restore_perm)
def test_dense_to_sparse(self):
""" Test if `dense_to_sparse` works properly."""
with tf.Session().as_default():
dense = tf.constant([[1., 2., 0.], [0., 0., 3.]], dtype=tf.float32)
sparse = dense_to_sparse(dense)
self.assertTrue(np.array_equal(sparse.indices.eval(), np.array([[0, [0, 2]])))
self.assertTrue(np.array_equal(sparse.values.eval(), np.array([1., 3.])))
mask = tf.constant([[0, 0]], dtype=tf.int32)
masked = dense_to_sparse(dense, mask)
self.assertTrue(np.array_equal(masked.indices.eval(), 0]])))
self.assertTrue(np.array_equal(masked.values.eval(), np.array([2., 0.])))
def test_repeat(self):
""" Test if `repeat` works the same as np.repeat."""
with tf.Session().as_default():
# try different tensor types
for npdtype, tfdtype in [(np.int32, tf.int32), (np.float32, tf.float32)]:
for init_value in [np.array([0, 2, 3], dtype=npdtype),
np.array([[0, [2, [4, 5]], dtype=npdtype)]:
# and all their axes
for axis in range(len(init_value.shape)):
for repeats in [1, 11]:
tensor = tf.constant(init_value, dtype=tfdtype)
repeated_value = repeat(tensor, repeats=repeats, axis=axis).eval()
expected_value = np.repeat(init_value, axis=axis)
self.assertTrue(np.all(repeated_value == expected_value))
def __init__(self, tag, x, summary_fn=tf.summary.scalar, summary_args=(), scope=None):
"""
Initializes an Average.
Arguments
x: Tensor to be averaged over multiple runs.
tag: Tag for the summary.
summary_fn: Function used for creating a summary.
summary_args: Arguments passed to the summary function.
"""
with tf.variable_scope(scope or type(self).__name__):
counter = tf.Variable(name="counter", initial_value=tf.constant(0),
dtype=tf.int32, trainable=False)
running_sum = tf.Variable(name="running_sum", initial_value=tf.constant(0.),
dtype=tf.float32, trainable=False)
self._running_average = running_sum / tf.cast(counter, tf.float32)
self._summary = summary_fn(tag or x.name + '_avg', self._running_average, **summary_args)
self._update_op = tf.group(counter.assign_add(1), running_sum.assign_add(x))
self._reset_op = tf.group(counter.assign(0), running_sum.assign(0.))
def simulate_dynamics(initial_pos, initial_vel, stepsize, n_steps, energy_fn):
def leapfrog(pos, vel, step, i):
de_dp_ = tf.gradients(tf.reduce_sum(energy_fn(pos)), pos)[0]
new_vel_ = vel - step * de_dp_
new_pos_ = pos + step * new_vel_
return [new_pos_, new_vel_, tf.add(i, 1)]
def condition(pos, i):
return tf.less(i, n_steps)
de_dp = tf.gradients(tf.reduce_sum(energy_fn(initial_pos)), initial_pos)[0]
vel_half_step = initial_vel - 0.5 * stepsize * de_dp
pos_full_step = initial_pos + stepsize * vel_half_step
i = tf.constant(0)
final_pos, new_vel, _ = tf.while_loop(condition, leapfrog, [pos_full_step, vel_half_step, i])
de_dp = tf.gradients(tf.reduce_sum(energy_fn(final_pos)), final_pos)[0]
final_vel = new_vel - 0.5 * stepsize * de_dp
return final_pos, final_vel
def smoothing_cross_entropy(self,logits, vocab_size, confidence=0.9): #confidence = 1.0 - label_smoothing. where label_smooth=0.1. from http://github.com/tensorflow/tensor2tensor
"""Cross entropy with label smoothing to limit over-confidence."""
with tf.name_scope("smoothing_cross_entropy", [logits, labels]):
# Low confidence is given to all non-true labels,uniformly.
low_confidence = (1.0 - confidence) / tf.to_float(vocab_size - 1)
# normalizing constant is the best cross-entropy value with soft targets.
# We subtract it just for readability,makes no difference on learning.
normalizing = -(confidence * tf.log(confidence) + tf.to_float(vocab_size - 1) * low_confidence * tf.log(low_confidence + 1e-20))
# Soft targets.
soft_targets = tf.one_hot(
tf.cast(labels,
depth=vocab_size,
on_value=confidence,
off_value=low_confidence)
xentropy = tf.nn.softmax_cross_entropy_with_logits(
logits=logits, labels=soft_targets)
return xentropy - normalizing
def smoothing_cross_entropy(self, labels=soft_targets)
return xentropy - normalizing
def apply_gradients(self, grads_and_vars, global_step=None):
"""Apply gradients to model variables specified in `grads_and_vars`.
`apply_gradients` returns an op that calls
`tf.train.Optimizer.apply_gradients` and then zeros the gradient
variables stored in `self.grads_and_vars`.
Args:
grads_and_vars (list): Description.
global_step (None,optional): tensorflow global_step variable.
Returns:
(tf.Operation): Applies gradient update to model followed by an
internal gradient zeroing operation to `self.grads_and_vars`.
"""
self.mini_flag = tf.assign(self.mini_flag, tf.constant([0], dtype = tf.float32))
# grads_and_vars = self.aggregate_gradients(grads_and_vars,method='average')
with tf.control_dependencies([self.mini_flag]):
optimize = self._optimizer.apply_gradients(grads_and_vars,
global_step=global_step)
#return [optimize,self.zero_grad()]
return optimize
def init_var(self):
trainable = self.trainable
with tf.variable_scope(self.scope):
if self.init_beta is None:
self.init_beta = tf.constant(0.0, shape=[self.n_out])
if self.init_gamma is None:
self.init_gamma = tf.constant(1.0, shape=[self.n_out])
self.beta = self.declare_var(
[self.n_out], init_val=self.init_beta,
trainable=trainable)
self.gamma = self.declare_var(
[self.n_out], init_val=self.init_gamma,
trainable=trainable)
self.ema = tf.train.ExponentialMovingAverage(decay=self.decay)
self.batch_mean = None
self.batch_var = None
self.ema_apply_op = None
pass
pass
def test(self):
def decode_greedily(beam_search: bool, merge_repeated: bool):
aa_ctc_blank_aa_logits = tf.constant(np.array([[[1.0, 0.0]], [[1.0, [[0.0, 1.0]],
[[1.0, 0.0]]], dtype=np.float32))
sequence_length = tf.constant(np.array([5], dtype=np.int32))
(decoded_list, log_probabilities = \
tf.nn.ctc_beam_search_decoder(inputs=aa_ctc_blank_aa_logits,
sequence_length=sequence_length,
merge_repeated=merge_repeated,
beam_width=1) \
if beam_search else \
tf.nn.ctc_greedy_decoder(inputs=aa_ctc_blank_aa_logits,
sequence_length=sequence_length,
merge_repeated=merge_repeated)
return list(tf.Session().run(tf.sparse_tensor_to_dense(decoded_list)[0]))
self.assertEqual([0], decode_greedily(beam_search=True, merge_repeated=True))
self.assertEqual([0, merge_repeated=False))
self.assertEqual([0, decode_greedily(beam_search=False, merge_repeated=False))
def bag_hinge_loss(config, preds, sent_mask, flip_sent_mask, hete_mask,
sent_trgt, sent_num):
""" HINGE LOSS:
DEFINED AS: MAX(0,M - MIN(SENT+) - MAX(SENT-))
THIS ONLY APPLIES TO HETE BAGS.
"""
flip_sent_trgt = \
tf.constant(1, shape=[config.batch_size,sent_num], dtype=config.data_type) - \
sent_trgt
pos_preds = preds + flip_sent_trgt + flip_sent_mask # [batch_size,sent_num]
neg_preds = preds * flip_sent_trgt * sent_mask # [batch_size,sent_num]
min_pos_pred = tf.reduce_min(pos_preds, 1)
# min_pos_pred = tf.Print(min_pos_pred,[min_pos_pred],message='min_pos_pred')
max_neg_pred = tf.reduce_max(neg_preds, 1)
# max_neg_pred = tf.Print(max_neg_pred,[max_neg_pred],message='max_neg_pred')
hinge_loss = hete_mask * tf.reduce_max(tf.pack(
[tf.constant(0, shape=[config.batch_size], dtype=config.data_type),
(0.20 - min_pos_pred + max_neg_pred)], axis=1), 1) # [batch_size]
# hinge_loss = tf.Print(hinge_loss,[hinge_loss],message='hinge_loss',summarize=20)
avg_hinge_loss = tf.reduce_sum(hinge_loss) / (tf.reduce_sum(hete_mask) + 1e-12)
return avg_hinge_loss
def loss(self, img_batch, label_batch):
"""Create the network,run inference on the input batch and compute loss.
Args:
input_batch: batch of pre-processed images.
Returns:
Pixel-wise softmax loss.
"""
raw_output = self._create_network(tf.cast(img_batch, tf.float32), keep_prob=tf.constant(0.5))
prediction = tf.reshape(raw_output, [-1, n_classes])
# Need to resize labels and convert using one-hot encoding.
label_batch = self.prepare_label(label_batch, tf.stack(raw_output.get_shape()[1:3]))
gt = tf.reshape(label_batch, n_classes])
# Pixel-wise softmax loss.
loss = tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=gt)
reduced_loss = tf.reduce_mean(loss)
return reduced_loss
def layer_norm_all(h, base, num_units, scope):
# Layer norm (faster version)
#
# Performs layer norm on multiple base at once (ie,i,g,j,o for lstm)
#
# Reshapes h in to perform layer norm in parallel
with tf.variable_scope(scope):
h_reshape = tf.reshape(h, num_units])
mean = tf.reduce_mean(h_reshape, [2], keep_dims=True)
var = tf.reduce_mean(tf.square(h_reshape - mean), keep_dims=True)
epsilon = tf.constant(1e-3)
rstd = tf.rsqrt(var + epsilon)
h_reshape = (h_reshape - mean) * rstd
# reshape back to original
h = tf.reshape(h_reshape, base * num_units])
alpha = tf.get_variable('layer_norm_alpha', [4 * num_units],
initializer=tf.constant_initializer(1.0), dtype=tf.float32)
bias = tf.get_variable('layer_norm_bias',
initializer=tf.constant_initializer(0.0), dtype=tf.float32)
return (h * alpha) + bias
def omniglot():
sess = tf.InteractiveSession()
""" def wrapper(v):
return tf.Print(v, name="Scan_Update")
return Z
def random_access_problem(which=1):
import raputil as ru
if which == 1:
opts = ru.Problem.scenario1()
else:
opts = ru.Problem.scenario2()
p = ru.Problem(**opts)
x1 = p.genX(1)
y1 = p.fwd(x1)
A = p.S
M,N = A.shape
nbatches = int(math.ceil(1000 /x1.shape[1]))
prob = NumpyGenerator(p=p,nbatches=nbatches,A=A,opts=opts,iid=(which==1))
if which==2:
prob.maskX_ = tf.expand_dims( tf.constant( (np.arange(N) % (N//2) < opts['Nu']).astype(np.float32) ) , 1)
_,prob.noise_var = p.add_noise(y1)
unused = p.genYX(nbatches) # for legacy reasons -- want to compare against a prevIoUs run
(prob.yval, prob.xval) = p.genYX(nbatches)
(prob.yinit, prob.xinit) = p.genYX(nbatches)
import multiprocessing as mp
prob.nsubprocs = mp.cpu_count()
return prob
def pwlin_grid(r_,rvar_,theta_,dtheta = .75):
"""piecewise linear with noise-adaptive grid spacing.
returns xhat,dxdr
where
q = r/dtheta/sqrt(rvar)
xhat = r * interp(q,theta)
all but the last dimensions of theta must broadcast to r_
e.g. r.shape = (500,1000) is compatible with theta.shape=(500,1,7)
"""
ntheta = int(theta_.get_shape()[-1])
scale_ = dtheta / tf.sqrt(rvar_)
ars_ = tf.clip_by_value( tf.expand_dims( tf.abs(r_)*scale_,-1),0.0, ntheta-1.0 )
centers_ = tf.constant( np.arange(ntheta),dtype=tf.float32 )
outer_distance_ = tf.maximum(0., 1.0-tf.abs(ars_ - centers_) ) # new dimension for distance to closest bin centers (or center)
gain_ = tf.reduce_sum( theta_ * outer_distance_,axis=-1) # apply the gain (learnable)
xhat_ = gain_ * r_
dxdr_ = tf.gradients(xhat_,r_)[0]
return (xhat_,dxdr_)
def interp1d_(xin_,xp,yp_):
"""
Interpolate a uniformly sampled piecewise linear function. Mapping elements
from xin_ to the result. Input values will be clipped to range of xp.
xin_ : input tensor (real)
xp : x grid (constant -- must be a 1d numpy array,uniformly spaced)
yp_ : tensor of the result values at the gridpoints xp
"""
import tensorflow as tf
x_ = tf.clip_by_value(xin_,xp.min(),xp.max())
dx = xp[1]-xp[0]
assert len(xp.shape)==1,'only 1d interpolation'
assert xp.shape[0]==int(yp_.get_shape()[0])
assert abs(np.diff(xp)/dx - 1.0).max() < 1e-6,'must be uniformly sampled'
newshape = [ ]
x1_ = tf.expand_dims(x_,-1)
dt = yp_.dtype
wt_ = tf.maximum(tf.constant(0.,dtype=dt), 1-abs(x1_ - tf.constant(xp,dtype=dt))/dx )
y_ = tf.reduce_sum(wt_ * yp_,axis=-1)
return y_
def __init__(self, config):
self.layers = {}
self.weights = {}
self.biases = {}
self.losses = {}
self.regular_losses = {}
self.trainable = {}
self.summaries = {}
# set parameters
self.lr_rates = {}
for key, val in config.lr_rates.iteritems():
self.lr_rates[key] = tf.get_variable('lr_rates/'+key, initializer=tf.constant(val), dtype=tf.float32)
self.momentum = tf.get_variable('momentum', initializer=tf.constant(config.momentum), dtype=tf.float32)
self.weight_decay = tf.get_variable('weight_decay', initializer=tf.constant(config.weight_decay), dtype=tf.float32)
self.lr_rate = tf.get_variable('lr_rate', initializer=tf.constant(config.lr_rate), dtype=tf.float32)
def _embed_sentences(self):
"""Tensorflow implementation of Simple but Tough-to-Beat Baseline"""
# Get word features
word_embeddings = self._get_embedding()
word_feats = tf.nn.embedding_lookup(word_embeddings, self.input)
# Get marginal estimates and scaling term
batch_size = tf.shape(word_feats)[0]
a = tf.pow(10.0, self._get_a_exp())
p = tf.constant(self.marginals, name='marginals')
q = tf.reshape(
a / (a + tf.nn.embedding_lookup(p, self.input)),
(batch_size, self.mx_len, 1)
)
# Compute initial sentence embedding
z = tf.reshape(1.0 / tf.to_float(self.input_lengths), (batch_size, 1))
S = z * tf.reduce_sum(q * word_feats, axis=1)
# Compute common component
S_centered = S - tf.reduce_mean(S, axis=0)
_, V = tf.svd(S_centered, full_matrices=False, compute_uv=True)
self.tf_ccx = tf.stop_gradient(tf.gather(tf.transpose(V), 0))
# Common component removal
ccx = tf.reshape(self._get_common_component(), (1, self.d))
sv = {'embeddings': word_embeddings, 'a': a, 'p': p, 'ccx': ccx}
return S - tf.matmul(S, ccx * tf.transpose(ccx)), sv
def _get_embedding(self):
"""
Return embedding tensor (either constant or variable)
Row 0 is 0 vector for no token
Row 1 is random initialization for UNKNowN
Rows 2 : 2 + len(self.embedding_words) are pretrained initialization
Remaining rows are random initialization
"""
zero = tf.constant(0.0, shape=(1, self.d))
s = self.seed - 1
unk = tf.Variable(tf.random_normal((1, self.d), stddev=SD, seed=s))
pretrain = tf.Variable(self.embeddings_train, dtype=tf.float32)
vecs = [zero, unk, pretrain]
n_r = self.word_dict.num_words() - len(self.embedding_words_train)
if n_r > 0:
r = tf.Variable(tf.random_normal((n_r, seed=s))
vecs.append(r)
self.U = tf.concat(vecs, axis=0, name='embedding_matrix')
return self.U
def pixel_wise_cross_entropy_loss_weighted(logits, class_weights):
'''
Weighted cross entropy loss,with a weight per class
:param logits: Network output before softmax
:param labels: Ground truth masks
:param class_weights: A list of the weights for each class
:return: weighted cross entropy loss
'''
n_class = len(class_weights)
flat_logits = tf.reshape(logits, n_class])
flat_labels = tf.reshape(labels, n_class])
class_weights = tf.constant(np.array(class_weights, dtype=np.float32))
weight_map = tf.multiply(flat_labels, class_weights)
weight_map = tf.reduce_sum(weight_map, axis=1)
loss_map = tf.nn.softmax_cross_entropy_with_logits(logits=flat_logits, labels=flat_labels)
weighted_loss = tf.multiply(loss_map, weight_map)
loss = tf.reduce_mean(weighted_loss)
return loss
def image_reading(path: str, resized_size: Tuple[int, int]=None, data_augmentation: bool=False,
padding: bool=False) -> Tuple[tf.Tensor, tf.Tensor]:
# Read image
image_content = tf.read_file(path, name='image_reader')
image = tf.cond(tf.equal(tf.string_split([path], '.').values[1], tf.constant('jpg', dtype=tf.string)),
true_fn=lambda: tf.image.decode_jpeg(image_content, channels=1, try_recover_truncated=True), # Todo channels = 3 ?
false_fn=lambda: tf.image.decode_png(image_content, channels=1), name='image_decoding')
# Data augmentation
if data_augmentation:
image = augment_data(image)
# Padding
if padding:
with tf.name_scope('padding'):
image, img_width = padding_inputs_width(image, resized_size, increment=CONST.DIMENSION_REDUCTION_W_POOLING)
# Resize
else:
image = tf.image.resize_images(image, size=resized_size)
img_width = tf.shape(image)[1]
with tf.control_dependencies([tf.assert_equal(image.shape[:2], resized_size)]):
return image, img_width
def add_input_op(self, xavier):
with tf.variable_scope('embed'):
# first the embed the input
if self.config.train_input_embeddings:
if self.config.input_embedding_matrix:
initializer = tf.constant_initializer(self.config.input_embedding_matrix)
else:
initializer = xavier
input_embed_matrix = tf.get_variable('input_embedding',
shape=(self.config.dictionary_size, self.config.embed_size),
initializer=initializer)
else:
input_embed_matrix = tf.constant(self.config.input_embedding_matrix)
# dictionary size x embed_size
assert input_embed_matrix.get_shape() == (self.config.dictionary_size, self.config.embed_size)
# Now embed the output
if self.config.train_output_embeddings:
output_embed_matrix = tf.get_variable('output_embedding',
shape=(self.config.output_size, self.config.output_embed_size),
initializer=xavier)
else:
output_embed_matrix = tf.constant(self.config.output_embedding_matrix)
assert output_embed_matrix.get_shape() == (self.config.output_size, self.config.output_embed_size)
inputs = tf.nn.embedding_lookup([input_embed_matrix], self.input_placeholder)
# batch size x max length x embed_size
assert inputs.get_shape()[1:] == (self.config.max_length, self.config.embed_size)
return inputs, output_embed_matrix
def calculate_loss_mix(self, **unused_params):
with tf.name_scope("loss_mix"):
float_labels = tf.cast(labels, tf.float32)
if FLAGS.support_type=="class":
seq = np.loadtxt(FLAGS.class_file)
tf_seq = tf.one_hot(tf.constant(seq,dtype=tf.int32),FLAGS.encoder_size)
float_classes_org = tf.matmul(float_labels,tf_seq)
class_true = tf.ones(tf.shape(float_classes_org))
class_false = tf.zeros(tf.shape(float_classes_org))
float_classes = tf.where(tf.greater(float_classes_org, class_false), class_true, class_false)
cross_entropy_class = self.calculate_loss(predictions_class,float_classes)
elif FLAGS.support_type=="frequent":
float_classes = float_labels[:,0:FLAGS.encoder_size]
cross_entropy_class = self.calculate_loss(predictions_class,float_classes)
elif FLAGS.support_type=="encoder":
float_classes = float_labels
for i in range(FLAGS.encoder_layers):
var_i = np.loadtxt(FLAGS.autoencoder_dir+'autoencoder_layer%d.model' % i)
weight_i = tf.constant(var_i[:-1,dtype=tf.float32)
bias_i = tf.reshape(tf.constant(var_i[-1,[-1])
float_classes = tf.nn.xw_plus_b(float_classes,bias_i)
if i<FLAGS.encoder_layers-1:
float_classes = tf.nn.relu(float_classes)
else:
float_classes = tf.nn.sigmoid(float_classes)
#float_classes = tf.nn.relu(tf.sign(float_classes - 0.5))
cross_entropy_class = self.calculate_mseloss(predictions_class,float_classes)
else:
float_classes = float_labels
for i in range(FLAGS.moe_layers-1):
float_classes = tf.concat((float_classes,float_labels),axis=1)
cross_entropy_class = self.calculate_loss(predictions_class,float_classes)
cross_entropy_loss = self.calculate_loss(predictions,labels)
return cross_entropy_loss + 0.1*cross_entropy_class
def calculate_loss_mix(self, **unused_params):
with tf.name_scope("loss_softmax_mix"):
vocab_size = labels.get_shape().as_list()[1]
cross_entropy_class = tf.constant(0.0)
for i in range(FLAGS.moe_layers):
predictions_subclass = predictions_class[:,i*vocab_size:(i+1)*vocab_size]
cross_entropy_class = cross_entropy_class + self.calculate_loss(predictions_subclass,labels)
cross_entropy_loss = self.calculate_loss(predictions,labels)
return cross_entropy_loss + 0.1*cross_entropy_class
def calculate_loss(self, **unused_params):
with tf.name_scope("loss_xent"):
epsilon = 10e-6
origin_labels = tf.cast(labels, tf.float32)
vocab_size = origin_labels.get_shape().as_list()[1]
float_labels = tf.tile(tf.reshape(origin_labels,[-1, vocab_size]),[1,FLAGS.top_k,1])
float_labels = tf.reshape(float_labels,vocab_size])
cross_entropy_loss = float_labels * tf.log(predictions + epsilon) + (
1 - float_labels) * tf.log(1 - predictions + epsilon)
cross_entropy_loss = tf.negative(cross_entropy_loss)
num_labels = tf.minimum(tf.reduce_sum(origin_labels,axis=1),tf.constant(FLAGS.top_k,dtype=tf.float32))
mask = tf.reshape(tf.sequence_mask(num_labels,[-1])
cross_entropy_loss = tf.reduce_sum(tf.reduce_sum(cross_entropy_loss, 1)*mask)/(tf.reduce_sum(mask)+epsilon)
return cross_entropy_loss
def threshold_from_predictions(y, y_pred, false_positive_margin=0, recall=1):
"""Determines a threshold for classifying examples as positive
Args:
y: labels
y_pred: scores from the classifier
recall: Threshold is set to classify at least this fraction of positive
labelled examples as positive
false_positive_margin: Threshold is set to acheive desired recall,and
then is extended to include an additional fraction of negative
labelled examples equal to false_positive_margin (This allows adding
a buffer to the threshold while maintaining a constant "cost")
"""
n_positive = np.count_nonzero(y)
n_negative = len(y) - n_positive
if n_positive == 0:
return np.max(y_pred)
if false_positive_margin == 0 and recall == 1:
return np.min(y_pred[y])
ind = np.argsort(y_pred)
y_pred_sorted = y_pred[ind]
y_sorted = y[ind]
so_far = [0, 0]
j = 0
for i in reversed(range(len(y_sorted))):
so_far[y_sorted[i]] += 1
if so_far[1] >= int(np.floor(recall * n_positive)):
j = i
break
so_far = [0, 0]
if false_positive_margin == 0:
return y_pred_sorted[j]
k = 0
for i in reversed(range(j)):
so_far[y_sorted[i]] += 1
if so_far[0] >= false_positive_margin * n_negative:
k = i
break
return y_pred_sorted[k]
def threshold_from_predictions(y, 0]
if false_positive_margin == 0:
return y_pred_sorted[j]
k = 0
for i in reversed(range(j)):
so_far[y_sorted[i]] += 1
if so_far[0] >= false_positive_margin * n_negative:
k = i
break
return y_pred_sorted[k]
def threshold_from_predictions(y, 0]
if false_positive_margin == 0:
return y_pred_sorted[j]
k = 0
for i in reversed(range(j)):
so_far[y_sorted[i]] += 1
if so_far[0] >= false_positive_margin * n_negative:
k = i
break
return y_pred_sorted[k]
def threshold_from_predictions(y, 0]
if false_positive_margin == 0:
return y_pred_sorted[j]
k = 0
for i in reversed(range(j)):
so_far[y_sorted[i]] += 1
if so_far[0] >= false_positive_margin * n_negative:
k = i
break
return y_pred_sorted[k]
def gauss_log_prob(mu, logstd, x):
var = tf.exp(2*logstd)
gp = -tf.square(x - mu)/(2*var) - .5*tf.log(tf.constant(2*np.pi)) - logstd
return tf.reduce_sum(gp, [1])
def gauss_ent(mu, logstd):
h = tf.reduce_sum(logstd + tf.constant(0.5*np.log(2*np.pi*np.e), tf.float32))
return h
版权声明:本文内容由互联网用户自发贡献,该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容, 请发送邮件至 dio@foxmail.com 举报,一经查实,本站将立刻删除。