Python tensorflow 模块,ConfigProto() 实例源码
我们从Python开源项目中,提取了以下50个代码示例,用于说明如何使用tensorflow.ConfigProto()。
def __init__(self, cluster, task, train_dir, log_device_placement=True):
""""Creates a Trainer.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
self.cluster = cluster
self.task = task
self.is_master = (task.type == "master" and task.index == 0)
self.train_dir = train_dir
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.2)
self.config = tf.ConfigProto(log_device_placement=log_device_placement,gpu_options=gpu_options)
if self.is_master and self.task.index > 0:
raise StandardError("%s: Only one replica of master expected",
task_as_string(self.task))
def __init__(self, log_device_placement=True):
""""Creates a Trainer.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
self.cluster = cluster
self.task = task
self.is_master = (task.type == "master" and task.index == 0)
self.train_dir = train_dir
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=FLAGS.gpu)
self.config = tf.ConfigProto(log_device_placement=log_device_placement)
if self.is_master and self.task.index > 0:
raise StandardError("%s: Only one replica of master expected",
task_as_string(self.task))
def __init__(self, log_device_placement=True):
""""Creates a Trainer.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
self.cluster = cluster
self.task = task
self.is_master = (task.type == "master" and task.index == 0)
self.train_dir = train_dir
self.config = tf.ConfigProto(log_device_placement=log_device_placement)
if self.is_master and self.task.index > 0:
raise StandardError("%s: Only one replica of master expected",
task_as_string(self.task))
def __init__(self, log_device_placement=True):
""""Creates a Trainer.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
self.cluster = cluster
self.task = task
self.is_master = (task.type == "master" and task.index == 0)
self.train_dir = train_dir
self.config = tf.ConfigProto(log_device_placement=log_device_placement)
if self.is_master and self.task.index > 0:
raise StandardError("%s: Only one replica of master expected",
task_as_string(self.task))
def create_config_proto(params):
"""Returns session config proto.
Args:
params: Params tuple,typically created by make_params or
make_params_from_flags.
"""
config = tf.ConfigProto()
config.allow_soft_placement = True
config.intra_op_parallelism_threads = params.num_intra_threads
config.inter_op_parallelism_threads = params.num_inter_threads
config.gpu_options.force_gpu_compatible = params.force_gpu_compatible
if params.gpu_memory_frac_for_testing > 0:
config.gpu_options.per_process_gpu_memory_fraction = (
params.gpu_memory_frac_for_testing)
if params.xla:
config.graph_options.optimizer_options.global_jit_level = (
tf.OptimizerOptions.ON_1)
if params.enable_layout_optimizer:
config.graph_options.rewrite_options.layout_optimizer = (
rewriter_config_pb2.RewriterConfig.ON)
return config
def configure_gpu_settings(gpu_cfg=None):
session_conf = None
if gpu_cfg:
with open(gpu_cfg) as f:
cfg = json.load(f)
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=cfg['per_process_gpu_memory_fraction'])
session_conf = tf.ConfigProto(
allow_soft_placement=cfg['allow_soft_placement'],
log_device_placement=cfg['log_device_placement'],
inter_op_parallelism_threads=cfg['inter_op_parallelism_threads'],
intra_op_parallelism_threads=cfg['intra_op_parallelism_threads'],
gpu_options=gpu_options)
# Timeline
# jit_level = 0
# session_conf.graph_options.optimizer_options.global_jit_level = jit_level
# sess = tf.Session(
# config=session_conf)
# else:
# sess = tf.Session()
return session_conf
def train_net(network, imdb, roidb, valroidb, output_dir, tb_dir,
pretrained_model=None,
max_iters=40000):
"""Train a Faster R-CNN network."""
roidb = filter_roidb(roidb)
valroidb = filter_roidb(valroidb)
tfconfig = tf.ConfigProto(allow_soft_placement=True)
tfconfig.gpu_options.allow_growth = True
with tf.Session(config=tfconfig) as sess:
sw = SolverWrapper(sess, network,
pretrained_model=pretrained_model)
print('Solving...')
sw.train_model(sess, max_iters)
print('done solving')
def setUp(self):
"""Set up class before _each_ test method is executed.
Creates a tensorflow session and instantiates a dbinterface.
"""
self.setup_model()
self.sess = tf.Session(
config=tf.ConfigProto(
allow_soft_placement=True,
gpu_options=tf.GPUOptions(allow_growth=True),
log_device_placement=self.params['log_device_placement'],
inter_op_parallelism_threads=self.params['inter_op_parallelism_threads']))
# Todo: Determine whether this should be called here or
# in dbinterface.initialize()
self.sess.run(tf.global_variables_initializer())
self.dbinterface = base.DBInterface(sess=self.sess,
params=self.params,
cache_dir=self.CACHE_DIR,
save_params=self.save_params,
load_params=self.load_params)
self.step = 0
def main(_):
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
model = UNet(args.experiment_dir, batch_size=args.batch_size, experiment_id=args.experiment_id,
input_width=args.image_size, output_width=args.image_size, embedding_num=args.embedding_num,
embedding_dim=args.embedding_dim, L1_penalty=args.L1_penalty, Lconst_penalty=args.Lconst_penalty,
Ltv_penalty=args.Ltv_penalty, Lcategory_penalty=args.Lcategory_penalty)
model.register_session(sess)
if args.flip_labels:
model.build_model(is_training=True, inst_norm=args.inst_norm, no_target_source=True)
else:
model.build_model(is_training=True, inst_norm=args.inst_norm)
fine_tune_list = None
if args.fine_tune:
ids = args.fine_tune.split(",")
fine_tune_list = set([int(i) for i in ids])
model.train(lr=args.lr, epoch=args.epoch, resume=args.resume,
schedule=args.schedule, freeze_encoder=args.freeze_encoder, fine_tune=fine_tune_list,
sample_steps=args.sample_steps, checkpoint_steps=args.checkpoint_steps,
flip_labels=args.flip_labels, no_val=args.no_val)
def tracking(dataset, seq, display, restore_path):
train_data = reader.read_seq(dataset, seq)
im_size = proc.load_image(train_data.data[seq].frames[0]).shape[:2]
config = Config(im_size)
# create session and saver
gpu_config = tf.ConfigProto(allow_soft_placement=True)
sess = tf.InteractiveSession(config=gpu_config)
# load model,weights
model = MDNet(config)
model.build_generator(config.batch_size, reuse=False, dropout=True)
tf.global_variables_initializer().run()
# create saver
saver = tf.train.Saver([v for v in tf.global_variables() if ('conv' in v.name or 'fc4' in v.name or 'fc5' in v.name) \
and 'lr_rate' not in v.name], max_to_keep=50)
# restore from model
saver.restore(sess, restore_path)
# run mdnet
mdnet_run(sess, model, train_data.data[seq].gts[0], train_data.data[seq].frames, config, display)
def predict():
# Only allocate part of the gpu memory when predicting.
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.2)
tf_config = tf.ConfigProto(gpu_options=gpu_options)
with tf.Session(config=tf_config) as sess:
predictor = EasyPredictor(sess)
sys.stdout.write("> ")
sys.stdout.flush()
line = sys.stdin.readline()
while line:
replies = predictor.predict(line)
for i, text in enumerate(replies):
print(i, text)
print("> ", end="")
sys.stdout.flush()
line = sys.stdin.readline()
def run(self):
"""Build a graph and run the model on random input data."""
_logger.info("Creating graph.")
with tf.Graph().as_default():
_logger.info("Building model.")
self.build_model_loss()
_logger.info("Starting session.")
config = tf.ConfigProto(log_device_placement=FLAGS.log_device_placement)
with tf.Session(config=config) as sess:
_logger.info("Initializing variables.")
sess.run(tf.global_variables_initializer())
_logger.info("Starting timing test.")
self.evaluate(sess)
_logger.info("Ending session.")
def main():
# Graph
with tf.device('/cpu:0'):
a = tf.Variable(tf.truncated_normal(shape=[2]),dtype=tf.float32)
b = tf.Variable(tf.truncated_normal(shape=[2]),dtype=tf.float32)
c=a+b
target = tf.constant(100.,shape=[2],dtype=tf.float32)
loss = tf.reduce_mean(tf.square(c-target))
opt = tf.train.GradientDescentOptimizer(.0001).minimize(loss)
# Session
#sv = tf.train.Supervisor(logdir='/tmp/mydir')
sv = tf.train.Supervisor(logdir='/tmp/mydir')
gpu_options = tf.GPUOptions(allow_growth=True,allocator_type="BFC",visible_device_list="%d"%FLAGS.gpu_id)
config = tf.ConfigProto(gpu_options=gpu_options,allow_soft_placement=False,device_count={'GPU':1},log_device_placement=True)
sess = sv.prepare_or_wait_for_session(config=config)
for i in range(1000):
sess.run(opt)
if i % 10 == 0:
r = sess.run(c)
print(r)
time.sleep(.1)
def main(_):
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=calc_gpu_fraction(FLAGS.gpu_fraction))
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
config = get_config(FLAGS) or FLAGS
if config.env_type == 'simple':
env = SimpleGymEnvironment(config)
else:
env = GymEnvironment(config)
if not tf.test.is_gpu_available() and FLAGS.use_gpu:
raise Exception("use_gpu flag is true when no GPUs are available")
if not FLAGS.use_gpu:
config.cnn_format = 'NHWC'
agent = Agent(config, env, sess)
if FLAGS.is_train:
agent.train()
else:
agent.play()
def evaluate():
"""Eval ocr for a number of steps."""
with tf.Graph().as_default() as g:
images, labels, seq_lengths = ocr.inputs()
logits, timesteps = ocr.inference(images, FLAGS.eval_batch_size, train=True)
ler = ocr.create_label_error_rate(logits, timesteps)
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
config = tf.ConfigProto(
device_count={'GPU': 0}
)
sess = tf.Session(config=config)
sess.run(init_op)
saver = tf.train.Saver()
summary_op = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(FLAGS.eval_dir, g)
while True:
eval_once(saver, summary_writer, ler, summary_op)
if FLAGS.run_once:
break
# print("Waiting for next evaluation for " + str(FLAGS.eval_interval_secs) + " sec")
time.sleep(FLAGS.eval_interval_secs)
def _initialize_session(self):
"""Initialize session,variables,saver"""
config = tf.ConfigProto()
# restrict model GPU memory utilization to min required
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
tf_ver = int(tf.__version__.split('.')[1])
if TF_VERSION <= 0.10:
self.sess.run(tf.initialize_all_variables())
logswriter = tf.train.SummaryWriter
else:
self.sess.run(tf.global_variables_initializer())
logswriter = tf.summary.FileWriter
self.saver = tf.train.Saver(tf.global_variables(), max_to_keep=0)
self.summary_writer = logswriter(self.logs_path, self.sess.graph)
# (Updated)
def main(argv=None): # pylint: disable=unused-argument
assert args.detect or args.segment, "Either detect or segment should be True"
assert args.ckpt > 0, "Specify the number of checkpoint"
net = resnet(config=net_config, depth=50, training=False)
loader = Loader()
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)) as sess:
detector = Detector(sess, net, loader, net_config, no_gt=args.no_seg_gt,
folder=osp.join(loader.folder, 'output'))
detector.restore_from_ckpt(args.ckpt)
for name in loader.get_filenames():
image = loader.load_image(name)
h, w = image.shape[:2]
print('Processing {}'.format(name + loader.data_format))
detector.Feed_forward(img=image, name=name, w=w, h=h, draw=True,
seg_gt=None, gt_bBoxes=None, gt_cats=None)
print('Done')
def train(env_id, num_timesteps, seed):
env=gym.make(env_id)
env = bench.Monitor(env, logger.get_dir() and os.path.join(logger.get_dir(), str(rank)))
set_global_seeds(seed)
env.seed(seed)
gym.logger.setLevel(logging.WARN)
with tf.Session(config=tf.ConfigProto()):
ob_dim = env.observation_space.shape[0]
ac_dim = env.action_space.shape[0]
with tf.variable_scope("vf"):
vf = NeuralNetValueFunction(ob_dim, ac_dim)
with tf.variable_scope("pi"):
policy = GaussianMlpPolicy(ob_dim, ac_dim)
learn(env, policy=policy, vf=vf,
gamma=0.99, lam=0.97, timesteps_per_batch=2500,
desired_kl=0.002,
num_timesteps=num_timesteps, animate=False)
env.close()
def get_session():
tf.reset_default_graph()
tf_config = tf.ConfigProto(
inter_op_parallelism_threads=1,
intra_op_parallelism_threads=1)
# This was the default provided in the starter code.
#session = tf.Session(config=tf_config)
# Use this if I want to see what is on the GPU.
#session = tf.Session(config=tf.ConfigProto(log_device_placement=True))
# Use this for limiting memory allocated for the GPU.
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.5)
session = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
print("AVAILABLE GPUS: ", get_available_gpus())
return session
def process(input_dir, model_dir, resizing_size, gpu):
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.3, visible_device_list=gpu)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)).as_default():
m = loader.LoadedModel(model_dir)
os.makedirs(output_dir, exist_ok=True)
input_filenames = glob(os.path.join(input_dir, '*.jpg')) + \
glob(os.path.join(input_dir, '*.png')) + \
glob(os.path.join(input_dir, '*.tif')) + \
glob(os.path.join(input_dir, '*.jp2'))
for path in tqdm(input_filenames):
img = Image.open(path).resize(resizing_size)
mat = np.asarray(img)
if len(mat.shape) == 2:
mat = np.stack([mat, mat, mat], axis=2)
predictions = m.predict(mat[None], prediction_key='labels')[0]
plt.imsave(os.path.join(output_dir, os.path.relpath(path, input_dir)), predictions)
def validation_check():
# Load graph
g = Graph(is_training=False); print("Graph loaded")
# Load data
X, Y = load_data(mode="val")
with g.graph.as_default():
sv = tf.train.Supervisor()
with sv.managed_session(config=tf.ConfigProto(allow_soft_placement=True)) as sess:
# Restore parameters
sv.saver.restore(sess, tf.train.latest_checkpoint(hp.logdir)); print("Restored!")
# Get model
mname = open(hp.logdir + '/checkpoint', 'r').read().split('"')[1] # model name
# Inference
if not os.path.exists(hp.results): os.mkdir(hp.results)
with open(os.path.join(hp.results, "validation_results.txt"), 'a') as fout:
expected, predicted = [], []
for step in range(len(X) // hp.batch_size):
x = X[step * hp.batch_size: (step + 1) * hp.batch_size]
y = Y[step * hp.batch_size: (step + 1) * hp.batch_size]
# predict intensities
logits = sess.run(g.logits, {g.x: x})
expected.extend(list(y))
predicted.extend(list(logits))
# Get spearman coefficients
score, _ = spearmanr(expected, predicted)
fout.write("{}\t{}\n".format(mname, score))
def __init__(self, checkpoint_file):
checkpoint_dir = os.path.dirname(checkpoint_file)
hparams_file = os.path.join(checkpoint_dir, "hparams.txt")
hparams_dict = {}
if os.path.isfile(hparams_file):
with open(hparams_file) as f:
hparams_dict = ast.literal_eval(f.read())
self.hparams = TensorflowClassifierHparams(**hparams_dict)
self.graph = tf.Graph()
with self.graph.as_default():
print("loading from file {}".format(checkpoint_file))
config = tf.ConfigProto(
device_count={'GPU': 0}, )
config.gpu_options.visible_device_list = ""
self.session = tf.Session(config=config)
new_saver = tf.train.import_Meta_graph(checkpoint_file + ".Meta", clear_devices=True)
new_saver.restore(self.session, checkpoint_file)
self.features = {}
if self.hparams.use_image:
self.features["image"] = self.graph.get_tensor_by_name("image:0")
if self.hparams.use_observation:
self.features["observation"] = self.graph.get_tensor_by_name("observation:0")
if self.hparams.use_action:
self.features["action"] = self.graph.get_tensor_by_name("action:0")
self.prediction = tf.get_collection('prediction')[0]
self.loss = tf.get_collection('loss')[0]
self.threshold = tf.get_collection('threshold')[0]
def __init__(self, checkpoint_file)
self.features = {}
if self.hparams.use_image:
self.features["image"] = self.graph.get_tensor_by_name("image:0")
if self.hparams.use_observation:
self.features["observation"] = self.graph.get_tensor_by_name("observation:0")
if self.hparams.use_action:
self.features["action"] = self.graph.get_tensor_by_name("action:0")
self.prediction = tf.get_collection('prediction')[0]
self.loss = tf.get_collection('loss')[0]
self.threshold = tf.get_collection('threshold')[0]
def __init__(self, checkpoint_file)
self.features = {}
if self.hparams.use_image:
self.features["image"] = self.graph.get_tensor_by_name("image:0")
if self.hparams.use_observation:
self.features["observation"] = self.graph.get_tensor_by_name("observation:0")
if self.hparams.use_action:
self.features["action"] = self.graph.get_tensor_by_name("action:0")
self.prediction = tf.get_collection('prediction')[0]
self.loss = tf.get_collection('loss')[0]
self.threshold = tf.get_collection('threshold')[0]
def __init__(self, checkpoint_file)
self.features = {}
if self.hparams.use_image:
self.features["image"] = self.graph.get_tensor_by_name("image:0")
if self.hparams.use_observation:
self.features["observation"] = self.graph.get_tensor_by_name("observation:0")
if self.hparams.use_action:
self.features["action"] = self.graph.get_tensor_by_name("action:0")
self.prediction = tf.get_collection('prediction')[0]
self.loss = tf.get_collection('loss')[0]
self.threshold = tf.get_collection('threshold')[0]
def get_genre(album_or_phrase):
try:
with graph.as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
sess = tf.Session(config=session_conf)
with sess.as_default():
# Load the saved Meta graph and restore variables
saver = tf.train.import_Meta_graph("{}.Meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
x_raw = [album_or_phrase]
all_predictions = []
x_test = np.array(list(vocab_processor.transform(x_raw)))
# Get the placeholders from the graph by name
input_x = graph.get_operation_by_name("input_x").outputs[0]
dropout_keep_prob = graph.get_operation_by_name("dropout_keep_prob").outputs[0]
# Tensors we want to evaluate
predictions = graph.get_operation_by_name("output/predictions").outputs[0]
# Generate batches for one epoch
batches = data_loader.batch_iter(list(x_test), FLAGS.batch_size, 1, shuffle=False)
for x_test_batch in batches:
batch_predictions = sess.run(predictions, {input_x: x_test_batch, dropout_keep_prob: 1.0})
all_predictions = np.concatenate([all_predictions, batch_predictions])
return data_loader.genre_ids[int(all_predictions[0])]
except Exception as e:
print e
def get_genre():
try:
with graph.as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement, checkpoint_file)
albums = request.args.get('albums')
x_raw = albums.split(',')
all_predictions = []
x_test = np.array(list(vocab_processor.transform(x_raw)))
# Get the placeholders from the graph by name
input_x = graph.get_operation_by_name("input_x").outputs[0]
dropout_keep_prob = graph.get_operation_by_name("dropout_keep_prob").outputs[0]
# Tensors we want to evaluate
predictions = graph.get_operation_by_name("output/predictions").outputs[0]
# Generate batches for one epoch
batches = data_loader.batch_iter(list(x_test), batch_predictions])
return jsonify({'results': map(lambda x: data_loader.genre_ids[int(x)], all_predictions)})
except Exception as e:
print e
def evaluate(num_Votes):
is_training = False
with tf.device('/gpu:'+str(GPU_INDEX)):
pointclouds_pl, labels_pl = MODEL.placeholder_inputs(BATCH_SIZE, NUM_POINT)
is_training_pl = tf.placeholder(tf.bool, shape=())
# simple model
pred, end_points = MODEL.get_model(pointclouds_pl, is_training_pl)
loss = MODEL.get_loss(pred, labels_pl, end_points)
# Add ops to save and restore all the variables.
saver = tf.train.Saver()
# Create a session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
config.log_device_placement = True
sess = tf.Session(config=config)
# Restore variables from disk.
saver.restore(sess, MODEL_PATH)
log_string("Model restored.")
ops = {'pointclouds_pl': pointclouds_pl,
'labels_pl': labels_pl,
'is_training_pl': is_training_pl,
'pred': pred,
'loss': loss}
eval_one_epoch(sess, ops, num_Votes)
def main():
if args.logdir is None:
raise ValueError('Please specify the logdir file')
ckpt = get_checkpoint(args.logdir)
if ckpt is None:
raise ValueError('No checkpoints in {}'.format(args.logdir))
with open(os.path.join(args.logdir, 'architecture.json')) as f:
arch = json.load(f)
reader = VCC2016TFRManager()
features = reader.read_whole(args.file_pattern, num_epochs=1)
x = features['frame']
y = features['label']
filename = features['filename']
y_conv = y * 0 + args.target_id
net = MLPcVAE(arch=arch, is_training=False)
z = net.encode(x)
xh = net.decode(z, y)
x_conv = net.decode(z, y_conv)
pre_train_saver = tf.train.Saver()
def load_pretrain(sess):
pre_train_saver.restore(sess, ckpt)
sv = tf.train.Supervisor(init_fn=load_pretrain)
gpu_options = tf.GPUOptions(allow_growth=True)
sess_config = tf.ConfigProto(
allow_soft_placement=True,
gpu_options=gpu_options)
with sv.managed_session(config=sess_config) as sess:
for _ in range(reader.n_files):
if sv.should_stop():
break
fetch_dict = {'x': x, 'xh': xh, 'x_conv': x_conv, 'f': filename}
results = sess.run(fetch_dict)
plot_spectra(results)
def train(self, nIter, machine=None, summary_op=None):
# Xh = self._validate(machine=machine,n=10)
run_Metadata = tf.RunMetadata()
sv = tf.train.Supervisor(
logdir=self.dirs['logdir'],
# summary_writer=summary_writer,
# summary_op=None,
# is_chief=True,
save_model_secs=300,
global_step=self.opt['global_step'])
# sess_config = configure_gpu_settings(args.gpu_cfg)
sess_config = tf.ConfigProto(
allow_soft_placement=True,
gpu_options=tf.GPUOptions(allow_growth=True))
with sv.managed_session(config=sess_config) as sess:
sv.loop(60, self._refresh_status, (sess,))
for step in range(self.arch['training']['max_iter']):
if sv.should_stop():
break
# main loop
sess.run(self.opt['g'])
# # output img
# if step % 1000 == 0:
# xh = sess.run(Xh)
# with tf.gfile.GFile(
# os.path.join(
# self.dirs['logdir'],
# 'img-anime-{:03d}k.png'.format(step // 1000),
# ),
# mode='wb',
# ) as fp:
# fp.write(xh)
def _create_session(self, session_config: Optional[dict]) -> tf.Session:
"""
Create and return TF Session for this model.
By default the session is configured with ``tf.ConfigProto`` created with
the given ``session_config`` as ``**kwargs``. nested dictionaries such as
``gpu_options`` or ``graph_options`` are handled automatically.
:param session_config: session configuration dict as specified in the config yaml
:return: TensorFlow session
"""
if session_config:
session_config = tf.ConfigProto(**session_config)
return tf.Session(graph=self._graph, config=session_config)
def __init__(self, energy_fn, prior, std=1.0,
inter_op_parallelism_threads=1, intra_op_parallelism_threads=1):
self.energy_fn = energy_fn
self.prior = prior
self.z = self.energy_fn.z
def fn(z, x):
z_ = z + tf.random_normal(tf.shape(self.z), 0.0, std)
accept = metropolis_hastings_accept(
energy_prev=energy_fn(z),
energy_next=energy_fn(z_)
)
return tf.where(accept, z_, z)
self.steps = tf.placeholder(tf.int32, [])
elems = tf.zeros([self.steps])
self.z_ = tf.scan(
fn, elems, self.z, back_prop=False
)
self.sess = tf.Session(
config=tf.ConfigProto(
inter_op_parallelism_threads=inter_op_parallelism_threads,
intra_op_parallelism_threads=intra_op_parallelism_threads
)
)
self.sess.run(tf.global_variables_initializer())
def train_test_validation():
M = read_dataset()
num_rating = np.count_nonzero(M)
idx = np.arange(num_rating)
np.random.seed(1)
np.random.shuffle(idx)
train_idx = idx[:int(0.85 * num_rating)]
valid_idx = idx[int(0.85 * num_rating):int(0.90 * num_rating)]
test_idx = idx[int(0.90 * num_rating):]
for hidden_encoder_dim, hidden_decoder_dim, latent_dim, learning_rate, batch_size, reg_param, vae in itertools.product(hedims, hddims, ldims, lrates, bsizes, regs, vaes):
result_path = "{0}_{1}_{2}_{3}_{4}_{5}_{6}".format(
hidden_encoder_dim, vae)
if not os.path.exists(result_path + "/model.ckpt.index"):
config = tf.ConfigProto()
config.gpu_options.allow_growth=True
with tf.Session(config=config) as sess:
model = VAEMF(sess, num_user, num_item,
hidden_encoder_dim=hidden_encoder_dim, hidden_decoder_dim=hidden_decoder_dim,
latent_dim=latent_dim, learning_rate=learning_rate, batch_size=batch_size, reg_param=reg_param, vae=vae)
print("Train size={0},Validation size={1},Test size={2}".format(
train_idx.size, valid_idx.size, test_idx.size))
print(result_path)
best_rmse = model.train_test_validation(M, train_idx=train_idx, test_idx=test_idx, valid_idx=valid_idx, n_steps=n_steps, result_path=result_path)
print("Best MSE = {0}".format(best_rmse))
with open('result.csv', 'a') as f:
f.write("{0},{1},{2},{3},{4},{5},{6},{7}\n".format(hidden_encoder_dim,
latent_dim, vae, best_rmse))
tf.reset_default_graph()
def __init__(self, fnames, shuffle=True, num_epochs=None):
"""Init from a list of filenames to enqueue.
Args:
fnames: list of .tfrecords filenames to enqueue.
shuffle: if true,shuffle the list at each epoch
"""
self._fnames = fnames
self._fname_queue = tf.train.string_input_producer(
self._fnames,
capacity=1000,
shuffle=shuffle,
num_epochs=num_epochs,
shared_name='input_files')
self._reader = tf.TFRecordReader()
# Read first record to initialize the shape parameters
with tf.Graph().as_default():
fname_queue = tf.train.string_input_producer(self._fnames)
reader = tf.TFRecordReader()
_, serialized = reader.read(fname_queue)
shapes = self._parse_shape(serialized)
dtypes = self._parse_dtype(serialized)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
self.shapes = sess.run(shapes)
self.shapes = {k: self.shapes[k+'_sz'].tolist() for k in self.FEATURES}
self.dtypes = sess.run(dtypes)
self.dtypes = {k: REVERSE_TYPEMAP[self.dtypes[k+'_dtype'][0]] for k in self.FEATURES}
coord.request_stop()
coord.join(threads)
def main():
# Create the TensorFlow placeholders
placeholders = create_placeholders()
# Get the training Feed dicts and define the length of the test set.
train_Feed_dicts, vocab = load_data(placeholders)
num_test = int(len(train_Feed_dicts) * (1 / 5))
print("Number of Feed Dicts: ", len(train_Feed_dicts))
print("Number of Test Dicts: ", num_test)
# Slice the dictionary list into training and test sets
final_test_Feed_dicts = train_Feed_dicts[0:num_test]
test_Feed_dicts = train_Feed_dicts[0:50]
train_Feed_dicts = train_Feed_dicts[num_test:]
# Do not take up all the GPU memory,all the time.
sess_config = tf.ConfigProto()
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
logits, loss, preds, accuracy, saver = train(placeholders, train_Feed_dicts, test_Feed_dicts, vocab, sess=sess)
print('============')
# Test on train data - later,test on test data
avg_acc = 0
count = 0
for j, batch in enumerate(final_test_Feed_dicts):
acc = sess.run(accuracy, Feed_dict=batch)
print("Accuracy on test set is: ", acc)
avg_acc += acc
count += 1
print('-----')
print("Overall Average Accuracy on the Test Set Is: ", avg_acc / count)
def compute_embeddings(images):
"""Runs inference on an image.
Args:
image: Image file names.
Returns:
Dict mapping image file name to embedding.
"""
# Creates graph from saved GraphDef.
create_graph()
filename_to_emb = {}
config = tf.ConfigProto(device_count = {'GPU': 0})
bar = progressbar.ProgressBar(widgets=[progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage()])
with tf.Session(config=config) as sess:
i = 0
for image in bar(images):
if not tf.gfile.Exists(image):
tf.logging.fatal('File does not exist %s', image)
image_data = tf.gfile.FastGFile(image, 'rb').read()
# Some useful tensors:
# 'softmax:0': A tensor containing the normalized prediction across
# 1000 labels.
# 'pool_3:0': A tensor containing the next-to-last layer containing 2048
# float description of the image.
# 'DecodeJpeg/contents:0': A tensor containing a string providing JPEG
# encoding of the image.
# Runs the softmax tensor by Feeding the image_data as input to the graph.
softmax_tensor = sess.graph.get_tensor_by_name('softmax:0')
embedding_tensor = sess.graph.get_tensor_by_name('pool_3:0')
embedding = sess.run(embedding_tensor,
{'DecodeJpeg/contents:0': image_data})
filename_to_emb[image] = embedding.reshape(2048)
i += 1
# print(image,i,len(images))
return filename_to_emb
# temp_dir is a subdir of temp
def main(_):
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
model = UNet(batch_size=args.batch_size)
model.register_session(sess)
model.build_model(is_training=False, inst_norm=args.inst_norm)
model.export_generator(save_dir=args.save_dir, model_dir=args.model_dir)
def main(_):
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
with tf.Session(config=config) as sess:
model = UNet(batch_size=args.batch_size)
model.register_session(sess)
model.build_model(is_training=False, inst_norm=args.inst_norm)
embedding_ids = [int(i) for i in args.embedding_ids.split(",")]
if not args.interpolate:
if len(embedding_ids) == 1:
embedding_ids = embedding_ids[0]
if args.compare:
model.infer_compare(model_dir=args.model_dir, source_obj=args.source_obj, embedding_ids=embedding_ids,
save_dir=args.save_dir, show_ssim=args.show_ssim)
else:
model.infer(model_dir=args.model_dir, progress_file=args.progress_file)
else:
if len(embedding_ids) < 2:
raise Exception("no need to interpolate yourself unless you are a narcissist")
chains = embedding_ids[:]
if args.uroboros:
chains.append(chains[0])
pairs = list()
for i in range(len(chains) - 1):
pairs.append((chains[i], chains[i + 1]))
for s, e in pairs:
model.interpolate(model_dir=args.model_dir, between=[s, e],
save_dir=args.save_dir, steps=args.steps)
if args.output_gif:
gif_path = os.path.join(args.save_dir, args.output_gif)
compile_frames_to_gif(args.save_dir, gif_path)
print("gif saved at %s" % gif_path)
def restrict_gpu_memory(per_process_gpu_memory_fraction: float = 0.9):
import os
import tensorflow as tf
import keras
thread_count = os.environ.get('OMP_NUM_THREADS')
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=per_process_gpu_memory_fraction)
config = tf.ConfigProto(gpu_options=gpu_options,
allow_soft_placement=True,
intra_op_parallelism_threads=thread_count) \
if thread_count else tf.ConfigProto(gpu_options=gpu_options, allow_soft_placement=True)
keras.backend.tensorflow_backend.set_session(tf.Session(config=config))
def main(flags):
# ???????
ds = mnist.Data(flags.directory)
# tensorflow ??
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# ??session
with tf.Session(config=config) as session:
# ??????
model = Model('Model',
session,
flags.height,
flags.width,
flags.depth,
flags.feature_size,
flags.num_classes).build()
# ??slot
train = model.get_slot('train')
predict = model.get_slot('predict')
# ?????
session.run(tf.global_variables_initializer())
# ????
for i in range(1, flags.nloop + 1):
# ????batch???
images_batch, labels_batch = ds.next_batch(flags.bsize)
loss, train_accuracy = train(images_batch, labels_batch, 0.5)
# ?100?????????????batch??accuracy
if i % 100 == 0:
print('Loop {}:\tloss={}\ttrain accuracy={}'.format(i, train_accuracy))
# ????????accuracy
accuracy = predict(ds.test_images, ds.test_labels, 1.0)
print('Accuracy on test set: {}'.format(accuracy))
return 0
def main(flags):
# ???????
ds = DataSource(flags.directory)
# tensorflow ??
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# ??session
with tf.Session(config=config) as session:
# ??????
model = Model('Model', session, flags.input_size, flags.num_classes).build()
# ??slot
train = model.get_slot('train')
predict = model.get_slot('predict')
# ?????
session.run(tf.global_variables_initializer())
# ????
for i in range(1, labels_batch = ds.next_batch(flags.bsize)
# ?????????
loss = train(images_batch, labels_batch)
print('Loop {}:\tloss= {}'.format(i, loss))
# ????????accuracy
accuracy = predict(ds.test_images, ds.test_labels)
print('Accuracy on test set: {}'.format(accuracy))
return 0
版权声明:本文内容由互联网用户自发贡献,该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容, 请发送邮件至 dio@foxmail.com 举报,一经查实,本站将立刻删除。