1.技术原理以某一个通道的平均值作为优化目标不断让神经网络去调整输入图像的像素值让输出的图片中逐渐出现该通道喜欢的纹理和图案2.TensorFlow中的Deep Dream模型实践导入Inception:下载文件后复制到chapter4目录下https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip# coding:utf-8 from __future__ import print_function import numpy as np import tensorflow.compat.v1 as tf tf.disable_v2_behavior() graph tf.Graph() sess tf.InteractiveSession(graphgraph) model_fn tensorflow_inception_graph.pb with tf.gfile.GFile(model_fn, rb) as f: graph_def tf.GraphDef() graph_def.ParseFromString(f.read()) t_input tf.placeholder(np.float32, nameinput) imagenet_mean 117.0 t_preprocessed tf.expand_dims(t_input - imagenet_mean, 0) tf.import_graph_def(graph_def, {input: t_preprocessed}) #找到卷积层 layers [op.name for op in graph.get_operations() if op.type Conv2D and import/ in op.name] #输出卷积层 print(Number of layers, len(layers)) #输出mixed4d_3x3_bottleneck_pre_relu的形状 name mixed4d_3x3_bottleneck_pre_relu print(shape of %s: %s % (name, str(graph.get_tensor_by_name(import/ name :0).get_shape()))) #t_input:设置一个占位符在后面程序把图像数据传递给t_input #tf.expand_dims:在原始height,width,channel输入前增加一堆batch,height,width,channel #减去均值在训练时Inception模型已经做了减去均值的预处理此处保持同样预处理才能输入一致 #导入模型tf.import_graph_def(graph_def, {input: t_preprocessed})运行结果:生成原始的Deep Dream图像# coding: utf-8 from __future__ import print_function import os from io import BytesIO import numpy as np from functools import partial import PIL.Image from PIL import Image import tensorflow.compat.v1 as tf tf.disable_v2_behavior() graph tf.Graph() model_fn tensorflow_inception_graph.pb sess tf.InteractiveSession(graphgraph) with tf.gfile.GFile(model_fn, rb) as f: graph_def tf.GraphDef() graph_def.ParseFromString(f.read()) t_input tf.placeholder(np.float32, nameinput) imagenet_mean 117.0 t_preprocessed tf.expand_dims(t_input - imagenet_mean, 0) tf.import_graph_def(graph_def, {input: t_preprocessed}) #定义一个保存图片的函数把numpy.ndarray保存成文件的形式 def savearray(img_array, img_name): # 用 PIL 替代 scipy.misc.toimage img Image.fromarray(np.uint8(img_array)) img.save(img_name) print(img saved: %s % img_name) def render_naive(t_obj, img0, iter_n20, step1.0): # t_score是优化目标。它是t_obj的平均值 # 结合调用处看实际上就是layer_output[:, :, :, channel]的平均值 t_score tf.reduce_mean(t_obj) # 计算t_score对t_input的梯度 t_grad tf.gradients(t_score, t_input)[0] # 创建新图 img img0.copy() for i in range(iter_n): # 在sess中计算梯度以及当前的score g, score sess.run([t_grad, t_score], {t_input: img}) # 对img应用梯度。step可以看做学习率 g / g.std() 1e-8 img g * step print(score(mean)%f % (score)) # 保存图片 savearray(img, naive.jpg) # 定义卷积层、通道数并取出对应的tensor name mixed4d_3x3_bottleneck_pre_relu channel 139 layer_output graph.get_tensor_by_name(import/%s:0 % name) # 定义原始的图像噪声 img_noise np.random.uniform(size(224, 224, 3)) 100.0 # 调用render_naive函数渲染 render_naive(layer_output[:, :, :, channel], img_noise, iter_n20) #t_obj卷积层某个通道的值t_score是t_obj的平均值运行结果生成更大尺寸的Deep Dream图像不是对整张图片做优化把图片分成多个部分每次对一个部分做优化达到优化时只消耗固定大小的内存。# coding:utf-8 from __future__ import print_function import os from io import BytesIO import numpy as np from functools import partial import PIL.Image from PIL import Image import tensorflow.compat.v1 as tf tf.disable_v2_behavior() graph tf.Graph() model_fn tensorflow_inception_graph.pb sess tf.InteractiveSession(graphgraph) with tf.gfile.GFile(model_fn, rb) as f: graph_def tf.GraphDef() graph_def.ParseFromString(f.read()) t_input tf.placeholder(np.float32, nameinput) imagenet_mean 117.0 t_preprocessed tf.expand_dims(t_input - imagenet_mean, 0) tf.import_graph_def(graph_def, {input: t_preprocessed}) def savearray(img_array, img_name): img Image.fromarray(np.uint8(np.clip(img_array, 0, 255))) img.save(img_name) print(img saved: %s % img_name) #放大rario倍先确定原先像素的范围计算img的最值用scipy.misc.imresize后再将像素缩放回去 def resize_ratio(img, ratio): min_val img.min() max_val img.max() img (img - min_val) / (max_val - min_val) * 255 # 用 PIL 替代 scipy.misc.imresize h, w img.shape[:2] new_h, new_w int(h * ratio), int(w * ratio) img Image.fromarray(np.uint8(np.clip(img, 0, 255))) img img.resize((new_w, new_h), Image.LANCZOS) img np.float32(img) img img / 255 * (max_val - min_val) min_val return img #对任意大小的图像计算梯度 def calc_grad_tiled(img, t_grad, tile_size512): sz tile_size h, w img.shape[:2] sx, sy np.random.randint(sz, size2) #整体移动在图像边缘的像素就会被移动到图像中间避免边缘效应 img_shift np.roll(np.roll(img, sx, 1), sy, 0) grad np.zeros_like(img) for y in range(0, max(h - sz // 2, sz), sz): for x in range(0, max(w - sz // 2, sz), sz): sub img_shift[y:y sz, x:x sz] g sess.run(t_grad, {t_input: sub}) grad[y:y sz, x:x sz] g return np.roll(np.roll(grad, -sx, 1), -sy, 0) #生成最大尺寸图像的函数先生成小尺寸的图像调用resize_ratio放大octave_scale倍再进行计算进行octave_n-1次 #octave_n决定图像大小 def render_multiscale(t_obj, img0, iter_n10, step1.0, octave_n3, octave_scale1.4): t_score tf.reduce_mean(t_obj) t_grad tf.gradients(t_score, t_input)[0] img img0.copy() for octave in range(octave_n): if octave 0: img resize_ratio(img, octave_scale) for i in range(iter_n): g calc_grad_tiled(img, t_grad) g / g.std() 1e-8 img g * step print(., end ) savearray(img, multiscale.jpg) if __name__ __main__: name mixed4d_3x3_bottleneck_pre_relu channel 139 img_noise np.random.uniform(size(224, 224, 3)) 100.0 layer_output graph.get_tensor_by_name(import/%s:0 % name) render_multiscale(layer_output[:, :, :, channel], img_noise, iter_n20)运行结果生成更高质量的Deep Dream图像高频图像灰度、颜色、明度变化比较强烈低频图像变化不大的地方风格、色块图片更柔和将梯度做分解拉普拉斯金字塔分为高频和低频梯度再放大低频梯度图片更柔和# coding:utf-8 from __future__ import print_function import os from io import BytesIO import numpy as np from functools import partial import PIL.Image from PIL import Image import tensorflow.compat.v1 as tf tf.disable_v2_behavior() graph tf.Graph() model_fn tensorflow_inception_graph.pb sess tf.InteractiveSession(graphgraph) with tf.gfile.GFile(model_fn, rb) as f: graph_def tf.GraphDef() graph_def.ParseFromString(f.read()) t_input tf.placeholder(np.float32, nameinput) imagenet_mean 117.0 t_preprocessed tf.expand_dims(t_input - imagenet_mean, 0) tf.import_graph_def(graph_def, {input: t_preprocessed}) def savearray(img_array, img_name): img Image.fromarray(np.uint8(np.clip(img_array, 0, 255))) img.save(img_name) print(img saved: %s % img_name) def resize_ratio(img, ratio): min_val img.min() max_val img.max() img (img - min_val) / (max_val - min_val) * 255 h, w img.shape[:2] new_h, new_w int(h * ratio), int(w * ratio) img Image.fromarray(np.uint8(np.clip(img, 0, 255))) img img.resize((new_w, new_h), Image.LANCZOS) img np.float32(img) img img / 255 * (max_val - min_val) min_val return img def calc_grad_tiled(img, t_grad, tile_size512): sz tile_size h, w img.shape[:2] sx, sy np.random.randint(sz, size2) img_shift np.roll(np.roll(img, sx, 1), sy, 0) grad np.zeros_like(img) for y in range(0, max(h - sz // 2, sz), sz): for x in range(0, max(w - sz // 2, sz), sz): sub img_shift[y:y sz, x:x sz] g sess.run(t_grad, {t_input: sub}) grad[y:y sz, x:x sz] g return np.roll(np.roll(grad, -sx, 1), -sy, 0) k np.float32([1, 4, 6, 4, 1]) k np.outer(k, k) k5x5 k[:, :, None, None] / k.sum() * np.eye(3, dtypenp.float32) #将图像分为高频和低频成分 def lap_split(img): with tf.name_scope(split): lo tf.nn.conv2d(img, k5x5, [1, 2, 2, 1], SAME) lo2 tf.nn.conv2d_transpose(lo, k5x5 * 4, tf.shape(img), [1, 2, 2, 1]) hi img - lo2 return lo, hi #分为n层金字塔 def lap_split_n(img, n): levels [] for i in range(n): #分为高低频高频保存到levels中低频继续分解 img, hi lap_split(img) levels.append(hi) levels.append(img) return levels[::-1] #将拉普拉斯金字塔还原到原始图像 def lap_merge(levels): img levels[0] for hi in levels[1:]: with tf.name_scope(merge): img tf.nn.conv2d_transpose(img, k5x5 * 4, tf.shape(hi), [1, 2, 2, 1]) hi return img #标准化 def normalize_std(img, eps1e-10): with tf.name_scope(normalize): std tf.sqrt(tf.reduce_mean(tf.square(img))) return img / tf.maximum(std, eps) #拉普拉斯金字塔标准化 def lap_normalize(img, scale_n4): img tf.expand_dims(img, 0) tlevels lap_split_n(img, scale_n) tlevels list(map(normalize_std, tlevels)) out lap_merge(tlevels) return out[0, :, :, :] #将一个对Tensor定义的函数转换成一个正常的对numpy.ndarray定义的函数 def tffunc(*argtypes): placeholders list(map(tf.placeholder, argtypes)) def wrap(f): out f(*placeholders) def wrapper(*args, **kw): return out.eval(dict(zip(placeholders, args)), sessionkw.get(session)) return wrapper return wrap #生成图像 def render_lapnorm(t_obj, img0, iter_n10, step1.0, octave_n3, octave_scale1.4, lap_n4): t_score tf.reduce_mean(t_obj) t_grad tf.gradients(t_score, t_input)[0] lap_norm_func tffunc(np.float32)(partial(lap_normalize, scale_nlap_n)) img img0.copy() for octave in range(octave_n): if octave 0: img resize_ratio(img, octave_scale) for i in range(iter_n): g calc_grad_tiled(img, t_grad) g lap_norm_func(g) img g * step print(., end ) savearray(img, lapnorm.jpg) if __name__ __main__: name mixed4d_3x3_bottleneck_pre_relu channel 139 img_noise np.random.uniform(size(224, 224, 3)) 100.0 layer_output graph.get_tensor_by_name(import/%s:0 % name) render_lapnorm(layer_output[:, :, :, channel], img_noise, iter_n20)运行结束最终的Deep Dream 模型添加背景# coding:utf-8 from __future__ import print_function import os from io import BytesIO import numpy as np from functools import partial import PIL.Image from PIL import Image import tensorflow.compat.v1 as tf tf.disable_v2_behavior() graph tf.Graph() model_fn tensorflow_inception_graph.pb sess tf.InteractiveSession(graphgraph) with tf.gfile.GFile(model_fn, rb) as f: graph_def tf.GraphDef() graph_def.ParseFromString(f.read()) t_input tf.placeholder(np.float32, nameinput) imagenet_mean 117.0 t_preprocessed tf.expand_dims(t_input - imagenet_mean, 0) tf.import_graph_def(graph_def, {input: t_preprocessed}) def savearray(img_array, img_name): img Image.fromarray(np.uint8(np.clip(img_array, 0, 255))) img.save(img_name) print(img saved: %s % img_name) def visstd(a, s0.1): return (a - a.mean()) / max(a.std(), 1e-4) * s 0.5 def resize_ratio(img, ratio): min_val img.min() max_val img.max() img (img - min_val) / (max_val - min_val) * 255 h, w img.shape[:2] new_h, new_w int(h * ratio), int(w * ratio) img Image.fromarray(np.uint8(np.clip(img, 0, 255))) img img.resize((new_w, new_h), Image.LANCZOS) img np.float32(img) img img / 255 * (max_val - min_val) min_val return img #缩小原图像 def resize(img, hw): min_val img.min() max_val img.max() img (img - min_val) / (max_val - min_val) * 255 img Image.fromarray(np.uint8(np.clip(img, 0, 255))) img img.resize((hw[1], hw[0]), Image.LANCZOS) img np.float32(img) img img / 255 * (max_val - min_val) min_val return img def calc_grad_tiled(img, t_grad, tile_size512): sz tile_size h, w img.shape[:2] sx, sy np.random.randint(sz, size2) img_shift np.roll(np.roll(img, sx, 1), sy, 0) grad np.zeros_like(img) for y in range(0, max(h - sz // 2, sz), sz): for x in range(0, max(w - sz // 2, sz), sz): sub img_shift[y:y sz, x:x sz] g sess.run(t_grad, {t_input: sub}) grad[y:y sz, x:x sz] g return np.roll(np.roll(grad, -sx, 1), -sy, 0) def tffunc(*argtypes): placeholders list(map(tf.placeholder, argtypes)) def wrap(f): out f(*placeholders) def wrapper(*args, **kw): return out.eval(dict(zip(placeholders, args)), sessionkw.get(session)) return wrapper return wrap #为了保证图像的质量进行高低频分解直接缩小原图像 def render_deepdream(t_obj, img0, iter_n10, step1.5, octave_n4, octave_scale1.4): t_score tf.reduce_mean(t_obj) t_grad tf.gradients(t_score, t_input)[0] img img0 octaves [] for i in range(octave_n - 1): hw img.shape[:2] lo resize(img, np.int32(np.float32(hw) / octave_scale)) hi img - resize(lo, hw) img lo octaves.append(hi) for octave in range(octave_n): if octave 0: hi octaves[-octave] img resize(img, hi.shape[:2]) hi for i in range(iter_n): g calc_grad_tiled(img, t_grad) img g * (step / (np.abs(g).mean() 1e-7)) print(., end ) img img.clip(0, 255) savearray(img, deepdream.jpg) if __name__ __main__: img0 PIL.Image.open(test.jpg) img0 np.float32(img0) name mixed4d_3x3_bottleneck_pre_relu channel 139 layer_output graph.get_tensor_by_name(import/%s:0 % name) render_deepdream(layer_output[:, :, :, channel], img0)运行结果带有动物的DeepDream图片# coding:utf-8 from __future__ import print_function import numpy as np import PIL.Image from PIL import Image import tensorflow.compat.v1 as tf tf.disable_v2_behavior() graph tf.Graph() model_fn tensorflow_inception_graph.pb sess tf.InteractiveSession(graphgraph) with tf.gfile.GFile(model_fn, rb) as f: graph_def tf.GraphDef() graph_def.ParseFromString(f.read()) t_input tf.placeholder(np.float32, nameinput) imagenet_mean 117.0 t_preprocessed tf.expand_dims(t_input - imagenet_mean, 0) tf.import_graph_def(graph_def, {input: t_preprocessed}) def savearray(img_array, img_name): img Image.fromarray(np.uint8(np.clip(img_array, 0, 255))) img.save(img_name) print(img saved: %s % img_name) def resize(img, hw): min_val img.min() max_val img.max() img (img - min_val) / (max_val - min_val) * 255 img Image.fromarray(np.uint8(np.clip(img, 0, 255))) img img.resize((hw[1], hw[0]), Image.LANCZOS) img np.float32(img) img img / 255 * (max_val - min_val) min_val return img def calc_grad_tiled(img, t_grad, tile_size512): sz tile_size h, w img.shape[:2] sx, sy np.random.randint(sz, size2) img_shift np.roll(np.roll(img, sx, 1), sy, 0) grad np.zeros_like(img) for y in range(0, max(h - sz // 2, sz), sz): for x in range(0, max(w - sz // 2, sz), sz): sub img_shift[y:y sz, x:x sz] g sess.run(t_grad, {t_input: sub}) grad[y:y sz, x:x sz] g return np.roll(np.roll(grad, -sx, 1), -sy, 0) def render_deepdream(t_obj, img0, iter_n10, step1.5, octave_n4, octave_scale1.4): t_score tf.reduce_mean(t_obj) t_grad tf.gradients(t_score, t_input)[0] img img0 octaves [] for i in range(octave_n - 1): hw img.shape[:2] lo resize(img, np.int32(np.float32(hw) / octave_scale)) hi img - resize(lo, hw) img lo octaves.append(hi) for octave in range(octave_n): if octave 0: hi octaves[-octave] img resize(img, hi.shape[:2]) hi for i in range(iter_n): g calc_grad_tiled(img, t_grad) img g * (step / (np.abs(g).mean() 1e-7)) print(., end ) print() img img.clip(0, 255) savearray(img, deepdream_animals.jpg) if __name__ __main__: # 用一张天空或风景照作为输入动物图案更容易显现 img0 PIL.Image.open(test.jpg) img0 np.float32(img0) # 关键使用 tf.square 放大 mixed4c 整个层的输出 # mixed4c 包含大量动物特征眼睛、羽毛、爪子等 name mixed4c layer_output graph.get_tensor_by_name(import/%s:0 % name) render_deepdream(tf.square(layer_output), img0, iter_n20)