ResNet_DeepUD_triplet_TwStreaming.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GO!!\n",
      "Preparing the training & validation data...\n",
      "WARNING:tensorflow:From <ipython-input-1-bc603345c58f>:206: slice_input_producer (from tensorflow.python.training.input) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Queue-based input pipelines have been replaced by `tf.data`. Use `tf.data.Dataset.from_tensor_slices(tuple(tensor_list)).shuffle(tf.shape(input_tensor, out_type=tf.int64)[0]).repeat(num_epochs)`. If `shuffle=False`, omit the `.shuffle(...)`.\n",
      "WARNING:tensorflow:From C:\\Users\\user\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\training\\input.py:372: range_input_producer (from tensorflow.python.training.input) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Queue-based input pipelines have been replaced by `tf.data`. Use `tf.data.Dataset.range(limit).shuffle(limit).repeat(num_epochs)`. If `shuffle=False`, omit the `.shuffle(...)`.\n",
      "WARNING:tensorflow:From C:\\Users\\user\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\training\\input.py:318: input_producer (from tensorflow.python.training.input) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Queue-based input pipelines have been replaced by `tf.data`. Use `tf.data.Dataset.from_tensor_slices(input_tensor).shuffle(tf.shape(input_tensor, out_type=tf.int64)[0]).repeat(num_epochs)`. If `shuffle=False`, omit the `.shuffle(...)`.\n",
      "WARNING:tensorflow:From C:\\Users\\user\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\training\\input.py:188: limit_epochs (from tensorflow.python.training.input) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Queue-based input pipelines have been replaced by `tf.data`. Use `tf.data.Dataset.from_tensors(tensor).repeat(num_epochs)`.\n",
      "WARNING:tensorflow:From C:\\Users\\user\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\training\\input.py:197: QueueRunner.__init__ (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "To construct input pipelines, use the `tf.data` module.\n",
      "WARNING:tensorflow:From C:\\Users\\user\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\training\\input.py:197: add_queue_runner (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "To construct input pipelines, use the `tf.data` module.\n",
      "WARNING:tensorflow:From <ipython-input-1-bc603345c58f>:221: batch (from tensorflow.python.training.input) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Queue-based input pipelines have been replaced by `tf.data`. Use `tf.data.Dataset.batch(batch_size)` (or `padded_batch(...)` if `dynamic_pad=True`).\n",
      "WARNING:tensorflow:From <ipython-input-1-bc603345c58f>:223: start_queue_runners (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "To construct input pipelines, use the `tf.data` module.\n",
      "Found 385198 training images, and 10000 validation images...\n",
      "Preparing the training model with learning rate = 0.00010...\n",
      "WARNING:tensorflow:From C:\\Users\\user\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\framework\\function.py:987: calling Graph.create_op (from tensorflow.python.framework.ops) with compute_shapes is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Shapes are always computed; don't use the compute_shapes as it has no effect.\n",
      "[L-1] Build 0th residual block 0 with 64 channels\n",
      "[L-2] Build 0th connection layer 1 from 64 to 96 channels\n",
      "[L-3] Build 1th residual block 0 with 96 channels\n",
      "[L-4] Build 1th residual block 1 with 96 channels\n",
      "[L-5] Build 1th residual block 2 with 96 channels\n",
      "[L-6] Build 1th connection layer 3 from 96 to 128 channels\n",
      "[L-7] Build 2th residual block 0 with 128 channels\n",
      "[L-8] Build 2th residual block 1 with 128 channels\n",
      "[L-9] Build 2th residual block 2 with 128 channels\n",
      "[L-10] Build 0th residual block 0 with 64 channels\n",
      "[L-11] Build 0th connection layer 1 from 64 to 96 channels\n",
      "[L-12] Build 1th residual block 0 with 96 channels\n",
      "[L-13] Build 1th residual block 1 with 96 channels\n",
      "[L-14] Build 1th residual block 2 with 96 channels\n",
      "[L-15] Build 1th connection layer 3 from 96 to 128 channels\n",
      "[L-16] Build 2th residual block 0 with 128 channels\n",
      "[L-17] Build 2th residual block 1 with 128 channels\n",
      "[L-18] Build 2th residual block 2 with 128 channels\n",
      "Layer33's shape [128, 3, 3, 128]\n",
      "Layer55's shape [128, 3, 3, 128]\n",
      "[L-1] Build 0th residual block 0 with 64 channels\n",
      "[L-2] Build 0th connection layer 1 from 64 to 96 channels\n",
      "[L-3] Build 1th residual block 0 with 96 channels\n",
      "[L-4] Build 1th residual block 1 with 96 channels\n",
      "[L-5] Build 1th residual block 2 with 96 channels\n",
      "[L-6] Build 1th connection layer 3 from 96 to 128 channels\n",
      "[L-7] Build 2th residual block 0 with 128 channels\n",
      "[L-8] Build 2th residual block 1 with 128 channels\n",
      "[L-9] Build 2th residual block 2 with 128 channels\n",
      "[L-10] Build 0th residual block 0 with 64 channels\n",
      "[L-11] Build 0th connection layer 1 from 64 to 96 channels\n",
      "[L-12] Build 1th residual block 0 with 96 channels\n",
      "[L-13] Build 1th residual block 1 with 96 channels\n",
      "[L-14] Build 1th residual block 2 with 96 channels\n",
      "[L-15] Build 1th connection layer 3 from 96 to 128 channels\n",
      "[L-16] Build 2th residual block 0 with 128 channels\n",
      "[L-17] Build 2th residual block 1 with 128 channels\n",
      "[L-18] Build 2th residual block 2 with 128 channels\n",
      "Layer33's shape [1000, 3, 3, 128]\n",
      "Layer55's shape [1000, 3, 3, 128]\n",
      "We are going to train fake detector using ResNet based on triplet loss!!!\n",
      "Iter=10240/epoch=0, Loss=8.433966, Triplet loss=1.933536, Training Accuracy=0.476562, lr=0.000100\n",
      "Iter=20480/epoch=0, Loss=4.824346, Triplet loss=2.015718, Training Accuracy=0.546875, lr=0.000100\n",
      "Iter=30720/epoch=0, Loss=8.755816, Triplet loss=1.938000, Training Accuracy=0.445312, lr=0.000100\n",
      "Iter=40960/epoch=0, Loss=7.857846, Triplet loss=1.930153, Training Accuracy=0.429688, lr=0.000100\n",
      "Iter=51200/epoch=0, Loss=7.933546, Triplet loss=1.860873, Training Accuracy=0.460938, lr=0.000100\n",
      "Iter=61440/epoch=0, Loss=7.696203, Triplet loss=2.015472, Training Accuracy=0.500000, lr=0.000100\n",
      "Iter=71680/epoch=0, Loss=7.021667, Triplet loss=1.860662, Training Accuracy=0.507812, lr=0.000100\n",
      "Iter=81920/epoch=0, Loss=7.516994, Triplet loss=1.938049, Training Accuracy=0.500000, lr=0.000100\n",
      "Iter=92160/epoch=0, Loss=8.457277, Triplet loss=2.004295, Training Accuracy=0.500000, lr=0.000100\n",
      "Iter=102400/epoch=0, Loss=7.764872, Triplet loss=1.987822, Training Accuracy=0.500000, lr=0.000100\n",
      "Iter=112640/epoch=0, Loss=7.040625, Triplet loss=2.037445, Training Accuracy=0.500000, lr=0.000100\n",
      "Iter=122880/epoch=0, Loss=7.272042, Triplet loss=2.026616, Training Accuracy=0.546875, lr=0.000100\n",
      "Iter=133120/epoch=0, Loss=7.825847, Triplet loss=1.984484, Training Accuracy=0.523438, lr=0.000100\n",
      "Iter=143360/epoch=0, Loss=5.616179, Triplet loss=1.984159, Training Accuracy=0.554688, lr=0.000100\n",
      "Iter=153600/epoch=0, Loss=5.840203, Triplet loss=1.966795, Training Accuracy=0.554688, lr=0.000100\n",
      "Iter=163840/epoch=0, Loss=7.344795, Triplet loss=1.915903, Training Accuracy=0.476562, lr=0.000100\n",
      "Iter=174080/epoch=0, Loss=7.784627, Triplet loss=1.893810, Training Accuracy=0.500000, lr=0.000100\n",
      "Iter=184320/epoch=0, Loss=7.104695, Triplet loss=1.949055, Training Accuracy=0.554688, lr=0.000100\n",
      "Iter=194560/epoch=0, Loss=6.088765, Triplet loss=1.943530, Training Accuracy=0.562500, lr=0.000100\n",
      "Iter=204800/epoch=0, Loss=6.062476, Triplet loss=1.949049, Training Accuracy=0.531250, lr=0.000100\n",
      "Iter=204800/epoch=0, Validation Accuracy=0.483300, Precision=0.366400, Recall=0.045800\n",
      "Iter=215040/epoch=0, Loss=8.849595, Triplet loss=1.993243, Training Accuracy=0.453125, lr=0.000100\n",
      "Iter=225280/epoch=0, Loss=6.607213, Triplet loss=1.960154, Training Accuracy=0.515625, lr=0.000100\n",
      "Iter=235520/epoch=0, Loss=6.946209, Triplet loss=1.993243, Training Accuracy=0.484375, lr=0.000100\n",
      "Iter=245760/epoch=0, Loss=8.544353, Triplet loss=1.830161, Training Accuracy=0.476562, lr=0.000100\n",
      "Iter=256000/epoch=0, Loss=7.020218, Triplet loss=2.022364, Training Accuracy=0.523438, lr=0.000100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter=266240/epoch=0, Loss=6.983248, Triplet loss=2.026384, Training Accuracy=0.515625, lr=0.000100\n",
      "Iter=276480/epoch=0, Loss=8.479557, Triplet loss=1.960097, Training Accuracy=0.453125, lr=0.000100\n",
      "Iter=286720/epoch=0, Loss=8.162851, Triplet loss=2.037437, Training Accuracy=0.398438, lr=0.000100\n",
      "Iter=296960/epoch=0, Loss=7.105334, Triplet loss=2.026388, Training Accuracy=0.507812, lr=0.000100\n",
      "Iter=307200/epoch=0, Loss=6.136365, Triplet loss=1.993770, Training Accuracy=0.585938, lr=0.000100\n",
      "Iter=317440/epoch=0, Loss=6.344234, Triplet loss=2.038882, Training Accuracy=0.554688, lr=0.000100\n",
      "Iter=327680/epoch=0, Loss=9.376978, Triplet loss=2.004310, Training Accuracy=0.484375, lr=0.000100\n",
      "Iter=337920/epoch=0, Loss=8.433361, Triplet loss=2.027363, Training Accuracy=0.437500, lr=0.000100\n",
      "Iter=348160/epoch=0, Loss=7.017676, Triplet loss=2.059534, Training Accuracy=0.570312, lr=0.000100\n",
      "Iter=358400/epoch=0, Loss=7.517006, Triplet loss=1.938099, Training Accuracy=0.523438, lr=0.000100\n",
      "Iter=368640/epoch=0, Loss=6.351534, Triplet loss=1.982194, Training Accuracy=0.500000, lr=0.000100\n",
      "Iter=378880/epoch=0, Loss=6.561686, Triplet loss=1.971158, Training Accuracy=0.570312, lr=0.000100\n",
      "Iter=389120/epoch=1, Loss=8.398564, Triplet loss=2.059546, Training Accuracy=0.515625, lr=0.000100\n",
      "Iter=399360/epoch=1, Loss=9.207681, Triplet loss=2.081631, Training Accuracy=0.468750, lr=0.000100\n",
      "Iter=409600/epoch=1, Loss=7.766735, Triplet loss=1.882757, Training Accuracy=0.445312, lr=0.000100\n",
      "Iter=409600/epoch=1, Validation Accuracy=0.488300, Precision=0.403306, Recall=0.048800\n",
      "Iter=419840/epoch=1, Loss=7.909161, Triplet loss=1.993243, Training Accuracy=0.523438, lr=0.000100\n",
      "Iter=430080/epoch=1, Loss=8.723133, Triplet loss=1.898538, Training Accuracy=0.445312, lr=0.000100\n",
      "Iter=440320/epoch=1, Loss=7.424674, Triplet loss=1.790397, Training Accuracy=0.484375, lr=0.000100\n",
      "Iter=450560/epoch=1, Loss=7.790693, Triplet loss=1.971297, Training Accuracy=0.515625, lr=0.000100\n",
      "Iter=460800/epoch=1, Loss=7.927724, Triplet loss=1.958654, Training Accuracy=0.460938, lr=0.000100\n",
      "Iter=471040/epoch=1, Loss=6.567026, Triplet loss=1.952226, Training Accuracy=0.554688, lr=0.000100\n",
      "Iter=481280/epoch=1, Loss=6.828516, Triplet loss=1.971974, Training Accuracy=0.546875, lr=0.000100\n",
      "Iter=491520/epoch=1, Loss=7.947622, Triplet loss=1.960632, Training Accuracy=0.500000, lr=0.000100\n",
      "Iter=501760/epoch=1, Loss=7.891742, Triplet loss=1.898378, Training Accuracy=0.546875, lr=0.000100\n",
      "Iter=512000/epoch=1, Loss=7.344414, Triplet loss=1.953372, Training Accuracy=0.500000, lr=0.000100\n",
      "Iter=522240/epoch=1, Loss=8.364200, Triplet loss=1.954633, Training Accuracy=0.484375, lr=0.000100\n",
      "Iter=532480/epoch=1, Loss=6.195478, Triplet loss=2.015355, Training Accuracy=0.578125, lr=0.000100\n",
      "Iter=542720/epoch=1, Loss=7.800344, Triplet loss=1.982521, Training Accuracy=0.507812, lr=0.000100\n",
      "Iter=552960/epoch=1, Loss=7.531824, Triplet loss=1.950578, Training Accuracy=0.500000, lr=0.000100\n",
      "Iter=563200/epoch=1, Loss=7.072955, Triplet loss=1.960097, Training Accuracy=0.546875, lr=0.000100\n",
      "Iter=573440/epoch=1, Loss=9.268734, Triplet loss=2.037444, Training Accuracy=0.445312, lr=0.000100\n",
      "Iter=583680/epoch=1, Loss=6.877580, Triplet loss=1.975397, Training Accuracy=0.523438, lr=0.000100\n",
      "Iter=593920/epoch=1, Loss=8.022186, Triplet loss=1.993396, Training Accuracy=0.484375, lr=0.000100\n",
      "Iter=604160/epoch=1, Loss=7.920438, Triplet loss=1.927135, Training Accuracy=0.484375, lr=0.000100\n",
      "Iter=614400/epoch=1, Loss=8.186878, Triplet loss=2.011192, Training Accuracy=0.507812, lr=0.000100\n",
      "Iter=614400/epoch=1, Validation Accuracy=0.483700, Precision=0.425161, Recall=0.092600\n",
      "Iter=624640/epoch=1, Loss=6.688638, Triplet loss=1.994643, Training Accuracy=0.539062, lr=0.000100\n",
      "Iter=634880/epoch=1, Loss=8.614460, Triplet loss=2.034406, Training Accuracy=0.492188, lr=0.000100\n",
      "Iter=645120/epoch=1, Loss=8.808632, Triplet loss=1.904867, Training Accuracy=0.484375, lr=0.000100\n",
      "Iter=655360/epoch=1, Loss=8.908401, Triplet loss=1.973986, Training Accuracy=0.609375, lr=0.000100\n",
      "Iter=665600/epoch=1, Loss=8.432974, Triplet loss=1.993246, Training Accuracy=0.484375, lr=0.000100\n",
      "Iter=675840/epoch=1, Loss=7.438204, Triplet loss=2.004294, Training Accuracy=0.546875, lr=0.000100\n",
      "Iter=686080/epoch=1, Loss=6.832325, Triplet loss=2.037469, Training Accuracy=0.468750, lr=0.000100\n",
      "Iter=696320/epoch=1, Loss=8.516408, Triplet loss=2.060307, Training Accuracy=0.539062, lr=0.000100\n",
      "Iter=706560/epoch=1, Loss=8.904750, Triplet loss=2.059357, Training Accuracy=0.507812, lr=0.000100\n",
      "Iter=716800/epoch=1, Loss=9.782690, Triplet loss=2.055923, Training Accuracy=0.414062, lr=0.000100\n",
      "Iter=727040/epoch=1, Loss=7.333721, Triplet loss=2.101764, Training Accuracy=0.539062, lr=0.000100\n",
      "Iter=737280/epoch=1, Loss=9.037836, Triplet loss=1.982194, Training Accuracy=0.468750, lr=0.000100\n",
      "Iter=747520/epoch=1, Loss=6.463580, Triplet loss=1.971148, Training Accuracy=0.539062, lr=0.000100\n",
      "Iter=757760/epoch=1, Loss=7.026705, Triplet loss=2.059772, Training Accuracy=0.562500, lr=0.000100\n",
      "Iter=768000/epoch=1, Loss=7.938468, Triplet loss=2.060138, Training Accuracy=0.515625, lr=0.000100\n",
      "Iter=778240/epoch=2, Loss=10.045883, Triplet loss=2.037438, Training Accuracy=0.429688, lr=0.000100\n",
      "Iter=788480/epoch=2, Loss=6.377109, Triplet loss=2.092684, Training Accuracy=0.500000, lr=0.000100\n",
      "Iter=798720/epoch=2, Loss=7.042637, Triplet loss=2.030530, Training Accuracy=0.546875, lr=0.000100\n",
      "Iter=808960/epoch=2, Loss=7.743351, Triplet loss=2.103728, Training Accuracy=0.546875, lr=0.000100\n",
      "Iter=819200/epoch=2, Loss=8.863624, Triplet loss=2.059539, Training Accuracy=0.460938, lr=0.000100\n",
      "Iter=819200/epoch=2, Validation Accuracy=0.479200, Precision=0.398438, Recall=0.081600\n",
      "Iter=829440/epoch=2, Loss=7.345371, Triplet loss=2.106476, Training Accuracy=0.523438, lr=0.000100\n",
      "Iter=839680/epoch=2, Loss=8.712448, Triplet loss=2.027956, Training Accuracy=0.468750, lr=0.000100\n",
      "Iter=849920/epoch=2, Loss=7.423706, Triplet loss=2.070588, Training Accuracy=0.523438, lr=0.000100\n",
      "Iter=860160/epoch=2, Loss=7.295979, Triplet loss=2.074758, Training Accuracy=0.468750, lr=0.000100\n",
      "Iter=870400/epoch=2, Loss=7.646593, Triplet loss=2.081631, Training Accuracy=0.507812, lr=0.000100\n",
      "Iter=880640/epoch=2, Loss=9.295105, Triplet loss=2.070572, Training Accuracy=0.492188, lr=0.000100\n",
      "Iter=890880/epoch=2, Loss=8.177097, Triplet loss=2.048486, Training Accuracy=0.515625, lr=0.000100\n",
      "Iter=901120/epoch=2, Loss=8.081587, Triplet loss=2.059566, Training Accuracy=0.515625, lr=0.000100\n",
      "Iter=911360/epoch=2, Loss=7.390896, Triplet loss=2.037437, Training Accuracy=0.531250, lr=0.000100\n",
      "Iter=921600/epoch=2, Loss=8.062107, Triplet loss=2.059535, Training Accuracy=0.523438, lr=0.000100\n",
      "Iter=931840/epoch=2, Loss=7.983008, Triplet loss=2.072404, Training Accuracy=0.531250, lr=0.000100\n",
      "Iter=942080/epoch=2, Loss=8.320904, Triplet loss=2.092680, Training Accuracy=0.492188, lr=0.000100\n",
      "Iter=952320/epoch=2, Loss=6.527860, Triplet loss=2.038084, Training Accuracy=0.585938, lr=0.000100\n",
      "Iter=962560/epoch=2, Loss=5.106957, Triplet loss=1.896996, Training Accuracy=0.632812, lr=0.000100\n",
      "Iter=972800/epoch=2, Loss=5.651457, Triplet loss=1.993243, Training Accuracy=0.617188, lr=0.000100\n",
      "Iter=983040/epoch=2, Loss=10.153805, Triplet loss=2.026388, Training Accuracy=0.445312, lr=0.000100\n",
      "Iter=993280/epoch=2, Loss=7.130306, Triplet loss=2.026243, Training Accuracy=0.468750, lr=0.000100\n",
      "Iter=1003520/epoch=2, Loss=7.668457, Triplet loss=2.026244, Training Accuracy=0.476562, lr=0.000100\n",
      "Iter=1013760/epoch=2, Loss=9.712446, Triplet loss=2.037437, Training Accuracy=0.445312, lr=0.000100\n",
      "Iter=1024000/epoch=2, Loss=7.812721, Triplet loss=2.059676, Training Accuracy=0.515625, lr=0.000100\n",
      "Iter=1024000/epoch=2, Validation Accuracy=0.483900, Precision=0.429075, Recall=0.097400\n",
      "Iter=1034240/epoch=2, Loss=8.019257, Triplet loss=2.013915, Training Accuracy=0.539062, lr=0.000100\n",
      "Iter=1044480/epoch=2, Loss=6.019717, Triplet loss=2.015406, Training Accuracy=0.554688, lr=0.000100\n",
      "Iter=1054720/epoch=2, Loss=8.315444, Triplet loss=2.048494, Training Accuracy=0.523438, lr=0.000100\n",
      "Iter=1064960/epoch=2, Loss=6.926987, Triplet loss=2.048486, Training Accuracy=0.562500, lr=0.000100\n",
      "Iter=1075200/epoch=2, Loss=8.524615, Triplet loss=2.004548, Training Accuracy=0.460938, lr=0.000100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter=1085440/epoch=2, Loss=6.513272, Triplet loss=2.026388, Training Accuracy=0.609375, lr=0.000100\n",
      "Iter=1095680/epoch=2, Loss=8.915289, Triplet loss=2.048486, Training Accuracy=0.476562, lr=0.000100\n",
      "Iter=1105920/epoch=2, Loss=6.652933, Triplet loss=1.949359, Training Accuracy=0.539062, lr=0.000100\n",
      "Iter=1116160/epoch=2, Loss=6.597641, Triplet loss=1.991145, Training Accuracy=0.585938, lr=0.000100\n",
      "Iter=1126400/epoch=2, Loss=7.322388, Triplet loss=1.983456, Training Accuracy=0.515625, lr=0.000100\n",
      "Iter=1136640/epoch=2, Loss=8.379923, Triplet loss=2.059534, Training Accuracy=0.492188, lr=0.000100\n",
      "Iter=1146880/epoch=2, Loss=7.541011, Triplet loss=2.015340, Training Accuracy=0.484375, lr=0.000100\n",
      "Iter=1157120/epoch=3, Loss=7.227589, Triplet loss=1.960097, Training Accuracy=0.484375, lr=0.000010\n",
      "Iter=1167360/epoch=3, Loss=8.630645, Triplet loss=2.092680, Training Accuracy=0.445312, lr=0.000010\n",
      "Iter=1177600/epoch=3, Loss=9.036190, Triplet loss=1.971146, Training Accuracy=0.546875, lr=0.000010\n",
      "Iter=1187840/epoch=3, Loss=7.703921, Triplet loss=1.993243, Training Accuracy=0.492188, lr=0.000010\n",
      "Iter=1198080/epoch=3, Loss=8.177653, Triplet loss=1.993243, Training Accuracy=0.468750, lr=0.000010\n",
      "Iter=1208320/epoch=3, Loss=7.509888, Triplet loss=2.090770, Training Accuracy=0.515625, lr=0.000010\n",
      "Iter=1218560/epoch=3, Loss=7.552525, Triplet loss=2.065742, Training Accuracy=0.515625, lr=0.000010\n",
      "Iter=1228800/epoch=3, Loss=7.444085, Triplet loss=2.048486, Training Accuracy=0.453125, lr=0.000010\n",
      "Iter=1228800/epoch=3, Validation Accuracy=0.478000, Precision=0.435823, Recall=0.149400\n",
      "Iter=1239040/epoch=3, Loss=10.277419, Triplet loss=2.081631, Training Accuracy=0.390625, lr=0.000010\n",
      "Iter=1249280/epoch=3, Loss=7.746727, Triplet loss=2.053622, Training Accuracy=0.484375, lr=0.000010\n",
      "Iter=1259520/epoch=3, Loss=8.432701, Triplet loss=1.982194, Training Accuracy=0.460938, lr=0.000010\n",
      "Iter=1269760/epoch=3, Loss=8.101978, Triplet loss=2.037437, Training Accuracy=0.476562, lr=0.000010\n",
      "Iter=1280000/epoch=3, Loss=6.655755, Triplet loss=1.993780, Training Accuracy=0.492188, lr=0.000010\n",
      "Iter=1290240/epoch=3, Loss=7.921432, Triplet loss=1.993243, Training Accuracy=0.484375, lr=0.000010\n",
      "Iter=1300480/epoch=3, Loss=7.370262, Triplet loss=2.026388, Training Accuracy=0.476562, lr=0.000010\n",
      "Iter=1310720/epoch=3, Loss=7.635904, Triplet loss=2.015341, Training Accuracy=0.476562, lr=0.000010\n",
      "Iter=1320960/epoch=3, Loss=7.047871, Triplet loss=1.949049, Training Accuracy=0.554688, lr=0.000010\n",
      "Iter=1331200/epoch=3, Loss=7.778413, Triplet loss=2.125825, Training Accuracy=0.492188, lr=0.000010\n",
      "Iter=1341440/epoch=3, Loss=9.062881, Triplet loss=2.015340, Training Accuracy=0.398438, lr=0.000010\n",
      "Iter=1351680/epoch=3, Loss=7.679291, Triplet loss=2.052973, Training Accuracy=0.539062, lr=0.000010\n",
      "Iter=1361920/epoch=3, Loss=8.486008, Triplet loss=2.015340, Training Accuracy=0.453125, lr=0.000010\n",
      "Iter=1372160/epoch=3, Loss=7.165142, Triplet loss=2.050958, Training Accuracy=0.492188, lr=0.000010\n",
      "Iter=1382400/epoch=3, Loss=8.022903, Triplet loss=2.048486, Training Accuracy=0.539062, lr=0.000010\n",
      "Iter=1392640/epoch=3, Loss=7.168943, Triplet loss=2.092680, Training Accuracy=0.539062, lr=0.000010\n",
      "Iter=1402880/epoch=3, Loss=7.247968, Triplet loss=2.007328, Training Accuracy=0.453125, lr=0.000010\n",
      "Iter=1413120/epoch=3, Loss=10.487133, Triplet loss=2.031761, Training Accuracy=0.382812, lr=0.000010\n",
      "Iter=1423360/epoch=3, Loss=5.595948, Triplet loss=1.997066, Training Accuracy=0.546875, lr=0.000010\n",
      "Iter=1433600/epoch=3, Loss=6.982337, Triplet loss=2.059534, Training Accuracy=0.500000, lr=0.000010\n",
      "Iter=1433600/epoch=3, Validation Accuracy=0.477800, Precision=0.437146, Recall=0.154400\n",
      "Iter=1443840/epoch=3, Loss=9.021721, Triplet loss=2.015340, Training Accuracy=0.453125, lr=0.000010\n",
      "Iter=1454080/epoch=3, Loss=6.589930, Triplet loss=2.022256, Training Accuracy=0.507812, lr=0.000010\n",
      "Iter=1464320/epoch=3, Loss=9.276071, Triplet loss=1.942432, Training Accuracy=0.468750, lr=0.000010\n",
      "Iter=1474560/epoch=3, Loss=6.541528, Triplet loss=2.048666, Training Accuracy=0.484375, lr=0.000010\n",
      "Iter=1484800/epoch=3, Loss=7.147716, Triplet loss=2.103728, Training Accuracy=0.453125, lr=0.000010\n",
      "Iter=1495040/epoch=3, Loss=6.980523, Triplet loss=1.971750, Training Accuracy=0.531250, lr=0.000010\n",
      "Iter=1505280/epoch=3, Loss=7.947017, Triplet loss=2.004433, Training Accuracy=0.507812, lr=0.000010\n",
      "Iter=1515520/epoch=3, Loss=7.352521, Triplet loss=2.004292, Training Accuracy=0.578125, lr=0.000010\n",
      "Iter=1525760/epoch=3, Loss=7.740722, Triplet loss=1.883765, Training Accuracy=0.523438, lr=0.000010\n",
      "Iter=1536000/epoch=3, Loss=7.726939, Triplet loss=1.993486, Training Accuracy=0.500000, lr=0.000010\n",
      "Iter=1546240/epoch=4, Loss=6.422946, Triplet loss=1.994267, Training Accuracy=0.468750, lr=0.000010\n",
      "Iter=1556480/epoch=4, Loss=4.912710, Triplet loss=2.004304, Training Accuracy=0.484375, lr=0.000010\n",
      "Iter=1566720/epoch=4, Loss=3.120022, Triplet loss=2.012161, Training Accuracy=0.445312, lr=0.000010\n",
      "Iter=1576960/epoch=4, Loss=2.300177, Triplet loss=2.015341, Training Accuracy=0.500000, lr=0.000010\n",
      "Iter=1587200/epoch=4, Loss=1.742818, Triplet loss=2.092683, Training Accuracy=0.632812, lr=0.000010\n",
      "Iter=1597440/epoch=4, Loss=2.023423, Triplet loss=2.015340, Training Accuracy=0.578125, lr=0.000010\n",
      "Iter=1607680/epoch=4, Loss=2.360322, Triplet loss=2.026492, Training Accuracy=0.500000, lr=0.000010\n",
      "Iter=1617920/epoch=4, Loss=1.717123, Triplet loss=2.037437, Training Accuracy=0.562500, lr=0.000010\n",
      "Iter=1628160/epoch=4, Loss=1.656332, Triplet loss=2.048486, Training Accuracy=0.593750, lr=0.000010\n",
      "Iter=1638400/epoch=4, Loss=1.853757, Triplet loss=1.860726, Training Accuracy=0.570312, lr=0.000010\n",
      "Iter=1638400/epoch=4, Validation Accuracy=0.526600, Precision=0.525616, Recall=0.545800\n",
      "Iter=1648640/epoch=4, Loss=1.959902, Triplet loss=1.952326, Training Accuracy=0.578125, lr=0.000010\n",
      "Iter=1658880/epoch=4, Loss=1.784942, Triplet loss=1.994273, Training Accuracy=0.601562, lr=0.000010\n",
      "Iter=1669120/epoch=4, Loss=1.973264, Triplet loss=1.805418, Training Accuracy=0.562500, lr=0.000010\n",
      "Iter=1679360/epoch=4, Loss=1.462048, Triplet loss=2.027748, Training Accuracy=0.593750, lr=0.000010\n",
      "Iter=1689600/epoch=4, Loss=1.571560, Triplet loss=2.026388, Training Accuracy=0.585938, lr=0.000010\n",
      "Iter=1699840/epoch=4, Loss=1.928800, Triplet loss=2.070583, Training Accuracy=0.562500, lr=0.000010\n",
      "Iter=1710080/epoch=4, Loss=1.776383, Triplet loss=2.060566, Training Accuracy=0.601562, lr=0.000010\n",
      "Iter=1720320/epoch=4, Loss=1.651716, Triplet loss=1.963008, Training Accuracy=0.539062, lr=0.000010\n",
      "Iter=1730560/epoch=4, Loss=1.915264, Triplet loss=2.038283, Training Accuracy=0.546875, lr=0.000010\n",
      "Iter=1740800/epoch=4, Loss=1.235567, Triplet loss=2.015340, Training Accuracy=0.609375, lr=0.000010\n",
      "Iter=1751040/epoch=4, Loss=1.285432, Triplet loss=2.017108, Training Accuracy=0.617188, lr=0.000010\n",
      "Iter=1761280/epoch=4, Loss=1.490028, Triplet loss=1.997664, Training Accuracy=0.609375, lr=0.000010\n",
      "Iter=1771520/epoch=4, Loss=1.692084, Triplet loss=1.993243, Training Accuracy=0.578125, lr=0.000010\n",
      "Iter=1781760/epoch=4, Loss=1.362421, Triplet loss=2.037461, Training Accuracy=0.585938, lr=0.000010\n",
      "Iter=1792000/epoch=4, Loss=1.711697, Triplet loss=2.062736, Training Accuracy=0.570312, lr=0.000010\n",
      "Iter=1802240/epoch=4, Loss=1.153120, Triplet loss=2.071617, Training Accuracy=0.640625, lr=0.000010\n",
      "Iter=1812480/epoch=4, Loss=1.454930, Triplet loss=2.048498, Training Accuracy=0.562500, lr=0.000010\n",
      "Iter=1822720/epoch=4, Loss=1.329995, Triplet loss=2.004292, Training Accuracy=0.601562, lr=0.000010\n",
      "Iter=1832960/epoch=4, Loss=1.274720, Triplet loss=2.015340, Training Accuracy=0.609375, lr=0.000010\n",
      "Iter=1843200/epoch=4, Loss=1.577901, Triplet loss=2.040034, Training Accuracy=0.554688, lr=0.000010\n",
      "Iter=1843200/epoch=4, Validation Accuracy=0.576300, Precision=0.571643, Recall=0.608800\n",
      "Iter=1853440/epoch=4, Loss=1.273665, Triplet loss=2.064216, Training Accuracy=0.679688, lr=0.000010\n",
      "Iter=1863680/epoch=4, Loss=1.174834, Triplet loss=2.048737, Training Accuracy=0.664062, lr=0.000010\n",
      "Iter=1873920/epoch=4, Loss=1.307892, Triplet loss=1.927324, Training Accuracy=0.609375, lr=0.000010\n",
      "Iter=1884160/epoch=4, Loss=1.364808, Triplet loss=2.044816, Training Accuracy=0.625000, lr=0.000010\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter=1894400/epoch=4, Loss=1.497506, Triplet loss=1.882766, Training Accuracy=0.609375, lr=0.000010\n",
      "Iter=1904640/epoch=4, Loss=1.289406, Triplet loss=2.050190, Training Accuracy=0.609375, lr=0.000010\n",
      "Iter=1914880/epoch=4, Loss=1.439663, Triplet loss=2.048551, Training Accuracy=0.617188, lr=0.000010\n",
      "Model saved in file at iteration 1920128: checkpoints/tf_deepUD_tri_model_iter_15001_for_progressGAN.ckpt\n",
      "Iter=1925120/epoch=4, Loss=1.406238, Triplet loss=2.081631, Training Accuracy=0.640625, lr=0.000010\n",
      "Iter=1935360/epoch=5, Loss=1.216218, Triplet loss=2.070998, Training Accuracy=0.625000, lr=0.000010\n",
      "Iter=1945600/epoch=5, Loss=1.429035, Triplet loss=1.972195, Training Accuracy=0.632812, lr=0.000010\n",
      "Iter=1955840/epoch=5, Loss=1.482830, Triplet loss=1.983072, Training Accuracy=0.593750, lr=0.000010\n",
      "Iter=1966080/epoch=5, Loss=1.031421, Triplet loss=1.942806, Training Accuracy=0.671875, lr=0.000010\n",
      "Iter=1976320/epoch=5, Loss=1.351353, Triplet loss=1.971253, Training Accuracy=0.585938, lr=0.000010\n",
      "Iter=1986560/epoch=5, Loss=1.293820, Triplet loss=1.971151, Training Accuracy=0.640625, lr=0.000010\n",
      "Iter=1996800/epoch=5, Loss=1.201836, Triplet loss=2.065275, Training Accuracy=0.570312, lr=0.000010\n",
      "Iter=2007040/epoch=5, Loss=1.157050, Triplet loss=2.048486, Training Accuracy=0.640625, lr=0.000010\n",
      "Iter=2017280/epoch=5, Loss=1.247916, Triplet loss=2.037544, Training Accuracy=0.617188, lr=0.000010\n",
      "Iter=2027520/epoch=5, Loss=1.243138, Triplet loss=2.015387, Training Accuracy=0.578125, lr=0.000010\n",
      "Iter=2037760/epoch=5, Loss=1.309837, Triplet loss=1.993262, Training Accuracy=0.664062, lr=0.000010\n",
      "Iter=2048000/epoch=5, Loss=0.906216, Triplet loss=1.983552, Training Accuracy=0.648438, lr=0.000010\n",
      "Iter=2048000/epoch=5, Validation Accuracy=0.611200, Precision=0.603039, Recall=0.650800\n",
      "Iter=2058240/epoch=5, Loss=0.928741, Triplet loss=2.048493, Training Accuracy=0.632812, lr=0.000010\n",
      "Iter=2068480/epoch=5, Loss=1.382523, Triplet loss=1.894186, Training Accuracy=0.617188, lr=0.000010\n",
      "Iter=2078720/epoch=5, Loss=1.399346, Triplet loss=1.971237, Training Accuracy=0.609375, lr=0.000010\n",
      "Iter=2088960/epoch=5, Loss=1.294693, Triplet loss=2.015400, Training Accuracy=0.632812, lr=0.000010\n",
      "Iter=2099200/epoch=5, Loss=0.888692, Triplet loss=2.048832, Training Accuracy=0.625000, lr=0.000010\n",
      "Iter=2109440/epoch=5, Loss=1.347039, Triplet loss=1.987734, Training Accuracy=0.585938, lr=0.000010\n",
      "Iter=2119680/epoch=5, Loss=0.924554, Triplet loss=2.114777, Training Accuracy=0.671875, lr=0.000010\n",
      "Iter=2129920/epoch=5, Loss=1.091440, Triplet loss=2.006713, Training Accuracy=0.656250, lr=0.000010\n",
      "Iter=2140160/epoch=5, Loss=1.127295, Triplet loss=2.026412, Training Accuracy=0.703125, lr=0.000010\n",
      "Iter=2150400/epoch=5, Loss=1.046118, Triplet loss=2.081639, Training Accuracy=0.648438, lr=0.000010\n",
      "Iter=2160640/epoch=5, Loss=0.894131, Triplet loss=1.946328, Training Accuracy=0.609375, lr=0.000010\n",
      "Iter=2170880/epoch=5, Loss=1.004433, Triplet loss=2.048486, Training Accuracy=0.710938, lr=0.000010\n",
      "Iter=2181120/epoch=5, Loss=1.276768, Triplet loss=1.971147, Training Accuracy=0.585938, lr=0.000010\n",
      "Iter=2191360/epoch=5, Loss=1.188921, Triplet loss=2.059534, Training Accuracy=0.656250, lr=0.000010\n",
      "Iter=2201600/epoch=5, Loss=1.233136, Triplet loss=2.004291, Training Accuracy=0.593750, lr=0.000010\n",
      "Iter=2211840/epoch=5, Loss=1.171281, Triplet loss=2.070583, Training Accuracy=0.632812, lr=0.000010\n",
      "Iter=2222080/epoch=5, Loss=1.317745, Triplet loss=2.041621, Training Accuracy=0.664062, lr=0.000010\n",
      "Iter=2232320/epoch=5, Loss=1.095455, Triplet loss=2.003744, Training Accuracy=0.617188, lr=0.000010\n",
      "Iter=2242560/epoch=5, Loss=0.910021, Triplet loss=2.082944, Training Accuracy=0.734375, lr=0.000010\n",
      "Iter=2252800/epoch=5, Loss=0.985414, Triplet loss=2.022220, Training Accuracy=0.648438, lr=0.000010\n",
      "Iter=2252800/epoch=5, Validation Accuracy=0.631200, Precision=0.620633, Recall=0.675000\n",
      "Iter=2263040/epoch=5, Loss=1.026211, Triplet loss=1.904855, Training Accuracy=0.695312, lr=0.000010\n",
      "Iter=2273280/epoch=5, Loss=0.961829, Triplet loss=1.941594, Training Accuracy=0.570312, lr=0.000010\n",
      "Iter=2283520/epoch=5, Loss=0.857087, Triplet loss=2.114777, Training Accuracy=0.710938, lr=0.000010\n",
      "Iter=2293760/epoch=5, Loss=0.894758, Triplet loss=2.009761, Training Accuracy=0.710938, lr=0.000010\n",
      "Iter=2304000/epoch=5, Loss=0.841910, Triplet loss=2.059534, Training Accuracy=0.710938, lr=0.000010\n",
      "Iter=2314240/epoch=6, Loss=1.111736, Triplet loss=2.103740, Training Accuracy=0.671875, lr=0.000010\n",
      "Iter=2324480/epoch=6, Loss=1.097304, Triplet loss=2.038112, Training Accuracy=0.632812, lr=0.000010\n",
      "Iter=2334720/epoch=6, Loss=1.100515, Triplet loss=2.039796, Training Accuracy=0.632812, lr=0.000010\n",
      "Iter=2344960/epoch=6, Loss=1.047746, Triplet loss=1.993243, Training Accuracy=0.640625, lr=0.000010\n",
      "Iter=2355200/epoch=6, Loss=1.160427, Triplet loss=2.037941, Training Accuracy=0.601562, lr=0.000010\n",
      "Iter=2365440/epoch=6, Loss=0.984350, Triplet loss=2.062985, Training Accuracy=0.726562, lr=0.000010\n",
      "Iter=2375680/epoch=6, Loss=0.793822, Triplet loss=2.059330, Training Accuracy=0.664062, lr=0.000010\n",
      "Iter=2385920/epoch=6, Loss=0.985355, Triplet loss=1.938000, Training Accuracy=0.718750, lr=0.000010\n",
      "Iter=2396160/epoch=6, Loss=0.827566, Triplet loss=2.059537, Training Accuracy=0.687500, lr=0.000010\n",
      "Iter=2406400/epoch=6, Loss=1.058289, Triplet loss=2.098911, Training Accuracy=0.648438, lr=0.000010\n",
      "Iter=2416640/epoch=6, Loss=0.713102, Triplet loss=2.004293, Training Accuracy=0.757812, lr=0.000010\n",
      "Iter=2426880/epoch=6, Loss=0.668166, Triplet loss=1.993243, Training Accuracy=0.703125, lr=0.000010\n",
      "Iter=2437120/epoch=6, Loss=0.985496, Triplet loss=1.972127, Training Accuracy=0.687500, lr=0.000010\n",
      "Iter=2447360/epoch=6, Loss=0.959070, Triplet loss=2.050195, Training Accuracy=0.687500, lr=0.000010\n",
      "Iter=2457600/epoch=6, Loss=0.857885, Triplet loss=2.037500, Training Accuracy=0.695312, lr=0.000010\n",
      "Iter=2457600/epoch=6, Validation Accuracy=0.644600, Precision=0.635596, Recall=0.677800\n",
      "Iter=2467840/epoch=6, Loss=1.138635, Triplet loss=2.026388, Training Accuracy=0.625000, lr=0.000010\n",
      "Iter=2478080/epoch=6, Loss=1.135091, Triplet loss=2.059585, Training Accuracy=0.656250, lr=0.000010\n",
      "Iter=2488320/epoch=6, Loss=0.781432, Triplet loss=2.081631, Training Accuracy=0.742188, lr=0.000010\n",
      "Iter=2498560/epoch=6, Loss=0.724156, Triplet loss=2.092668, Training Accuracy=0.726562, lr=0.000010\n",
      "Iter=2508800/epoch=6, Loss=0.576523, Triplet loss=2.001453, Training Accuracy=0.734375, lr=0.000010\n",
      "Iter=2519040/epoch=6, Loss=0.821377, Triplet loss=2.015340, Training Accuracy=0.718750, lr=0.000010\n",
      "Iter=2529280/epoch=6, Loss=0.809005, Triplet loss=2.070583, Training Accuracy=0.718750, lr=0.000010\n",
      "Iter=2539520/epoch=6, Loss=1.190749, Triplet loss=2.076901, Training Accuracy=0.632812, lr=0.000010\n",
      "Iter=2549760/epoch=6, Loss=0.927133, Triplet loss=2.059542, Training Accuracy=0.718750, lr=0.000010\n",
      "Iter=2560000/epoch=6, Loss=0.858026, Triplet loss=2.037437, Training Accuracy=0.656250, lr=0.000010\n",
      "Iter=2570240/epoch=6, Loss=0.767530, Triplet loss=1.946351, Training Accuracy=0.734375, lr=0.000010\n",
      "Iter=2580480/epoch=6, Loss=0.692401, Triplet loss=2.048486, Training Accuracy=0.742188, lr=0.000010\n",
      "Iter=2590720/epoch=6, Loss=0.745020, Triplet loss=1.960258, Training Accuracy=0.695312, lr=0.000010\n",
      "Iter=2600960/epoch=6, Loss=0.606907, Triplet loss=2.015443, Training Accuracy=0.742188, lr=0.000010\n",
      "Iter=2611200/epoch=6, Loss=0.663407, Triplet loss=2.070286, Training Accuracy=0.710938, lr=0.000010\n",
      "Iter=2621440/epoch=6, Loss=0.702545, Triplet loss=1.995030, Training Accuracy=0.687500, lr=0.000010\n",
      "Iter=2631680/epoch=6, Loss=0.780199, Triplet loss=2.061129, Training Accuracy=0.734375, lr=0.000010\n",
      "Iter=2641920/epoch=6, Loss=0.860087, Triplet loss=2.059535, Training Accuracy=0.687500, lr=0.000010\n",
      "Iter=2652160/epoch=6, Loss=0.877294, Triplet loss=1.978274, Training Accuracy=0.656250, lr=0.000010\n",
      "Iter=2662400/epoch=6, Loss=0.760157, Triplet loss=2.070591, Training Accuracy=0.695312, lr=0.000010\n",
      "Iter=2662400/epoch=6, Validation Accuracy=0.653700, Precision=0.637355, Recall=0.713200\n",
      "Iter=2672640/epoch=6, Loss=1.007465, Triplet loss=2.053557, Training Accuracy=0.593750, lr=0.000010\n",
      "Iter=2682880/epoch=6, Loss=0.771282, Triplet loss=2.070819, Training Accuracy=0.679688, lr=0.000010\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter=2693120/epoch=6, Loss=0.868857, Triplet loss=2.026574, Training Accuracy=0.640625, lr=0.000010\n",
      "Iter=2703360/epoch=7, Loss=0.866418, Triplet loss=1.991269, Training Accuracy=0.726562, lr=0.000010\n",
      "Iter=2713600/epoch=7, Loss=0.787856, Triplet loss=2.059535, Training Accuracy=0.687500, lr=0.000010\n",
      "Iter=2723840/epoch=7, Loss=0.867753, Triplet loss=2.026473, Training Accuracy=0.687500, lr=0.000010\n",
      "Iter=2734080/epoch=7, Loss=0.648542, Triplet loss=1.983895, Training Accuracy=0.687500, lr=0.000010\n",
      "Iter=2744320/epoch=7, Loss=0.730713, Triplet loss=2.004291, Training Accuracy=0.703125, lr=0.000010\n",
      "Iter=2754560/epoch=7, Loss=0.760583, Triplet loss=2.026388, Training Accuracy=0.679688, lr=0.000010\n",
      "Iter=2764800/epoch=7, Loss=0.649434, Triplet loss=1.975194, Training Accuracy=0.718750, lr=0.000010\n",
      "Iter=2775040/epoch=7, Loss=0.791311, Triplet loss=2.047060, Training Accuracy=0.656250, lr=0.000010\n",
      "Iter=2785280/epoch=7, Loss=0.652202, Triplet loss=2.081372, Training Accuracy=0.718750, lr=0.000010\n",
      "Iter=2795520/epoch=7, Loss=0.887228, Triplet loss=2.048486, Training Accuracy=0.648438, lr=0.000010\n",
      "Iter=2805760/epoch=7, Loss=0.853837, Triplet loss=1.982414, Training Accuracy=0.695312, lr=0.000010\n",
      "Iter=2816000/epoch=7, Loss=0.887247, Triplet loss=2.026388, Training Accuracy=0.695312, lr=0.000010\n",
      "Iter=2826240/epoch=7, Loss=0.720595, Triplet loss=2.059796, Training Accuracy=0.734375, lr=0.000010\n",
      "Iter=2836480/epoch=7, Loss=0.800604, Triplet loss=2.092680, Training Accuracy=0.671875, lr=0.000010\n",
      "Iter=2846720/epoch=7, Loss=0.809880, Triplet loss=2.116934, Training Accuracy=0.695312, lr=0.000010\n",
      "Iter=2856960/epoch=7, Loss=0.705524, Triplet loss=2.015340, Training Accuracy=0.648438, lr=0.000010\n",
      "Iter=2867200/epoch=7, Loss=0.733957, Triplet loss=2.048486, Training Accuracy=0.750000, lr=0.000010\n",
      "Iter=2867200/epoch=7, Validation Accuracy=0.658000, Precision=0.643846, Recall=0.707200\n",
      "Iter=2877440/epoch=7, Loss=0.683663, Triplet loss=2.059534, Training Accuracy=0.664062, lr=0.000010\n",
      "Iter=2887680/epoch=7, Loss=0.615072, Triplet loss=2.103728, Training Accuracy=0.765625, lr=0.000010\n",
      "Iter=2897920/epoch=7, Loss=0.616393, Triplet loss=1.916043, Training Accuracy=0.765625, lr=0.000010\n",
      "Iter=2908160/epoch=7, Loss=0.692010, Triplet loss=2.037437, Training Accuracy=0.664062, lr=0.000010\n",
      "Iter=2918400/epoch=7, Loss=0.626605, Triplet loss=2.081631, Training Accuracy=0.734375, lr=0.000010\n",
      "Iter=2928640/epoch=7, Loss=0.606155, Triplet loss=2.018268, Training Accuracy=0.726562, lr=0.000010\n",
      "Iter=2938880/epoch=7, Loss=0.618808, Triplet loss=2.048486, Training Accuracy=0.789062, lr=0.000010\n",
      "Iter=2949120/epoch=7, Loss=0.815862, Triplet loss=1.993264, Training Accuracy=0.648438, lr=0.000010\n",
      "Iter=2959360/epoch=7, Loss=0.689793, Triplet loss=2.026390, Training Accuracy=0.710938, lr=0.000010\n",
      "Iter=2969600/epoch=7, Loss=0.710130, Triplet loss=2.119232, Training Accuracy=0.695312, lr=0.000010\n",
      "Iter=2979840/epoch=7, Loss=0.748704, Triplet loss=1.994626, Training Accuracy=0.687500, lr=0.000010\n",
      "Iter=2990080/epoch=7, Loss=0.695645, Triplet loss=2.015340, Training Accuracy=0.718750, lr=0.000010\n",
      "Iter=3000320/epoch=7, Loss=0.760482, Triplet loss=2.004291, Training Accuracy=0.742188, lr=0.000010\n",
      "Iter=3010560/epoch=7, Loss=0.576221, Triplet loss=2.059537, Training Accuracy=0.781250, lr=0.000010\n",
      "Iter=3020800/epoch=7, Loss=0.704713, Triplet loss=2.081631, Training Accuracy=0.679688, lr=0.000010\n",
      "Iter=3031040/epoch=7, Loss=0.475350, Triplet loss=2.042105, Training Accuracy=0.773438, lr=0.000010\n",
      "Iter=3041280/epoch=7, Loss=0.706482, Triplet loss=2.038001, Training Accuracy=0.734375, lr=0.000010\n",
      "Iter=3051520/epoch=7, Loss=0.727461, Triplet loss=2.081631, Training Accuracy=0.726562, lr=0.000010\n",
      "Iter=3061760/epoch=7, Loss=0.583127, Triplet loss=1.971148, Training Accuracy=0.734375, lr=0.000010\n",
      "Iter=3072000/epoch=7, Loss=0.602738, Triplet loss=2.029400, Training Accuracy=0.695312, lr=0.000010\n",
      "Iter=3072000/epoch=7, Validation Accuracy=0.665100, Precision=0.646236, Recall=0.729600\n",
      "Iter=3082240/epoch=8, Loss=0.686803, Triplet loss=2.028281, Training Accuracy=0.703125, lr=0.000010\n",
      "Iter=3092480/epoch=8, Loss=0.595713, Triplet loss=2.052031, Training Accuracy=0.781250, lr=0.000010\n",
      "Iter=3102720/epoch=8, Loss=0.711361, Triplet loss=1.983650, Training Accuracy=0.687500, lr=0.000010\n",
      "Iter=3112960/epoch=8, Loss=0.772800, Triplet loss=2.004292, Training Accuracy=0.710938, lr=0.000010\n",
      "Iter=3123200/epoch=8, Loss=0.561517, Triplet loss=1.949300, Training Accuracy=0.773438, lr=0.000010\n",
      "Iter=3133440/epoch=8, Loss=0.728776, Triplet loss=1.993256, Training Accuracy=0.710938, lr=0.000010\n",
      "Iter=3143680/epoch=8, Loss=0.597740, Triplet loss=2.016392, Training Accuracy=0.656250, lr=0.000010\n",
      "Iter=3153920/epoch=8, Loss=0.714136, Triplet loss=2.004735, Training Accuracy=0.687500, lr=0.000010\n",
      "Iter=3164160/epoch=8, Loss=0.791022, Triplet loss=2.070583, Training Accuracy=0.757812, lr=0.000010\n",
      "Iter=3174400/epoch=8, Loss=0.618996, Triplet loss=2.037948, Training Accuracy=0.765625, lr=0.000010\n",
      "Iter=3184640/epoch=8, Loss=0.743825, Triplet loss=1.974776, Training Accuracy=0.734375, lr=0.000010\n",
      "Iter=3194880/epoch=8, Loss=0.628446, Triplet loss=1.982196, Training Accuracy=0.726562, lr=0.000010\n",
      "Iter=3205120/epoch=8, Loss=0.835491, Triplet loss=2.103615, Training Accuracy=0.664062, lr=0.000010\n",
      "Iter=3215360/epoch=8, Loss=0.763179, Triplet loss=1.971146, Training Accuracy=0.726562, lr=0.000010\n",
      "Iter=3225600/epoch=8, Loss=0.777329, Triplet loss=2.070591, Training Accuracy=0.710938, lr=0.000010\n",
      "Iter=3235840/epoch=8, Loss=0.752220, Triplet loss=2.005848, Training Accuracy=0.750000, lr=0.000010\n",
      "Iter=3246080/epoch=8, Loss=0.668633, Triplet loss=2.008290, Training Accuracy=0.726562, lr=0.000010\n",
      "Iter=3256320/epoch=8, Loss=0.542119, Triplet loss=2.004296, Training Accuracy=0.781250, lr=0.000010\n",
      "Iter=3266560/epoch=8, Loss=0.753675, Triplet loss=2.048486, Training Accuracy=0.703125, lr=0.000010\n",
      "Iter=3276800/epoch=8, Loss=0.538720, Triplet loss=1.960156, Training Accuracy=0.781250, lr=0.000010\n",
      "Iter=3276800/epoch=8, Validation Accuracy=0.678200, Precision=0.658091, Recall=0.741800\n",
      "Iter=3287040/epoch=8, Loss=0.707448, Triplet loss=2.104067, Training Accuracy=0.734375, lr=0.000010\n",
      "Iter=3297280/epoch=8, Loss=0.817411, Triplet loss=1.960097, Training Accuracy=0.671875, lr=0.000010\n",
      "Iter=3307520/epoch=8, Loss=0.605668, Triplet loss=2.037437, Training Accuracy=0.765625, lr=0.000010\n",
      "Iter=3317760/epoch=8, Loss=0.648436, Triplet loss=1.982194, Training Accuracy=0.757812, lr=0.000010\n",
      "Iter=3328000/epoch=8, Loss=0.591826, Triplet loss=2.029549, Training Accuracy=0.710938, lr=0.000010\n",
      "Iter=3338240/epoch=8, Loss=0.718741, Triplet loss=2.059534, Training Accuracy=0.687500, lr=0.000010\n",
      "Iter=3348480/epoch=8, Loss=0.693178, Triplet loss=1.958608, Training Accuracy=0.734375, lr=0.000010\n",
      "Iter=3358720/epoch=8, Loss=0.509637, Triplet loss=2.081711, Training Accuracy=0.828125, lr=0.000010\n",
      "Iter=3368960/epoch=8, Loss=0.721807, Triplet loss=2.015340, Training Accuracy=0.703125, lr=0.000010\n",
      "Iter=3379200/epoch=8, Loss=0.738754, Triplet loss=2.016774, Training Accuracy=0.742188, lr=0.000010\n",
      "Iter=3389440/epoch=8, Loss=0.723275, Triplet loss=2.038032, Training Accuracy=0.757812, lr=0.000010\n",
      "Iter=3399680/epoch=8, Loss=0.605392, Triplet loss=2.104232, Training Accuracy=0.796875, lr=0.000010\n",
      "Iter=3409920/epoch=8, Loss=0.889146, Triplet loss=1.993243, Training Accuracy=0.710938, lr=0.000010\n",
      "Iter=3420160/epoch=8, Loss=0.687204, Triplet loss=2.061639, Training Accuracy=0.687500, lr=0.000010\n",
      "Iter=3430400/epoch=8, Loss=0.571159, Triplet loss=2.015503, Training Accuracy=0.742188, lr=0.000010\n",
      "Iter=3440640/epoch=8, Loss=0.648500, Triplet loss=2.004291, Training Accuracy=0.742188, lr=0.000010\n",
      "Iter=3450880/epoch=8, Loss=0.413825, Triplet loss=2.026391, Training Accuracy=0.812500, lr=0.000010\n",
      "Iter=3461120/epoch=8, Loss=0.608745, Triplet loss=2.016499, Training Accuracy=0.773438, lr=0.000010\n",
      "Iter=3471360/epoch=9, Loss=0.445695, Triplet loss=2.092687, Training Accuracy=0.859375, lr=0.000010\n",
      "Iter=3481600/epoch=9, Loss=0.642394, Triplet loss=2.048486, Training Accuracy=0.726562, lr=0.000010\n",
      "Iter=3481600/epoch=9, Validation Accuracy=0.678800, Precision=0.654138, Recall=0.758800\n",
      "Iter=3491840/epoch=9, Loss=0.551181, Triplet loss=2.071047, Training Accuracy=0.796875, lr=0.000010\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter=3502080/epoch=9, Loss=0.477972, Triplet loss=2.040533, Training Accuracy=0.765625, lr=0.000010\n",
      "Iter=3512320/epoch=9, Loss=0.773734, Triplet loss=2.004296, Training Accuracy=0.664062, lr=0.000010\n",
      "Iter=3522560/epoch=9, Loss=0.792031, Triplet loss=2.049706, Training Accuracy=0.656250, lr=0.000010\n",
      "Iter=3532800/epoch=9, Loss=0.587582, Triplet loss=2.043339, Training Accuracy=0.695312, lr=0.000010\n",
      "Iter=3543040/epoch=9, Loss=0.549168, Triplet loss=2.073378, Training Accuracy=0.765625, lr=0.000010\n",
      "Iter=3553280/epoch=9, Loss=0.584930, Triplet loss=2.050632, Training Accuracy=0.710938, lr=0.000010\n",
      "Iter=3563520/epoch=9, Loss=0.708348, Triplet loss=2.093266, Training Accuracy=0.671875, lr=0.000010\n",
      "Iter=3573760/epoch=9, Loss=0.729024, Triplet loss=2.048486, Training Accuracy=0.687500, lr=0.000010\n",
      "Iter=3584000/epoch=9, Loss=0.444472, Triplet loss=2.008837, Training Accuracy=0.843750, lr=0.000010\n",
      "Iter=3594240/epoch=9, Loss=0.492763, Triplet loss=2.081631, Training Accuracy=0.789062, lr=0.000010\n",
      "Iter=3604480/epoch=9, Loss=0.483047, Triplet loss=2.051361, Training Accuracy=0.757812, lr=0.000010\n",
      "Iter=3614720/epoch=9, Loss=0.547228, Triplet loss=2.004452, Training Accuracy=0.750000, lr=0.000010\n",
      "Iter=3624960/epoch=9, Loss=0.467363, Triplet loss=2.015340, Training Accuracy=0.781250, lr=0.000010\n",
      "Iter=3635200/epoch=9, Loss=0.602816, Triplet loss=1.971163, Training Accuracy=0.742188, lr=0.000010\n",
      "Iter=3645440/epoch=9, Loss=0.676269, Triplet loss=2.026389, Training Accuracy=0.710938, lr=0.000010\n",
      "Iter=3655680/epoch=9, Loss=0.445978, Triplet loss=2.037437, Training Accuracy=0.804688, lr=0.000010\n",
      "Iter=3665920/epoch=9, Loss=0.523836, Triplet loss=2.081631, Training Accuracy=0.796875, lr=0.000010\n",
      "Iter=3676160/epoch=9, Loss=0.605945, Triplet loss=2.021682, Training Accuracy=0.757812, lr=0.000010\n",
      "Iter=3686400/epoch=9, Loss=0.621281, Triplet loss=1.977730, Training Accuracy=0.757812, lr=0.000010\n",
      "Iter=3686400/epoch=9, Validation Accuracy=0.676600, Precision=0.641870, Recall=0.799000\n",
      "Iter=3696640/epoch=9, Loss=0.622318, Triplet loss=2.093678, Training Accuracy=0.734375, lr=0.000010\n",
      "Iter=3706880/epoch=9, Loss=0.756472, Triplet loss=2.037437, Training Accuracy=0.703125, lr=0.000010\n",
      "Iter=3717120/epoch=9, Loss=0.633362, Triplet loss=2.081728, Training Accuracy=0.718750, lr=0.000010\n",
      "Iter=3727360/epoch=9, Loss=0.520306, Triplet loss=1.992047, Training Accuracy=0.765625, lr=0.000010\n",
      "Iter=3737600/epoch=9, Loss=0.577916, Triplet loss=2.015340, Training Accuracy=0.781250, lr=0.000010\n",
      "Iter=3747840/epoch=9, Loss=0.611255, Triplet loss=2.079305, Training Accuracy=0.742188, lr=0.000010\n",
      "Iter=3758080/epoch=9, Loss=0.642547, Triplet loss=2.020744, Training Accuracy=0.765625, lr=0.000010\n",
      "Iter=3768320/epoch=9, Loss=0.595287, Triplet loss=2.050308, Training Accuracy=0.734375, lr=0.000010\n",
      "Iter=3778560/epoch=9, Loss=0.709526, Triplet loss=2.048944, Training Accuracy=0.742188, lr=0.000010\n",
      "Iter=3788800/epoch=9, Loss=0.548309, Triplet loss=2.026388, Training Accuracy=0.781250, lr=0.000010\n",
      "Iter=3799040/epoch=9, Loss=0.507338, Triplet loss=1.982194, Training Accuracy=0.789062, lr=0.000010\n",
      "Iter=3809280/epoch=9, Loss=0.632629, Triplet loss=1.915914, Training Accuracy=0.734375, lr=0.000010\n",
      "Iter=3819520/epoch=9, Loss=0.626363, Triplet loss=2.092784, Training Accuracy=0.687500, lr=0.000010\n",
      "Iter=3829760/epoch=9, Loss=0.485717, Triplet loss=2.031291, Training Accuracy=0.796875, lr=0.000010\n",
      "Iter=3840000/epoch=9, Loss=0.574980, Triplet loss=2.060285, Training Accuracy=0.750000, lr=0.000010\n",
      "Model saved in file at iteration 3840128: checkpoints/tf_deepUD_tri_model_iter_30001_for_progressGAN.ckpt\n",
      "Iter=3850240/epoch=9, Loss=0.647617, Triplet loss=2.059534, Training Accuracy=0.781250, lr=0.000010\n",
      "Iter=3860480/epoch=10, Loss=0.520219, Triplet loss=2.092693, Training Accuracy=0.765625, lr=0.000010\n",
      "Iter=3870720/epoch=10, Loss=0.532447, Triplet loss=2.114778, Training Accuracy=0.773438, lr=0.000010\n",
      "Iter=3880960/epoch=10, Loss=0.507404, Triplet loss=2.033070, Training Accuracy=0.773438, lr=0.000010\n",
      "Iter=3891200/epoch=10, Loss=0.619568, Triplet loss=2.059783, Training Accuracy=0.726562, lr=0.000010\n",
      "Iter=3891200/epoch=10, Validation Accuracy=0.691300, Precision=0.662477, Recall=0.780000\n",
      "Iter=3901440/epoch=10, Loss=0.439495, Triplet loss=2.048592, Training Accuracy=0.804688, lr=0.000010\n",
      "Iter=3911680/epoch=10, Loss=0.485717, Triplet loss=1.993243, Training Accuracy=0.804688, lr=0.000010\n",
      "Iter=3921920/epoch=10, Loss=0.466720, Triplet loss=2.072031, Training Accuracy=0.851562, lr=0.000010\n",
      "Iter=3932160/epoch=10, Loss=0.581531, Triplet loss=2.092747, Training Accuracy=0.757812, lr=0.000010\n",
      "Iter=3942400/epoch=10, Loss=0.636381, Triplet loss=2.063556, Training Accuracy=0.765625, lr=0.000010\n",
      "Iter=3952640/epoch=10, Loss=0.623488, Triplet loss=1.991008, Training Accuracy=0.789062, lr=0.000010\n",
      "Iter=3962880/epoch=10, Loss=0.442477, Triplet loss=2.059570, Training Accuracy=0.773438, lr=0.000010\n",
      "Iter=3973120/epoch=10, Loss=0.434259, Triplet loss=2.070583, Training Accuracy=0.773438, lr=0.000010\n",
      "Iter=3983360/epoch=10, Loss=0.665124, Triplet loss=2.114779, Training Accuracy=0.789062, lr=0.000010\n",
      "Iter=3993600/epoch=10, Loss=0.684511, Triplet loss=2.125850, Training Accuracy=0.750000, lr=0.000010\n",
      "Iter=4003840/epoch=10, Loss=0.487870, Triplet loss=2.068697, Training Accuracy=0.812500, lr=0.000010\n",
      "Iter=4014080/epoch=10, Loss=0.449535, Triplet loss=2.015340, Training Accuracy=0.765625, lr=0.000010\n",
      "Iter=4024320/epoch=10, Loss=0.614929, Triplet loss=2.015340, Training Accuracy=0.757812, lr=0.000010\n",
      "Iter=4034560/epoch=10, Loss=0.575343, Triplet loss=2.070583, Training Accuracy=0.703125, lr=0.000010\n",
      "Iter=4044800/epoch=10, Loss=0.343133, Triplet loss=2.072065, Training Accuracy=0.859375, lr=0.000010\n",
      "Iter=4055040/epoch=10, Loss=0.541279, Triplet loss=2.005673, Training Accuracy=0.734375, lr=0.000010\n",
      "Iter=4065280/epoch=10, Loss=0.576592, Triplet loss=2.103676, Training Accuracy=0.804688, lr=0.000010\n",
      "Iter=4075520/epoch=10, Loss=0.461467, Triplet loss=1.949049, Training Accuracy=0.812500, lr=0.000010\n",
      "Iter=4085760/epoch=10, Loss=0.481709, Triplet loss=2.029902, Training Accuracy=0.820312, lr=0.000010\n",
      "Iter=4096000/epoch=10, Loss=0.493273, Triplet loss=2.026389, Training Accuracy=0.796875, lr=0.000010\n",
      "Iter=4096000/epoch=10, Validation Accuracy=0.696600, Precision=0.663724, Recall=0.797000\n",
      "Iter=4106240/epoch=10, Loss=0.415122, Triplet loss=2.070615, Training Accuracy=0.820312, lr=0.000010\n",
      "Iter=4116480/epoch=10, Loss=0.613183, Triplet loss=2.114777, Training Accuracy=0.757812, lr=0.000010\n",
      "Iter=4126720/epoch=10, Loss=0.499941, Triplet loss=2.103724, Training Accuracy=0.750000, lr=0.000010\n",
      "Iter=4136960/epoch=10, Loss=0.476025, Triplet loss=2.114782, Training Accuracy=0.820312, lr=0.000010\n",
      "Iter=4147200/epoch=10, Loss=0.695999, Triplet loss=2.081631, Training Accuracy=0.718750, lr=0.000010\n",
      "Iter=4157440/epoch=10, Loss=0.609029, Triplet loss=2.015399, Training Accuracy=0.781250, lr=0.000010\n",
      "Iter=4167680/epoch=10, Loss=0.535199, Triplet loss=2.037470, Training Accuracy=0.734375, lr=0.000010\n",
      "Iter=4177920/epoch=10, Loss=0.505107, Triplet loss=2.070603, Training Accuracy=0.796875, lr=0.000010\n",
      "Iter=4188160/epoch=10, Loss=0.528481, Triplet loss=2.026388, Training Accuracy=0.757812, lr=0.000010\n",
      "Iter=4198400/epoch=10, Loss=0.533953, Triplet loss=2.048560, Training Accuracy=0.765625, lr=0.000010\n",
      "Iter=4208640/epoch=10, Loss=0.502882, Triplet loss=2.070583, Training Accuracy=0.820312, lr=0.000010\n",
      "Iter=4218880/epoch=10, Loss=0.574958, Triplet loss=2.048519, Training Accuracy=0.750000, lr=0.000010\n",
      "Iter=4229120/epoch=10, Loss=0.517424, Triplet loss=1.993243, Training Accuracy=0.734375, lr=0.000010\n",
      "Iter=4239360/epoch=11, Loss=0.616541, Triplet loss=2.059771, Training Accuracy=0.789062, lr=0.000010\n",
      "Iter=4249600/epoch=11, Loss=0.658346, Triplet loss=2.004291, Training Accuracy=0.781250, lr=0.000010\n",
      "Iter=4259840/epoch=11, Loss=0.484738, Triplet loss=2.094211, Training Accuracy=0.765625, lr=0.000010\n",
      "Iter=4270080/epoch=11, Loss=0.507418, Triplet loss=2.026388, Training Accuracy=0.781250, lr=0.000010\n",
      "Iter=4280320/epoch=11, Loss=0.549920, Triplet loss=1.987346, Training Accuracy=0.757812, lr=0.000010\n",
      "Iter=4290560/epoch=11, Loss=0.428817, Triplet loss=2.024372, Training Accuracy=0.796875, lr=0.000010\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter=4300800/epoch=11, Loss=0.421449, Triplet loss=1.993243, Training Accuracy=0.820312, lr=0.000010\n",
      "Iter=4300800/epoch=11, Validation Accuracy=0.692400, Precision=0.654613, Recall=0.814600\n",
      "Iter=4311040/epoch=11, Loss=0.488290, Triplet loss=2.035113, Training Accuracy=0.812500, lr=0.000010\n",
      "Iter=4321280/epoch=11, Loss=0.460552, Triplet loss=2.158971, Training Accuracy=0.796875, lr=0.000010\n",
      "Iter=4331520/epoch=11, Loss=0.546017, Triplet loss=2.070583, Training Accuracy=0.757812, lr=0.000010\n",
      "Iter=4341760/epoch=11, Loss=0.496995, Triplet loss=2.114777, Training Accuracy=0.765625, lr=0.000010\n",
      "Iter=4352000/epoch=11, Loss=0.495262, Triplet loss=2.059541, Training Accuracy=0.789062, lr=0.000010\n",
      "Iter=4362240/epoch=11, Loss=0.578273, Triplet loss=2.048273, Training Accuracy=0.750000, lr=0.000010\n",
      "Iter=4372480/epoch=11, Loss=0.345754, Triplet loss=2.073644, Training Accuracy=0.828125, lr=0.000010\n",
      "Iter=4382720/epoch=11, Loss=0.443725, Triplet loss=2.093158, Training Accuracy=0.781250, lr=0.000010\n",
      "Iter=4392960/epoch=11, Loss=0.452358, Triplet loss=2.048487, Training Accuracy=0.859375, lr=0.000010\n",
      "Iter=4403200/epoch=11, Loss=0.403355, Triplet loss=2.125826, Training Accuracy=0.812500, lr=0.000010\n",
      "Iter=4413440/epoch=11, Loss=0.508033, Triplet loss=2.037438, Training Accuracy=0.781250, lr=0.000010\n",
      "Iter=4423680/epoch=11, Loss=0.394140, Triplet loss=2.070586, Training Accuracy=0.789062, lr=0.000010\n",
      "Iter=4433920/epoch=11, Loss=0.381020, Triplet loss=2.004291, Training Accuracy=0.828125, lr=0.000010\n",
      "Iter=4444160/epoch=11, Loss=0.539747, Triplet loss=2.048466, Training Accuracy=0.734375, lr=0.000010\n",
      "Iter=4454400/epoch=11, Loss=0.415116, Triplet loss=2.125898, Training Accuracy=0.843750, lr=0.000010\n",
      "Iter=4464640/epoch=11, Loss=0.572338, Triplet loss=2.037437, Training Accuracy=0.742188, lr=0.000010\n",
      "Iter=4474880/epoch=11, Loss=0.507423, Triplet loss=2.086239, Training Accuracy=0.820312, lr=0.000010\n",
      "Iter=4485120/epoch=11, Loss=0.468480, Triplet loss=2.092680, Training Accuracy=0.828125, lr=0.000010\n",
      "Iter=4495360/epoch=11, Loss=0.391901, Triplet loss=2.032885, Training Accuracy=0.820312, lr=0.000010\n",
      "Iter=4505600/epoch=11, Loss=0.479854, Triplet loss=2.026392, Training Accuracy=0.835938, lr=0.000010\n",
      "Iter=4505600/epoch=11, Validation Accuracy=0.700700, Precision=0.662090, Recall=0.819800\n",
      "Iter=4515840/epoch=11, Loss=0.448191, Triplet loss=2.070583, Training Accuracy=0.796875, lr=0.000010\n",
      "Iter=4526080/epoch=11, Loss=0.533327, Triplet loss=2.026388, Training Accuracy=0.781250, lr=0.000010\n",
      "Iter=4536320/epoch=11, Loss=0.373577, Triplet loss=2.015340, Training Accuracy=0.820312, lr=0.000010\n",
      "Iter=4546560/epoch=11, Loss=0.386484, Triplet loss=2.015340, Training Accuracy=0.812500, lr=0.000010\n",
      "Iter=4556800/epoch=11, Loss=0.487496, Triplet loss=2.074273, Training Accuracy=0.812500, lr=0.000010\n",
      "Iter=4567040/epoch=11, Loss=0.455307, Triplet loss=2.094245, Training Accuracy=0.820312, lr=0.000010\n",
      "Iter=4577280/epoch=11, Loss=0.425389, Triplet loss=2.060204, Training Accuracy=0.789062, lr=0.000010\n",
      "Iter=4587520/epoch=11, Loss=0.432842, Triplet loss=2.049553, Training Accuracy=0.812500, lr=0.000010\n",
      "Iter=4597760/epoch=11, Loss=0.543019, Triplet loss=2.070583, Training Accuracy=0.773438, lr=0.000010\n",
      "Iter=4608000/epoch=11, Loss=0.416437, Triplet loss=2.103734, Training Accuracy=0.812500, lr=0.000010\n",
      "Iter=4618240/epoch=11, Loss=0.413922, Triplet loss=2.048602, Training Accuracy=0.781250, lr=0.000010\n",
      "Iter=4628480/epoch=12, Loss=0.610404, Triplet loss=2.059534, Training Accuracy=0.796875, lr=0.000010\n",
      "Iter=4638720/epoch=12, Loss=0.462105, Triplet loss=2.052710, Training Accuracy=0.781250, lr=0.000010\n",
      "Iter=4648960/epoch=12, Loss=0.361477, Triplet loss=2.059772, Training Accuracy=0.843750, lr=0.000010\n",
      "Iter=4659200/epoch=12, Loss=0.502591, Triplet loss=2.015344, Training Accuracy=0.773438, lr=0.000010\n",
      "Iter=4669440/epoch=12, Loss=0.419154, Triplet loss=2.048486, Training Accuracy=0.835938, lr=0.000010\n",
      "Iter=4679680/epoch=12, Loss=0.399483, Triplet loss=2.038472, Training Accuracy=0.796875, lr=0.000010\n",
      "Iter=4689920/epoch=12, Loss=0.339227, Triplet loss=2.103709, Training Accuracy=0.843750, lr=0.000010\n",
      "Iter=4700160/epoch=12, Loss=0.345009, Triplet loss=2.040098, Training Accuracy=0.851562, lr=0.000010\n",
      "Iter=4710400/epoch=12, Loss=0.489743, Triplet loss=2.076579, Training Accuracy=0.781250, lr=0.000010\n",
      "Iter=4710400/epoch=12, Validation Accuracy=0.699600, Precision=0.659323, Recall=0.826000\n",
      "Iter=4720640/epoch=12, Loss=0.460924, Triplet loss=2.105178, Training Accuracy=0.812500, lr=0.000010\n",
      "Iter=4730880/epoch=12, Loss=0.392490, Triplet loss=2.059660, Training Accuracy=0.859375, lr=0.000010\n",
      "Iter=4741120/epoch=12, Loss=0.452674, Triplet loss=2.037437, Training Accuracy=0.828125, lr=0.000010\n",
      "Iter=4751360/epoch=12, Loss=0.457689, Triplet loss=2.059623, Training Accuracy=0.765625, lr=0.000010\n",
      "Iter=4761600/epoch=12, Loss=0.401473, Triplet loss=2.083213, Training Accuracy=0.835938, lr=0.000010\n",
      "Iter=4771840/epoch=12, Loss=0.420594, Triplet loss=2.093356, Training Accuracy=0.820312, lr=0.000010\n",
      "Iter=4782080/epoch=12, Loss=0.409324, Triplet loss=2.015305, Training Accuracy=0.820312, lr=0.000010\n",
      "Iter=4792320/epoch=12, Loss=0.412944, Triplet loss=2.136874, Training Accuracy=0.828125, lr=0.000010\n",
      "Iter=4802560/epoch=12, Loss=0.476913, Triplet loss=2.026390, Training Accuracy=0.804688, lr=0.000010\n",
      "Iter=4812800/epoch=12, Loss=0.385885, Triplet loss=1.971147, Training Accuracy=0.835938, lr=0.000010\n",
      "Iter=4823040/epoch=12, Loss=0.417366, Triplet loss=2.016427, Training Accuracy=0.804688, lr=0.000010\n",
      "Iter=4833280/epoch=12, Loss=0.468035, Triplet loss=2.103397, Training Accuracy=0.773438, lr=0.000010\n",
      "Iter=4843520/epoch=12, Loss=0.496437, Triplet loss=2.059534, Training Accuracy=0.781250, lr=0.000010\n",
      "Iter=4853760/epoch=12, Loss=0.535452, Triplet loss=1.958470, Training Accuracy=0.796875, lr=0.000010\n",
      "Iter=4864000/epoch=12, Loss=0.460499, Triplet loss=2.015340, Training Accuracy=0.820312, lr=0.000010\n",
      "Iter=4874240/epoch=12, Loss=0.391311, Triplet loss=2.081680, Training Accuracy=0.796875, lr=0.000010\n",
      "Iter=4884480/epoch=12, Loss=0.465820, Triplet loss=2.065011, Training Accuracy=0.820312, lr=0.000010\n",
      "Iter=4894720/epoch=12, Loss=0.521607, Triplet loss=1.994221, Training Accuracy=0.804688, lr=0.000010\n",
      "Iter=4904960/epoch=12, Loss=0.433589, Triplet loss=2.092689, Training Accuracy=0.843750, lr=0.000010\n",
      "Iter=4915200/epoch=12, Loss=0.299663, Triplet loss=2.092680, Training Accuracy=0.898438, lr=0.000010\n",
      "Iter=4915200/epoch=12, Validation Accuracy=0.709600, Precision=0.666826, Recall=0.837800\n",
      "Iter=4925440/epoch=12, Loss=0.501235, Triplet loss=2.041178, Training Accuracy=0.750000, lr=0.000010\n",
      "Iter=4935680/epoch=12, Loss=0.499896, Triplet loss=2.070590, Training Accuracy=0.820312, lr=0.000010\n",
      "Iter=4945920/epoch=12, Loss=0.412000, Triplet loss=1.993254, Training Accuracy=0.843750, lr=0.000010\n",
      "Iter=4956160/epoch=12, Loss=0.491824, Triplet loss=2.081631, Training Accuracy=0.789062, lr=0.000010\n",
      "Iter=4966400/epoch=12, Loss=0.549008, Triplet loss=2.015340, Training Accuracy=0.726562, lr=0.000010\n",
      "Iter=4976640/epoch=12, Loss=0.391940, Triplet loss=2.136874, Training Accuracy=0.851562, lr=0.000010\n",
      "Iter=4986880/epoch=12, Loss=0.494139, Triplet loss=2.059534, Training Accuracy=0.757812, lr=0.000010\n",
      "Iter=4997120/epoch=12, Loss=0.468889, Triplet loss=2.015368, Training Accuracy=0.796875, lr=0.000010\n",
      "Iter=5007360/epoch=12, Loss=0.427349, Triplet loss=2.070650, Training Accuracy=0.828125, lr=0.000010\n",
      "Iter=5017600/epoch=13, Loss=0.511656, Triplet loss=2.081631, Training Accuracy=0.789062, lr=0.000010\n",
      "Iter=5027840/epoch=13, Loss=0.425983, Triplet loss=2.059850, Training Accuracy=0.820312, lr=0.000010\n",
      "Iter=5038080/epoch=13, Loss=0.398919, Triplet loss=2.037437, Training Accuracy=0.859375, lr=0.000010\n",
      "Iter=5048320/epoch=13, Loss=0.511707, Triplet loss=2.004315, Training Accuracy=0.781250, lr=0.000010\n",
      "Iter=5058560/epoch=13, Loss=0.453871, Triplet loss=2.048486, Training Accuracy=0.851562, lr=0.000010\n",
      "Iter=5068800/epoch=13, Loss=0.406617, Triplet loss=2.059534, Training Accuracy=0.804688, lr=0.000010\n",
      "Iter=5079040/epoch=13, Loss=0.520401, Triplet loss=2.016109, Training Accuracy=0.773438, lr=0.000010\n",
      "Iter=5089280/epoch=13, Loss=0.447066, Triplet loss=2.115019, Training Accuracy=0.789062, lr=0.000010\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter=5099520/epoch=13, Loss=0.411122, Triplet loss=2.114794, Training Accuracy=0.835938, lr=0.000010\n",
      "Iter=5109760/epoch=13, Loss=0.509869, Triplet loss=2.092680, Training Accuracy=0.781250, lr=0.000010\n",
      "Iter=5120000/epoch=13, Loss=0.500766, Triplet loss=2.070656, Training Accuracy=0.781250, lr=0.000010\n",
      "Iter=5120000/epoch=13, Validation Accuracy=0.706900, Precision=0.663920, Recall=0.838000\n",
      "Iter=5130240/epoch=13, Loss=0.361668, Triplet loss=2.081631, Training Accuracy=0.851562, lr=0.000010\n",
      "Iter=5140480/epoch=13, Loss=0.364563, Triplet loss=2.048519, Training Accuracy=0.835938, lr=0.000010\n",
      "Iter=5150720/epoch=13, Loss=0.420039, Triplet loss=2.088181, Training Accuracy=0.804688, lr=0.000010\n",
      "Iter=5160960/epoch=13, Loss=0.520986, Triplet loss=2.020895, Training Accuracy=0.789062, lr=0.000010\n",
      "Iter=5171200/epoch=13, Loss=0.452856, Triplet loss=2.070583, Training Accuracy=0.828125, lr=0.000010\n",
      "Iter=5181440/epoch=13, Loss=0.536114, Triplet loss=2.126131, Training Accuracy=0.804688, lr=0.000010\n",
      "Iter=5191680/epoch=13, Loss=0.424937, Triplet loss=2.048486, Training Accuracy=0.812500, lr=0.000010\n",
      "Iter=5201920/epoch=13, Loss=0.544457, Triplet loss=2.070583, Training Accuracy=0.757812, lr=0.000010\n",
      "Iter=5212160/epoch=13, Loss=0.340317, Triplet loss=2.103730, Training Accuracy=0.867188, lr=0.000010\n",
      "Iter=5222400/epoch=13, Loss=0.553183, Triplet loss=1.971148, Training Accuracy=0.757812, lr=0.000010\n",
      "Iter=5232640/epoch=13, Loss=0.433401, Triplet loss=2.048486, Training Accuracy=0.828125, lr=0.000010\n",
      "Iter=5242880/epoch=13, Loss=0.614284, Triplet loss=2.002968, Training Accuracy=0.750000, lr=0.000010\n",
      "Iter=5253120/epoch=13, Loss=0.372249, Triplet loss=2.081631, Training Accuracy=0.828125, lr=0.000010\n",
      "Iter=5263360/epoch=13, Loss=0.403025, Triplet loss=2.048522, Training Accuracy=0.843750, lr=0.000010\n",
      "Iter=5273600/epoch=13, Loss=0.546914, Triplet loss=2.125855, Training Accuracy=0.835938, lr=0.000010\n",
      "Iter=5283840/epoch=13, Loss=0.465959, Triplet loss=1.994748, Training Accuracy=0.726562, lr=0.000010\n",
      "Iter=5294080/epoch=13, Loss=0.365207, Triplet loss=2.026391, Training Accuracy=0.812500, lr=0.000010\n",
      "Iter=5304320/epoch=13, Loss=0.478629, Triplet loss=2.015347, Training Accuracy=0.812500, lr=0.000010\n",
      "Iter=5314560/epoch=13, Loss=0.367168, Triplet loss=2.037510, Training Accuracy=0.828125, lr=0.000010\n",
      "Iter=5324800/epoch=13, Loss=0.434010, Triplet loss=2.059684, Training Accuracy=0.843750, lr=0.000010\n",
      "Iter=5324800/epoch=13, Validation Accuracy=0.713700, Precision=0.671097, Recall=0.838200\n",
      "Iter=5335040/epoch=13, Loss=0.405575, Triplet loss=2.075574, Training Accuracy=0.812500, lr=0.000010\n",
      "Iter=5345280/epoch=13, Loss=0.300451, Triplet loss=2.073328, Training Accuracy=0.882812, lr=0.000010\n",
      "Iter=5355520/epoch=13, Loss=0.562075, Triplet loss=2.037437, Training Accuracy=0.789062, lr=0.000010\n",
      "Iter=5365760/epoch=13, Loss=0.383675, Triplet loss=1.997864, Training Accuracy=0.843750, lr=0.000010\n",
      "Iter=5376000/epoch=13, Loss=0.369435, Triplet loss=2.070643, Training Accuracy=0.890625, lr=0.000010\n",
      "Iter=5386240/epoch=13, Loss=0.338259, Triplet loss=2.070596, Training Accuracy=0.867188, lr=0.000010\n",
      "Iter=5396480/epoch=14, Loss=0.341854, Triplet loss=2.078913, Training Accuracy=0.851562, lr=0.000010\n",
      "Iter=5406720/epoch=14, Loss=0.478600, Triplet loss=1.993243, Training Accuracy=0.820312, lr=0.000010\n",
      "Iter=5416960/epoch=14, Loss=0.376974, Triplet loss=2.070596, Training Accuracy=0.804688, lr=0.000010\n",
      "Iter=5427200/epoch=14, Loss=0.339453, Triplet loss=2.081632, Training Accuracy=0.851562, lr=0.000010\n",
      "Iter=5437440/epoch=14, Loss=0.406782, Triplet loss=2.070599, Training Accuracy=0.820312, lr=0.000010\n",
      "Iter=5447680/epoch=14, Loss=0.487933, Triplet loss=2.028349, Training Accuracy=0.789062, lr=0.000010\n",
      "Iter=5457920/epoch=14, Loss=0.411777, Triplet loss=2.048486, Training Accuracy=0.859375, lr=0.000010\n",
      "Iter=5468160/epoch=14, Loss=0.519258, Triplet loss=2.016984, Training Accuracy=0.843750, lr=0.000010\n",
      "Iter=5478400/epoch=14, Loss=0.305068, Triplet loss=2.015340, Training Accuracy=0.859375, lr=0.000010\n",
      "Iter=5488640/epoch=14, Loss=0.396387, Triplet loss=1.904854, Training Accuracy=0.796875, lr=0.000010\n",
      "Iter=5498880/epoch=14, Loss=0.437634, Triplet loss=2.059550, Training Accuracy=0.781250, lr=0.000010\n",
      "Iter=5509120/epoch=14, Loss=0.444394, Triplet loss=2.103728, Training Accuracy=0.851562, lr=0.000010\n",
      "Iter=5519360/epoch=14, Loss=0.457464, Triplet loss=2.015341, Training Accuracy=0.781250, lr=0.000010\n",
      "Iter=5529600/epoch=14, Loss=0.427093, Triplet loss=2.008445, Training Accuracy=0.812500, lr=0.000010\n",
      "Iter=5529600/epoch=14, Validation Accuracy=0.708800, Precision=0.661961, Recall=0.853400\n",
      "Iter=5539840/epoch=14, Loss=0.385181, Triplet loss=2.136874, Training Accuracy=0.843750, lr=0.000010\n",
      "Iter=5550080/epoch=14, Loss=0.339761, Triplet loss=2.015926, Training Accuracy=0.867188, lr=0.000010\n",
      "Iter=5560320/epoch=14, Loss=0.423725, Triplet loss=2.091362, Training Accuracy=0.843750, lr=0.000010\n",
      "Iter=5570560/epoch=14, Loss=0.328870, Triplet loss=2.103693, Training Accuracy=0.843750, lr=0.000010\n",
      "Iter=5580800/epoch=14, Loss=0.440526, Triplet loss=2.059535, Training Accuracy=0.804688, lr=0.000010\n",
      "Iter=5591040/epoch=14, Loss=0.403991, Triplet loss=2.059539, Training Accuracy=0.828125, lr=0.000010\n",
      "Iter=5601280/epoch=14, Loss=0.375973, Triplet loss=2.048489, Training Accuracy=0.867188, lr=0.000010\n",
      "Iter=5611520/epoch=14, Loss=0.384728, Triplet loss=1.993243, Training Accuracy=0.820312, lr=0.000010\n",
      "Iter=5621760/epoch=14, Loss=0.453238, Triplet loss=1.982194, Training Accuracy=0.812500, lr=0.000010\n",
      "Iter=5632000/epoch=14, Loss=0.371501, Triplet loss=2.048486, Training Accuracy=0.820312, lr=0.000010\n",
      "Iter=5642240/epoch=14, Loss=0.379955, Triplet loss=2.059578, Training Accuracy=0.812500, lr=0.000010\n",
      "Iter=5652480/epoch=14, Loss=0.336109, Triplet loss=2.037813, Training Accuracy=0.859375, lr=0.000010\n",
      "Iter=5662720/epoch=14, Loss=0.302491, Triplet loss=2.037499, Training Accuracy=0.882812, lr=0.000010\n",
      "Iter=5672960/epoch=14, Loss=0.304823, Triplet loss=2.084918, Training Accuracy=0.875000, lr=0.000010\n",
      "Iter=5683200/epoch=14, Loss=0.431950, Triplet loss=2.048488, Training Accuracy=0.820312, lr=0.000010\n",
      "Iter=5693440/epoch=14, Loss=0.380268, Triplet loss=2.026389, Training Accuracy=0.835938, lr=0.000010\n",
      "Iter=5703680/epoch=14, Loss=0.383772, Triplet loss=2.059542, Training Accuracy=0.835938, lr=0.000010\n",
      "Iter=5713920/epoch=14, Loss=0.333152, Triplet loss=2.117572, Training Accuracy=0.882812, lr=0.000010\n",
      "Iter=5724160/epoch=14, Loss=0.403397, Triplet loss=2.092680, Training Accuracy=0.812500, lr=0.000010\n",
      "Iter=5734400/epoch=14, Loss=0.481626, Triplet loss=2.038496, Training Accuracy=0.773438, lr=0.000010\n",
      "Iter=5734400/epoch=14, Validation Accuracy=0.708600, Precision=0.663071, Recall=0.848200\n",
      "Iter=5744640/epoch=14, Loss=0.443438, Triplet loss=1.993435, Training Accuracy=0.820312, lr=0.000010\n",
      "Iter=5754880/epoch=14, Loss=0.313473, Triplet loss=2.041384, Training Accuracy=0.882812, lr=0.000010\n",
      "Model saved in file at iteration 5760128: checkpoints/tf_deepUD_tri_model_iter_45001_for_progressGAN.ckpt\n",
      "Iter=5765120/epoch=14, Loss=0.444511, Triplet loss=2.114612, Training Accuracy=0.812500, lr=0.000010\n",
      "Iter=5775360/epoch=14, Loss=0.412774, Triplet loss=2.092674, Training Accuracy=0.820312, lr=0.000010\n",
      "Optimization Finished!\n",
      "Model saved in file: checkpoints/tf_deepUD_tri_model_progressGAN.ckpt\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import os, pdb\n",
    "import cv2\n",
    "import numpy as np\n",
    "import random as rn\n",
    "import threading\n",
    "import time\n",
    "from sklearn import metrics\n",
    "import utils\n",
    "global n_classes\n",
    "import triplet_loss as tri\n",
    "import os.path\n",
    "\n",
    "\n",
    "'''\n",
    "#===========================================================================\n",
    "Parameters:\n",
    "        TRAIN_WO_SPEC_GAN: Excluding GAN. E.f. progressGAN means We dont include progressGAN for tranining phase\n",
    "        n_classes: Number of classes (2 for now. one for fake and one for real)\n",
    "        data_dir:  The path to the file list directory\n",
    "        image_dir: The path to the images directory (if the image list is stored in absoluate path, set this to './')\n",
    "        margin:    Marginal value in triplet loss function\n",
    "Data Preparation\n",
    "        All training image list should put on the subfolder 'data' named by train_wo_[TRAIN_WO_SPEC_GAN].txt, wheere\n",
    "        the text file should have image path with its label (which GAN) such that \n",
    "        image_path1 0\n",
    "        image_path2 1\n",
    "        image_path3 5\n",
    "        image_path4 0\n",
    "        The data list in validation set is the same structure with training set.\n",
    "#===========================================================================\n",
    "'''\n",
    "TRAIN_WO_SPEC_GAN = 'progressGAN'                         \n",
    "n_classes = 2\n",
    "data_dir = '../../DCGAN-LSGAN-WGAN-WGAN-GP-Tensorflow/data/'\n",
    "image_dir = '../../DCGAN-LSGAN-WGAN-WGAN-GP-Tensorflow/'\n",
    "batch_size = 128\n",
    "display_step = 80\n",
    "learning_rate = tf.placeholder(tf.float32)      # Learning rate to be fed\n",
    "lr = 1e-4     \n",
    "margin = 0.8\n",
    "\n",
    "#========================Mode basic components============================\n",
    "def activation(x,name=\"activation\"):\n",
    "    return tf.nn.swish(x, name=name)\n",
    "    \n",
    "def conv2d(name, l_input, w, b, s, p):\n",
    "    l_input = tf.nn.conv2d(l_input, w, strides=[1,s,s,1], padding=p, name=name)\n",
    "    l_input = l_input+b\n",
    "\n",
    "    return l_input\n",
    "\n",
    "def batchnorm(conv, isTraining, name='bn'):\n",
    "    return tf.layers.batch_normalization(conv, training=isTraining, name=\"bn\"+name)\n",
    "\n",
    "def initializer(in_filters, out_filters, name, k_size=3):\n",
    "    w1 = tf.get_variable(name+\"W\", [k_size, k_size, in_filters, out_filters], initializer=tf.truncated_normal_initializer())\n",
    "    b1 = tf.get_variable(name+\"B\", [out_filters], initializer=tf.truncated_normal_initializer())\n",
    "    return w1, b1\n",
    "  \n",
    "def residual_block(in_x, in_filters, out_filters, stride, isDownSampled, name, isTraining, k_size=3):\n",
    "    global ema_gp\n",
    "    # first convolution layer\n",
    "    if isDownSampled:\n",
    "      in_x = tf.nn.avg_pool(in_x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='VALID')\n",
    "      \n",
    "    x = batchnorm(in_x, isTraining, name=name+'FirstBn')\n",
    "    x = activation(x)\n",
    "    w1, b1 = initializer(in_filters, in_filters, name+\"first_res\", k_size=k_size)\n",
    "    x = conv2d(name+'r1', x, w1, b1, 1, \"SAME\")\n",
    "\n",
    "    # second convolution layer\n",
    "    x = batchnorm(x, isTraining, name=name+'SecondBn')\n",
    "    x = activation(x)\n",
    "    w2, b2 = initializer(in_filters, out_filters, name+\"Second_res\",k_size=k_size)\n",
    "    x = conv2d(name+'r2', x, w2, b2, 1, \"SAME\")\n",
    "    \n",
    "    if in_filters != out_filters:\n",
    "        difference = out_filters - in_filters\n",
    "        left_pad = difference // 2\n",
    "        right_pad = difference - left_pad\n",
    "        identity = tf.pad(in_x, [[0, 0], [0, 0], [0, 0], [left_pad, right_pad]])\n",
    "        return x + identity\n",
    "    else:\n",
    "        return in_x + x\n",
    "\n",
    "\n",
    "'''\n",
    "#===========================================================================\n",
    "Network architecture based on ResNet\n",
    "#===========================================================================\n",
    "'''      \n",
    "def ResNet(_X, isTraining):\n",
    "    global n_classes\n",
    "    w1 = tf.get_variable(\"initWeight\", [7, 7, 3, 64], initializer=tf.truncated_normal_initializer())\n",
    "    b1 = tf.get_variable(\"initBias\", [64], initializer=tf.truncated_normal_initializer())\n",
    "    initx = conv2d('conv1', _X, w1, b1, 4, \"VALID\")\n",
    "    \n",
    "    filters_num = [64,96,128]\n",
    "    block_num = [2,4,3]\n",
    "    l_cnt = 1\n",
    "    x = initx\n",
    "    \n",
    "    # ============Feature extraction network with kernel size 3x3============\n",
    "    \n",
    "    for i in range(len(filters_num)):\n",
    "        for j in range(block_num[i]):\n",
    "          \n",
    "            if ((j==block_num[i]-1) & (i<len(filters_num)-1)):\n",
    "                x = residual_block(x, filters_num[i], filters_num[i+1], 2, True, 'ResidualBlock%d'%(l_cnt), isTraining)\n",
    "                print('[L-%d] Build %dth connection layer %d from %d to %d channels' % (l_cnt, i, j, filters_num[i], filters_num[i+1]))\n",
    "            else:\n",
    "                x = residual_block(x, filters_num[i], filters_num[i], 1, False, 'ResidualBlock%d'%(l_cnt), isTraining)\n",
    "                print('[L-%d] Build %dth residual block %d with %d channels' % (l_cnt,i, j, filters_num[i]))\n",
    "            l_cnt +=1\n",
    "    \n",
    "    layer_33 = x\n",
    "    x = initx\n",
    "    \n",
    "    # ============Feature extraction network with kernel size 5x5============\n",
    "    for i in range(len(filters_num)):\n",
    "        for j in range(block_num[i]):\n",
    "          \n",
    "            if ((j==block_num[i]-1) & (i<len(filters_num)-1)):\n",
    "                x = residual_block(x, filters_num[i], filters_num[i+1], 2, True, 'Residual5Block%d'%(l_cnt), isTraining, k_size=5)\n",
    "                print('[L-%d] Build %dth connection layer %d from %d to %d channels' % (l_cnt, i, j, filters_num[i], filters_num[i+1]))\n",
    "            else:\n",
    "                x = residual_block(x, filters_num[i], filters_num[i], 1, False, 'Residual5Block%d'%(l_cnt), isTraining, k_size=5)\n",
    "                print('[L-%d] Build %dth residual block %d with %d channels' % (l_cnt,i, j, filters_num[i]))\n",
    "            l_cnt +=1\n",
    "    layer_55 = x\n",
    "    print(\"Layer33's shape\", layer_33.get_shape().as_list())\n",
    "    print(\"Layer55's shape\", layer_55.get_shape().as_list())\n",
    "\n",
    "    x = tf.concat([layer_33, layer_55], 3)\n",
    "    \n",
    "    # ============ Classifier Learning============\n",
    "    \n",
    "    x_shape = x.get_shape().as_list()\n",
    "    dense1 = x_shape[1]*x_shape[2]*x_shape[3]\n",
    "    W = tf.get_variable(\"featW\", [dense1, 128], initializer=tf.truncated_normal_initializer())\n",
    "    b = tf.get_variable(\"featB\", [128], initializer=tf.truncated_normal_initializer())\n",
    "    dense1 = tf.reshape(x, [-1, dense1])\n",
    "    feat = tf.nn.softmax(tf.matmul(dense1, W) + b)\n",
    "    \n",
    "    with tf.variable_scope('Final'):\n",
    "        x = batchnorm(x, isTraining, name='FinalBn')\n",
    "        x = activation(x)\n",
    "        wo, bo=initializer(filters_num[-1]*2, n_classes, \"FinalOutput\")\n",
    "        x = conv2d('final', x, wo, bo, 1, \"SAME\")\n",
    "        saliency = tf.argmax(x, 3)\n",
    "        x=tf.reduce_mean(x, [1, 2])\n",
    "\n",
    "        W = tf.get_variable(\"FinalW\", [n_classes, n_classes], initializer=tf.truncated_normal_initializer())\n",
    "        b = tf.get_variable(\"FinalB\", [n_classes], initializer=tf.truncated_normal_initializer())\n",
    "\n",
    "        out = tf.matmul(x, W) + b\n",
    "                            \n",
    "\n",
    "    return out, feat, saliency\n",
    "\n",
    "\n",
    "#==========================================================================\n",
    "#=============Reading data in multithreading manner========================\n",
    "#==========================================================================\n",
    "def read_labeled_image_list(image_list_file, training_img_dir):\n",
    "    f = open(image_list_file, 'r')\n",
    "    filenames = []\n",
    "    labels = []\n",
    "\n",
    "    for line in f:\n",
    "        filename, label = line[:-1].split(' ')\n",
    "        filename = training_img_dir+filename\n",
    "        filenames.append(filename)\n",
    "        labels.append(int(label))\n",
    "        \n",
    "    return filenames, labels\n",
    "    \n",
    "    \n",
    "def read_images_from_disk(input_queue, size1=64):\n",
    "    label = input_queue[1]\n",
    "    fn=input_queue[0]\n",
    "    file_contents = tf.read_file(input_queue[0])\n",
    "    example = tf.image.decode_jpeg(file_contents, channels=3)\n",
    "    \n",
    "    #example = tf.image.decode_png(file_contents, channels=3, name=\"dataset_image\") # png fo rlfw\n",
    "    example=tf.image.resize_images(example, [size1,size1])\n",
    "    return example, label, fn\n",
    "    \n",
    "def setup_inputs(sess, filenames, training_img_dir, image_size=64, crop_size=64, isTest=False, batch_size=128):\n",
    "    \n",
    "    # Read each image file\n",
    "    image_list, label_list = read_labeled_image_list(filenames, training_img_dir)\n",
    "\n",
    "    images = tf.cast(image_list, tf.string)\n",
    "    labels = tf.cast(label_list, tf.int64)\n",
    "     # Makes an input queue\n",
    "    if isTest is False:\n",
    "        isShuffle = True\n",
    "        numThr = 4\n",
    "    else:\n",
    "        isShuffle = False\n",
    "        numThr = 1\n",
    "        \n",
    "    input_queue = tf.train.slice_input_producer([images, labels], shuffle=isShuffle)\n",
    "    image, y,fn = read_images_from_disk(input_queue)\n",
    "\n",
    "    channels = 3\n",
    "    image.set_shape([None, None, channels])\n",
    "        \n",
    "    # Crop and other random augmentations\n",
    "    if isTest is False:\n",
    "        image = tf.image.random_flip_left_right(image)\n",
    "        image = tf.image.random_saturation(image, .95, 1.05)\n",
    "        image = tf.image.random_brightness(image, .05)\n",
    "        image = tf.image.random_contrast(image, .95, 1.05)\n",
    "        \n",
    "    image = tf.cast(image, tf.float32)/255.0\n",
    "    \n",
    "    image, y,fn = tf.train.batch([image, y, fn], batch_size=batch_size, capacity=batch_size*3, num_threads=numThr, name='labels_and_images')\n",
    "\n",
    "    tf.train.start_queue_runners(sess=sess)\n",
    "\n",
    "    return image, y, fn, len(label_list)\n",
    "\n",
    "'''\n",
    "Maia training function:\n",
    "\n",
    "\n",
    "'''\n",
    "if not os.path.isdir('logs'):\n",
    "    os.mkdir('logs')\n",
    "if not os.path.isdir('saliency_img'):\n",
    "    os.mkdir('saliency_img')\n",
    "if not os.path.isdir('logs/pair'):\n",
    "    os.mkdir('logs/pair')\n",
    "if not os.path.isdir('logs/pair/%s/'%(TRAIN_WO_SPEC_GAN)):\n",
    "    os.mkdir('logs/pair/%s/'%(TRAIN_WO_SPEC_GAN))\n",
    "\n",
    "                   # Learning rate start\n",
    "tst = tf.placeholder(tf.bool)\n",
    "iter = tf.placeholder(tf.int32)\n",
    "print('GO!!')\n",
    "\n",
    "\n",
    "# In[ ]:\n",
    "\n",
    "\n",
    "# Setup the tensorflow...\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "sess = tf.Session(config=config)\n",
    "\n",
    "print(\"Preparing the training & validation data...\")\n",
    "pth1 = os.path.join(data_dir, \"train_wo_%s.txt\"%(TRAIN_WO_SPEC_GAN))\n",
    "train_data, train_labels, filelist1, glen1 = setup_inputs(sess, pth1, image_dir, batch_size=batch_size)\n",
    "pth2 = os.path.join(data_dir, \"val_wo_%s.txt\"%(TRAIN_WO_SPEC_GAN))\n",
    "val_data, val_labels, filelist2, tlen1 = setup_inputs(sess,pth2, image_dir, batch_size=1000,isTest=True)\n",
    "print(\"Found %d training images, and %d validation images...\" % (glen1, tlen1))\n",
    "\n",
    "max_iter = glen1*15\n",
    "print(\"Preparing the training model with learning rate = %.5f...\" % (lr))\n",
    "\n",
    "# Initialize the model for training set and validation sets\n",
    "with tf.variable_scope(\"ResNet\") as scope:\n",
    "    pred, feat,_ = ResNet(train_data, True)\n",
    "    scope.reuse_variables()\n",
    "    valpred, _, saliency = ResNet(val_data, False)\n",
    "\n",
    "\n",
    "# Forming the triplet loss by hard-triplet sampler  \n",
    "with tf.name_scope('Triplet_loss'):\n",
    "    \n",
    "    sialoss = tri.batch_hard_triplet_loss(train_labels, feat, margin, squared=False)\n",
    "\n",
    "# Forming the cross-entropy loss and accuracy for classifier learning\n",
    "with tf.name_scope('Loss_and_Accuracy'):\n",
    "    update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "    with tf.control_dependencies(update_ops):\n",
    "        t_vars=tf.trainable_variables() \n",
    "        #t_vars=[var for var in t_vars if 'Final']\n",
    "        cost = tf.losses.sparse_softmax_cross_entropy(labels=train_labels, logits=pred)\n",
    "        optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost, var_list=t_vars)\n",
    "        sia_optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(sialoss)\n",
    "\n",
    "    correct_prediction = tf.equal(tf.argmax(pred, 1), train_labels)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    correct_prediction2 = tf.equal(tf.argmax(valpred, 1), val_labels)\n",
    "    accuracy2 = tf.reduce_mean(tf.cast(correct_prediction2, tf.float32))\n",
    "\n",
    "  \n",
    "tf.summary.scalar(\"Triplet_loss\", sialoss)\n",
    "tf.summary.scalar('Loss', cost)\n",
    "tf.summary.scalar('Training_Accuracy', accuracy)\n",
    "tf.summary.scalar('Validation_Accuracy', accuracy2)\n",
    "# In[ ]:\n",
    "\n",
    "\n",
    "saver = tf.train.Saver()\n",
    "init = tf.global_variables_initializer()\n",
    "sess.run(init)\n",
    "step = 0\n",
    "\n",
    "writer = tf.summary.FileWriter(\"logs/pair/%s/\"%(TRAIN_WO_SPEC_GAN), sess.graph)\n",
    "summaries = tf.summary.merge_all()\n",
    "\n",
    "print(\"We are going to train fake detector using ResNet based on triplet loss!!!\")\n",
    "start_lr = lr\n",
    "while (step * batch_size) < max_iter:\n",
    "    epoch1=np.floor((step*batch_size)/glen1)\n",
    "    if (((step*batch_size)%glen1 < batch_size) & (lr==1e-4) & (epoch1 >=3)):\n",
    "        lr /= 10\n",
    "    \n",
    "    if epoch1 <=3:\n",
    "        sess.run([sia_optimizer],  feed_dict={learning_rate: lr})\n",
    "    else:\n",
    "        sess.run([optimizer],  feed_dict={learning_rate: lr})\n",
    "        \n",
    "    if (step % 15000==1) & (step>15000):\n",
    "        save_path = saver.save(sess, \"checkpoints/tf_deepUD_tri_model_iter_%d_for_%s.ckpt\" % (step,TRAIN_WO_SPEC_GAN))\n",
    "        print(\"Model saved in file at iteration %d: %s\" % (step*batch_size,save_path))\n",
    "\n",
    "    if step>0 and step % display_step == 0:\n",
    "        # calculate the loss\n",
    "        loss, acc, summaries_string, sia_val = sess.run([cost, accuracy, summaries, sialoss])\n",
    "        print(\"Iter=%d/epoch=%d, Loss=%.6f, Triplet loss=%.6f, Training Accuracy=%.6f, lr=%f\" % (step*batch_size, epoch1 ,loss, sia_val, acc, lr))\n",
    "        writer.add_summary(summaries_string, step)\n",
    "    \n",
    "    if step>0 and (step % (display_step*20) == 0):\n",
    "        rounds = tlen1 // 1000\n",
    "        #pdb.set_trace()\n",
    "        valacc=[]\n",
    "        vis=[]\n",
    "        tis=[]\n",
    "        for k in range(rounds):\n",
    "            a2, vi, ti = sess.run([accuracy2, tf.argmax(valpred, 1), val_labels])\n",
    "            valacc.append(a2)\n",
    "            vis.append(vi)\n",
    "            tis.append(ti)\n",
    "        tis = np.reshape(np.asarray(tis), [-1])\n",
    "        vis = np.reshape(np.asarray(vis), [-1])\n",
    "        precision=metrics.precision_score(tis, vis) \n",
    "        recall=metrics.recall_score(tis, vis)\n",
    "        \n",
    "        sal, valimg = sess.run([saliency, val_data])\n",
    "        utils.batchsalwrite(valimg, sal, tis, vis, 'saliency_img/%s_Detected_'%(TRAIN_WO_SPEC_GAN))\n",
    "        \n",
    "\n",
    "        print(\"Iter=%d/epoch=%d, Validation Accuracy=%.6f, Precision=%.6f, Recall=%.6f\" % (step*batch_size, epoch1 , np.mean(valacc), precision, recall))\n",
    "\n",
    "  \n",
    "    step += 1\n",
    "print(\"Optimization Finished!\")\n",
    "save_path = saver.save(sess, \"checkpoints/tf_deepUD_tri_model_%s.ckpt\" % (TRAIN_WO_SPEC_GAN))\n",
    "print(\"Model saved in file: %s\" % save_path)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}