fit_generator returning None. How to do this properly?

[AlexMarshall12]

This is my code:

def extract_hypercolumn(model, layer_indexes, instance):
    layers = [model.layers[li].output for li in layer_indexes]
    get_feature = K.function([model.layers[0].input],layers)
    assert instance.shape == (1,3,224,224)
    feature_maps = get_feature([instance])
    hypercolumns = []
    for convmap in feature_maps:
        for fmap in convmap[0]:
            upscaled = scipy.misc.imresize(fmap, size=(224, 224),
                                        mode="F", interp='bilinear')
            hypercolumns.append(upscaled)

    return np.asarray(hypercolumns)

def generate_batch_from_hdf5():
    f = h5py.File("raw_image_tensors.h5","r")
    dset_X = f.get('X')
    dset_y = f.get('y')

    print dset_X.shape,dset_y.shape
    for i in range(dset_X.shape[0]):
        X = dset_X[i:i+1,:,:,:]
        X = np.tile(X,(1,3,1,1))
        hc = extract_hypercolumn(model,[3,8,15,22],X)
        yield dset_X[i:i+1,:,:,:],dset_y[i:i+1,:,:]

def VGG_16(weights_path='vgg16_weights.h5'):
    model = Sequential()
    model.add(ZeroPadding2D((1,1),input_shape=(3,224,224)))
    model.add(Convolution2D(64, 3, 3, activation='relu'))
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(64, 3, 3, activation='relu'))
    model.add(MaxPooling2D((2,2), strides=(2,2)))

    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(128, 3, 3, activation='relu'))
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(128, 3, 3, activation='relu'))
    model.add(MaxPooling2D((2,2), strides=(2,2)))

    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(256, 3, 3, activation='relu'))
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(256, 3, 3, activation='relu'))
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(256, 3, 3, activation='relu'))
    model.add(MaxPooling2D((2,2), strides=(2,2)))

    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(512, 3, 3, activation='relu'))
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(512, 3, 3, activation='relu'))
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(512, 3, 3, activation='relu'))
    model.add(MaxPooling2D((2,2), strides=(2,2)))

    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(512, 3, 3, activation='relu'))
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(512, 3, 3, activation='relu'))
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(512, 3, 3, activation='relu'))
    model.add(MaxPooling2D((2,2), strides=(2,2)))

    model.add(Flatten())
    model.add(Dense(4096, activation='relu'))
    model.add(Dropout(0.5))
    model.add(Dense(4096, activation='relu'))
    model.add(Dropout(0.5))
    model.add(Dense(1000, activation='softmax'))

    if weights_path:
        model.load_weights(weights_path)

    return model


model = VGG_16()
sgd = SGD(lr=0.1,decay=1e-6,momentum=0.9,nesterov=True)
model.compile(optimizer=sgd,loss='categorical_crossentropy')

def Colorize(weights_path=None):
    model = Sequential()
    # input: 100x100 images with 3 channels -> (3, 100, 100) tensors.
    # this applies 32 convolution filters of size 3x3 each.

    model.add(Convolution2D(512, 1, 1, border_mode='valid',input_shape=(960,224,224)))
    model.add(Activation('relu'))
    model.add(normalization.BatchNormalization())

    model.add(Convolution2D(256, 1, 1, border_mode='valid'))
    model.add(Activation('relu'))
    model.add(normalization.BatchNormalization())

    model.add(Convolution2D(112, 1, 1, border_mode='valid'))
    model.add(Activation('relu'))
    model.add(normalization.BatchNormalization())

    print "output shape: ",model.output_shape
    #softmax
    model.add(Reshape((112,224*224)))

    print "output_shape after reshaped: ",model.output_shape
    model.add(Activation('softmax'))

    if weights_path:
        model.load_weights(weights_path)

    return model

color = Colorize()    
color.compile(optimizer=sgd,loss='categorical_crossentropy',metrics=["accuracy"])
color.fit_generator(generate_batch_from_hdf5(),samples_per_epoch=5,nb_epoch=5)

So this actually works ---UNTIL you add in that line hc = extract_hypercolumn(model,[3,8,15,22],X). Because hc isn't being yielded, I don't understand why it would have anything to do with whether the loop works or not. The eventual goal is to yield hc, but this is a baby step in that direction. Unfortunately, when you run the code as is you get this traceback:

Using TensorFlow backend.
output shape:  (None, 112, 224, 224)
output_shape after reshaped:  (None, 112, 50176)
Epoch 1/5
(100, 1, 224, 224) (100, 112, 50176)
Exception in thread Thread-1:
Traceback (most recent call last):
  File "/Users/alex/anaconda2/lib/python2.7/threading.py", line 801, in __bootstrap_inner
    self.run()
  File "/Users/alex/anaconda2/lib/python2.7/threading.py", line 754, in run
    self.__target(*self.__args, **self.__kwargs)
  File "/Users/alex/anaconda2/lib/python2.7/site-packages/keras/engine/training.py", line 404, in data_generator_task
    generator_output = next(generator)
StopIteration

Traceback (most recent call last):
  File "load.py", line 143, in <module>
    color.fit_generator(generate_batch_from_hdf5(),samples_per_epoch=5,nb_epoch=5)
  File "/Users/alex/anaconda2/lib/python2.7/site-packages/keras/models.py", line 651, in fit_generator
    max_q_size=max_q_size)
  File "/Users/alex/anaconda2/lib/python2.7/site-packages/keras/engine/training.py", line 1358, in fit_generator
    'or (x, y). Found: ' + str(generator_output))
Exception: output of generator should be a tuple (x, y, sample_weight) or (x, y). Found: None

If you want to run this for yourself, you will have to download vgg16_weights.h5 for yourself - easily google-able but a large - 500MB file. Otherwise you can take my word for it that the extract_hypercolumn functions works as intended. Though you will still need raw_image_tensors.h5 (attached and quite small). In fact its funny because if you run this directly:

f = h5py.File("raw_image_tensors.h5","r")
dset_X = f.get('X')
dset_y = f.get('y')
for i in range(dset_X.shape[0]):
    X = dset_X[i:i+1,:,:,:]
    X = np.tile(X,(1,3,1,1))
    hc = extract_hypercolumn(model,[3,8,15,22],X)
    print hc,dset_y[i:i+1,:,:]

it prints out just fine - try it! Its like the combination of yielding something into fit_generator with that extract_hypercolumn thing is just too much for it to handle and it yields None. I hope this is replaceable enough, and if theres anything thats unclear, please don't hesitate to ask.

[saicoco]

/home/sai/anaconda2/bin/python /home/sai/code/frame_play/face_alone_BigBang.py Using TensorFlow backend. I tensorflow/stream_executor/dso_loader.cc:105] successfully opened CUDA library libcublas.so locally I tensorflow/stream_executor/dso_loader.cc:105] successfully opened CUDA library libcudnn.so locally I tensorflow/stream_executor/dso_loader.cc:105] successfully opened CUDA library libcufft.so locally I tensorflow/stream_executor/dso_loader.cc:105] successfully opened CUDA library libcuda.so.1 locally I tensorflow/stream_executor/dso_loader.cc:105] successfully opened CUDA library libcurand.so locally I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:900] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero I tensorflow/core/common_runtime/gpu/gpu_init.cc:102] Found device 0 with properties: name: GeForce GTX 960 major: 5 minor: 2 memoryClockRate (GHz) 1.291 pciBusID 0000:01:00.0 Total memory: 4.00GiB Free memory: 3.12GiB I tensorflow/core/common_runtime/gpu/gpu_init.cc:126] DMA: 0 I tensorflow/core/common_runtime/gpu/gpu_init.cc:136] 0: Y I tensorflow/core/common_runtime/gpu/gpu_device.cc:755] Creating TensorFlow device (/gpu:0) -> (device: 0, name: GeForce GTX 960, pci bus id: 0000:01:00.0) Epoch 1/50 shuffle Process Process-1: Traceback (most recent call last): File "/home/sai/anaconda2/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap self.run() File "/home/sai/anaconda2/lib/python2.7/multiprocessing/process.py", line 114, in run self._target(*self._args, **self._kwargs) File "/home/sai/anaconda2/lib/python2.7/site-packages/Keras-1.0.5-py2.7.egg/keras/engine/training.py", line 427, in data_generator_task generator_output = next(generator) File "/home/sai/code/frame_play/data_flow.py", line 21, in next return self.it.next() File "/home/sai/code/frame_play/data_flow.py", line 68, in image_generator img = img.resize((50, 50)) File "/home/sai/anaconda2/lib/python2.7/site-packages/PIL/Image.py", line 1526, in resize self.load() File "/home/sai/anaconda2/lib/python2.7/site-packages/PIL/ImageFile.py", line 235, in load raise_ioerror(e) File "/home/sai/anaconda2/lib/python2.7/site-packages/PIL/ImageFile.py", line 59, in raise_ioerror raise IOError(message + " when reading image file") IOError: broken data stream when reading image file Traceback (most recent call last): File "/home/sai/code/frame_play/face_alone_BigBang.py", line 17, in <module> validation_data=test_generator, nb_val_samples=38584, pickle_safe=True) File "/home/sai/anaconda2/lib/python2.7/site-packages/Keras-1.0.5-py2.7.egg/keras/models.py", line 683, in fit_generator pickle_safe=pickle_safe) File "/home/sai/anaconda2/lib/python2.7/site-packages/Keras-1.0.5-py2.7.egg/keras/engine/training.py", line 1419, in fit_generator 'or (x, y). Found: ' + str(generator_output)) Exception: output of generator should be a tuple (x, y, sample_weight) or (x, y). Found: None
the same to you, I just see someone talk about this, there is a problem about thread-safe.

[michaelosthege]

I also got
Exception: output of generator should be a tuple (x, y, sample_weight) or (x, y). Found: None
The problem is in line 498 of image.py where the labels are filtered by an array of indices.
If the labels are actually an HDF5 object, only sorted lists are supported for slicing. A few lines above the input is not sliced but accessed in a loop (that's why it works for X).

A workaround is to convert all labels to a numpy array when you load your dataset.

The fix will be to make a sorted list out of index_array
cheers

[stale]

This issue has been automatically marked as stale because it has not had recent activity. It will be closed after 30 days if no further activity occurs, but feel free to re-open a closed issue if needed.