TensorFlow: TypeError when itterating through model architectures

Question

I have a script which goes through a simple 2d CNN and I'm trying to run through a range of different values for the number of layers and neurons per layer in my final dense layers after the convolution.

I have the models being produced correctly (which I can see through model.summary()), but when it comes to training I'm getting a type error after the first model is trained, just before it starts on a second model (shown below).

  File "/media/physsh/Working SSD/Python_scripts/Machine_Learning/First_attempt/2dcnn-SiB.py", line 222, in <module>
    validation_data=validation_generator)
  File "/home/physsh/Softwares/miniconda3/envs/Dylan/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py", line 1064, in fit
    steps_per_execution=self._steps_per_execution)
  File "/home/physsh/Softwares/miniconda3/envs/Dylan/lib/python3.7/site-packages/tensorflow/python/keras/engine/data_adapter.py", line 1105, in __init__
    epochs=epochs - initial_epoch,
TypeError: unsupported operand type(s) for -: 'range' and 'int'

I'm not changing anything between runs, so I don't know if it's some variable being left over between them? Either way I'm not sure how to proceed from here other than running each manually which is obviously not optimal.

Below is my whole script if it helps.

from keras.preprocessing.image import ImageDataGenerator
import tensorflow as tf

import matplotlib.pyplot as plt
import pandas as pd
from pathlib import Path
import os
import random
from shutil import copyfile

def create_train_val_dirs(root_path):
    """
    Creates directories for the train and test sets
    
    Args:
        root_path (string) - the base directory path to create subdirectories from
    
    Returns:
        None
    """  

    os.makedirs(root_path)

    val_path = os.path.join(root_path, 'validation')
    train_path = os.path.join(root_path, 'training')
    os.makedirs(train_path)
    os.makedirs(val_path)

    os.makedirs(os.path.join(val_path, 'Good'))
    os.makedirs(os.path.join(val_path, 'Bad'))
    os.makedirs(os.path.join(train_path, 'Good'))
    os.makedirs(os.path.join(train_path, 'Bad'))

def split_data(SOURCE_DIR, TRAINING_DIR, VALIDATION_DIR, SPLIT_SIZE):

    """
    Splits the data into train and test sets
    
    Args:
        SOURCE_DIR (string): directory path containing the images
        TRAINING_DIR (string): directory path to be used for training
        VALIDATION_DIR (string): directory path to be used for validation
        SPLIT_SIZE (float): proportion of the dataset to be used for training
        
    Returns:
        None
    """

    im_list = random.sample(os.listdir(SOURCE_DIR), len(os.listdir(SOURCE_DIR)))

    for index, im in enumerate(im_list):
        if os.path.getsize(os.path.join(SOURCE_DIR, im)) == 0:
            print(f"{im} is zero length, so ignoring.")
            continue
        if index <= SPLIT_SIZE*len(im_list):
            copyfile(os.path.join(SOURCE_DIR, im), os.path.join(TRAINING_DIR, im))
        else:
            copyfile(os.path.join(SOURCE_DIR, im), os.path.join(VALIDATION_DIR, im))

def train_val_generators(TRAINING_DIR, VALIDATION_DIR, batch_size, input_size):
    """
    Creates the training and validation data generators
  
    Args:
        TRAINING_DIR (string): directory path containing the training images
        VALIDATION_DIR (string): directory path containing the testing/validation images
        batch_size (integer): Integer number of batches (amount of images trained per loop).
        input_size (tensor: (int, int)): Tensor containing the size of the input image, or 
        preferred input size (if different to the actual size it will be scaled). 
    
    Returns:
        train_generator, validation_generator - tuple containing the generators
    """


    # Instantiate the ImageDataGenerator class (don't forget to set the arguments to augment the images)
    train_datagen = ImageDataGenerator(rescale=1./255.0,
                                    #  rotation_range=40,
                                    #  width_shift_range=0.2,
                                    #  height_shift_range=0.2,
                                    #  shear_range=0.2,
                                    #  zoom_range=0.2,
                                    horizontal_flip=True,
                                    vertical_flip=True,
                                    #  fill_mode='nearest'
                                    )

    # Pass in the appropriate arguments to the flow_from_directory method
    train_generator = train_datagen.flow_from_directory(directory=TRAINING_DIR,
                                                        batch_size=batch_size,
                                                        class_mode='binary',
                                                        target_size=target_size,
                                                        color_mode= 'grayscale')

    # Instantiate the ImageDataGenerator class (don't forget to set the rescale argument)
    validation_datagen = ImageDataGenerator(rescale=1./255.0)

    # Pass in the appropriate arguments to the flow_from_directory method
    validation_generator = validation_datagen.flow_from_directory(directory=VALIDATION_DIR,
                                                                batch_size=batch_size,
                                                                class_mode='binary',
                                                                target_size=target_size,
                                                                color_mode= 'grayscale')
    ### END CODE HERE
    return train_generator, validation_generator

# set tf_xla_enable_xla_devices flags
os.environ['TF_XLA_FLAGS'] = '--tf_xla_enable_xla_devices'

# Define paths
SOURCE_DIR = '/media/physsh/Working SSD/Extracted_images/B-Si/300_sample/Splits/Binary_1/Input_data'
ROOT_DIR = '/media/physsh/Working SSD/Extracted_images/B-Si/300_sample/Splits/Binary_1/ForRuns'

GOOD_SOURCE_DIR = os.path.join(SOURCE_DIR, "Good")
BAD_SOURCE_DIR = os.path.join(SOURCE_DIR,"Bad")

TRAINING_DIR = os.path.join(ROOT_DIR, 'training')
VALIDATION_DIR = os.path.join(ROOT_DIR, 'validation')

TRAINING_GOOD_DIR = os.path.join(TRAINING_DIR, "Good/")
VALIDATION_GOOD_DIR = os.path.join(VALIDATION_DIR, "Good/")

TRAINING_BAD_DIR = os.path.join(TRAINING_DIR, "Bad/")
VALIDATION_BAD_DIR = os.path.join(VALIDATION_DIR, "Bad/")

try:
  create_train_val_dirs(root_path=ROOT_DIR)
  print(f'Creating directory at {ROOT_DIR}...')
except FileExistsError:
  print(f"{ROOT_DIR} already exists.")

# Empty directories in case you run this cell multiple times
if len(os.listdir(TRAINING_BAD_DIR)) > 0:
    for file in os.scandir(TRAINING_BAD_DIR):
        os.remove(file.path)
if len(os.listdir(TRAINING_GOOD_DIR)) > 0:
    for file in os.scandir(TRAINING_GOOD_DIR):
        os.remove(file.path)
if len(os.listdir(VALIDATION_BAD_DIR)) > 0:
    for file in os.scandir(VALIDATION_BAD_DIR):
        os.remove(file.path)
if len(os.listdir(VALIDATION_GOOD_DIR)) > 0:
    for file in os.scandir(VALIDATION_GOOD_DIR):
        os.remove(file.path)

# Define proportion of images used for training
split_size = .9

# Run the function
# NOTE: Messages about zero length images should be printed out
split_data(GOOD_SOURCE_DIR, TRAINING_GOOD_DIR, VALIDATION_GOOD_DIR, split_size)
split_data(BAD_SOURCE_DIR, TRAINING_BAD_DIR, VALIDATION_BAD_DIR, split_size)

batch_size = 20
target_size = (140, 140)
epochs = 1

train_generator, validation_generator = train_val_generators(TRAINING_DIR, VALIDATION_DIR, batch_size=batch_size, input_size=target_size)

neurons = [32,64,128,256,512,1024,2048] 
# Apparently a maximum of two layers is all that we would ever need - I'll look into the theory later. 
num_layers = [1,2]

for i in neurons:
  for j in num_layers:
    tmp_i = i
    # Set initial layers - Usually input, conv and flatten before dense layers
    layers = [
        tf.keras.layers.Conv2D(16, (3,3), activation='relu', input_shape= (140,140,1)),
        tf.keras.layers.MaxPooling2D((2,2)),
        tf.keras.layers.Conv2D(32, (3,3), activation='relu'),
        tf.keras.layers.MaxPooling2D((2,2)),
        tf.keras.layers.Conv2D(64, (3,3), activation='relu'),
        tf.keras.layers.MaxPooling2D((2,2)),
        tf.keras.layers.Conv2D(128, (3,3), activation='relu'),
        tf.keras.layers.MaxPooling2D((2,2)),
        tf.keras.layers.Flatten()]

    for k in range(1, j+1):
      layers.append(tf.keras.layers.Dense(tmp_i, activation='relu'))
      layers.append(tf.keras.layers.Dropout(0.5))
      tmp_i *= 2

    # Add in the output layer
    layers.append(tf.keras.layers.Dense(1, activation='sigmoid'))

    model = tf.keras.models.Sequential(layers)
    model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])

    model.summary()

    # Bigin training!
    history = model.fit(train_generator,
                        epochs=epochs,
                        verbose=1,
                        validation_data=validation_generator)


    # Save the model, history and accuracy plot
    save_name = f'{epochs}epochs_{j}layers_{i}Starting'
    Models_dir = '/media/physsh/Working SSD/Extracted_images/B-Si/300_sample/Splits/Binary_1/Trained_models'

    acc = history.history['accuracy']
    val_acc = history.history['val_accuracy']
    loss = history.history['loss']
    val_loss = history.history['val_loss']

    epochs = range(len(acc))

    plt.plot(epochs, acc, 'r', label='Training accuracy')
    plt.plot(epochs, val_acc, 'b', label='Validation accuracy')
    plt.title('Training and validation accuracy')
    plt.legend(loc=0)

    plt.savefig(os.path.join(Models_dir, f'{save_name}.png'))
    model.save(os.path.join(Models_dir, f'{save_name}.h5'))


    # convert the history.history dict to a pandas DataFrame:     
    hist_df = pd.DataFrame(history.history) 

    # or save to csv: 
    hist_csv_file = os.path.join(Models_dir, f'{save_name}.csv')
    with open(hist_csv_file, mode='w') as f:
        hist_df.to_csv(f)

The task is a simple binary classification task and with the few tests I've done I'm already getting decent results, but I'd like to see something on how changing the number of neurons and number of layers changes things in terms of accuracy.

Thanks in advance for any help.

Answer 1

You use the same variable name epochs in the fit method and also in the plots after : epochs = range(len(acc)), that is why you get the error for the second model fitting.