#python #tensorflow #machine-learning
#python #тензорный поток #машинное обучение
Вопрос:
Я обучаю набор данных MNIST, используя CNN в Google colab, и хочу сохранить модель с помощью pickle, и когда я пытаюсь сохранить модель, я получаю сообщение об ошибке не удается рассолить _thread.Объекты RLock
Мой код
import pickle
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import keras
from keras.datasets import mnist
from keras.utils import to_categorical
from keras.models import Sequential
from keras.layers import Conv2D , MaxPooling2D, Dense, Flatten,Dropout
from keras.optimizers import Adam
from keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import train_test_split
testRatio = 0.2
valRatio = 0.2
imageDimensions = (28,28,3)
batchSizeVal = 50
EPOCHS = 2
stepsPerEpoch = 2000
(X_train, y_train), (X_test, y_test) = mnist.load_data()
X_train, X_validation , y_train , y_validation = train_test_split(X_train, y_train, test_size= valRatio)
X_train = X_train.reshape((48000, 28, 28, 1))
X_test = X_test.reshape((10000, 28, 28, 1))
X_validation = X_validation.reshape((12000, 28, 28, 1))
dataGen = ImageDataGenerator(width_shift_range = 0.1,
height_shift_range = 0.1,
zoom_range = 0.2,
shear_range = 0.1,
rotation_range= 10)
dataGen.fit(X_train)
y_train = to_categorical(y_train,10)
y_test= to_categorical(y_test,10)
y_validation = to_categorical(y_validation,10)
def myModel():
noOfFiters = 60
sizeOfFilter1 = (5,5)
sizeOfFilter2 = (3,3)
sizeOfPool = (2,2)
noOfNode = 500
model = Sequential()
model.add((Conv2D(noOfFiters, sizeOfFilter1,input_shape=(imageDimensions[0]
,imageDimensions[1],
1),
activation = "relu")))
model.add((Conv2D(noOfFiters, sizeOfFilter1, activation = "relu")))
model.add(MaxPooling2D(pool_size=sizeOfPool))
model.add((Conv2D(noOfFiters//2, sizeOfFilter2, activation = "relu")))
model.add((Conv2D(noOfFiters//2, sizeOfFilter2, activation = "relu")))
model.add(MaxPooling2D(pool_size=sizeOfPool))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(noOfNode,activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(10,activation='softmax'))
model.compile(Adam(lr=0.001),loss='categorical_crossentropy',
metrics=['accuracy'])
return model
model = myModel()
history = model.fit(dataGen.flow(X_train, y_train,
batch_size= batchSizeVal),
steps_per_epoch = stepsPerEpoch,
epochs =EPOCHS,
validation_data = (X_validation,y_validation),
shuffle= True)
plt.figure(1)
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.legend(['training ', 'validation'])
plt.title("Loss")
plt.xlabel('epoch')
plt.figure(2)
plt.plot(history.history['accuracy'])
plt.plot(history.history['val_accuracy'])
plt.legend(['training ', 'validation'])
plt.title("Accuracy")
plt.xlabel('epoch')
plt.show()
score = model.evaluate(X_test,y_test,verbose=0)
print("Test Score = ",score[0])
print("Test Accuracy = ",score[1])
pickle_out = open("model_trained.pickle","wb" )
model = pickle.dump(model,pickle_out)
pickle_out.close()
Что я должен сделать, чтобы заставить его работать.
Я попытался изменить время выполнения на cpu, потому что я думал, что это вызвано gpu, но даже тогда это не работает
Ответ №1:
Keras не поддерживает Pickle для сериализации своих объектов (моделей). В принципе, если у объекта есть методы __getstate__
и __setstate__
, pickle будет использовать их для сериализации объекта. Проблема в том, что модель Keras не реализует их.
@Zach Moshe, предложенное исправление для решения этой проблемы. Для получения более подробной информации, пожалуйста, обратитесь к его блогу.
# Hotfix function
def make_keras_picklable():
def __getstate__(self):
model_str = ""
with tempfile.NamedTemporaryFile(suffix='.hdf5', delete=True) as fd:
save_model(self, fd.name, overwrite=True)
model_str = fd.read()
d = {'model_str': model_str}
return d
def __setstate__(self, state):
with tempfile.NamedTemporaryFile(suffix='.hdf5', delete=True) as fd:
fd.write(state['model_str'])
fd.flush()
model = load_model(fd.name)
self.__dict__ = model.__dict__
cls = Model
cls.__getstate__ = __getstate__
cls.__setstate__ = __setstate__
# Run the function
make_keras_picklable()
Пожалуйста, обратитесь к рабочему коду ниже
import pickle
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import keras
from keras.datasets import mnist
from keras.utils import to_categorical
from tensorflow.keras.models import Sequential, load_model, save_model, Model
from keras.layers import Conv2D , MaxPooling2D, Dense, Flatten,Dropout
from keras.optimizers import Adam
from keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import train_test_split
import tempfile
testRatio = 0.2
valRatio = 0.2
imageDimensions = (28,28,3)
batchSizeVal = 50
EPOCHS = 2
stepsPerEpoch = 2000
(X_train, y_train), (X_test, y_test) = mnist.load_data()
X_train, X_validation , y_train , y_validation = train_test_split(X_train, y_train, test_size= valRatio)
X_train = X_train.reshape((48000, 28, 28, 1))
X_test = X_test.reshape((10000, 28, 28, 1))
X_validation = X_validation.reshape((12000, 28, 28, 1))
# Hotfix function
def make_keras_picklable():
def __getstate__(self):
model_str = ""
with tempfile.NamedTemporaryFile(suffix='.hdf5', delete=True) as fd:
save_model(self, fd.name, overwrite=True)
model_str = fd.read()
d = {'model_str': model_str}
return d
def __setstate__(self, state):
with tempfile.NamedTemporaryFile(suffix='.hdf5', delete=True) as fd:
fd.write(state['model_str'])
fd.flush()
model = load_model(fd.name)
self.__dict__ = model.__dict__
cls = Model
cls.__getstate__ = __getstate__
cls.__setstate__ = __setstate__
# Run the function
make_keras_picklable()
dataGen = ImageDataGenerator(width_shift_range = 0.1,
height_shift_range = 0.1,
zoom_range = 0.2,
shear_range = 0.1,
rotation_range= 10)
dataGen.fit(X_train)
y_train = to_categorical(y_train,10)
y_test= to_categorical(y_test,10)
y_validation = to_categorical(y_validation,10)
def myModel():
noOfFiters = 60
sizeOfFilter1 = (5,5)
sizeOfFilter2 = (3,3)
sizeOfPool = (2,2)
noOfNode = 500
model = Sequential()
model.add((Conv2D(noOfFiters, sizeOfFilter1,input_shape=(imageDimensions[0]
,imageDimensions[1],
1),
activation = "relu")))
model.add((Conv2D(noOfFiters, sizeOfFilter1, activation = "relu")))
model.add(MaxPooling2D(pool_size=sizeOfPool))
model.add((Conv2D(noOfFiters//2, sizeOfFilter2, activation = "relu")))
model.add((Conv2D(noOfFiters//2, sizeOfFilter2, activation = "relu")))
model.add(MaxPooling2D(pool_size=sizeOfPool))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(noOfNode,activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(10,activation='softmax'))
model.compile(Adam(lr=0.001),loss='categorical_crossentropy',
metrics=['accuracy'])
return model
model = myModel()
history = model.fit(dataGen.flow(X_train, y_train,
batch_size= batchSizeVal),
steps_per_epoch = X_train.shape[0]//batchSizeVal,
epochs =EPOCHS,
validation_data = (X_validation,y_validation),
shuffle= True)
score = model.evaluate(X_test,y_test,verbose=0)
print("Test Score = ",score[0])
print("Test Accuracy = ",score[1])
with open('model.pkl', 'wb') as f:
pickle.dump(model, f)
Вывод:
Epoch 1/2
960/960 [==============================] - 338s 352ms/step - loss: 1.0066 - accuracy: 0.6827 - val_loss: 0.1417 - val_accuracy: 0.9536
Epoch 2/2
960/960 [==============================] - 338s 352ms/step - loss: 0.3542 - accuracy: 0.8905 - val_loss: 0.0935 - val_accuracy: 0.9719
Test Score = 0.07476004958152771
Test Accuracy = 0.9761999845504761