#python-3.x #tensorflow #visual-studio-code
#python-3.x #тензорный поток #visual-studio-code
Вопрос:
- Windows 10
- RTX 3070
- CUDA 11.1
- cuDNN 8.0.5 (для CUDA 11.1)
- python 3.8.5
- tf-nightly-gpu 2.5.0.dev20201113
- использование среды Anaconda
Моя программа работала нормально до обновления до 3070, однако я заранее использовал обычный tensorflow-gpu. Я получаю Could not create cudnn handle: CUDNN_STATUS_ALLOC_FAILED , а также кучу других предупреждений и ошибок
Вот мой полный код и вывод:
import numpy as np
import os
import tensorflow as tf
from keras.callbacks import EarlyStopping
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.layers import Conv2D, MaxPooling2D, BatchNormalization, LeakyReLU
from keras.models import Sequential
from keras.preprocessing.image import ImageDataGenerator
from keras.regularizers import l2
IMG_SIZE = 350
Version = 1
batch_size = 8
val_aug = ImageDataGenerator(rescale=1/255)
aug = ImageDataGenerator(
rescale=1/255,
rotation_range=30,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range=0.2,
zoom_range=0.2,
channel_shift_range=25,
horizontal_flip=True,
fill_mode='constant')
train_gen = aug.flow_from_directory('F:/Storage/DataSet_Bal/Train',
target_size=(IMG_SIZE, IMG_SIZE),
batch_size=batch_size,
class_mode='binary',
shuffle=True)
val_gen = val_aug.flow_from_directory('F:/Storage/DataSet_Bal/Val',
target_size=(IMG_SIZE, IMG_SIZE),
batch_size=batch_size,
class_mode='binary',
shuffle=True)
model = Sequential()
model.add(Conv2D(64, 3, strides=(1,1), padding = 'same', activation = 'relu', input_shape = (IMG_SIZE, IMG_SIZE, 3)))
model.add(BatchNormalization())
model.add(Conv2D(64, 3, strides=(1,1), activation = 'relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2), strides=2))
model.add(BatchNormalization())
model.add(Conv2D(128, 3, strides=(1,1), activation = 'relu'))
model.add(BatchNormalization())
model.add(Conv2D(128, 3, strides=(1,1), activation = 'relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2), strides=2))
model.add(BatchNormalization())
model.add(Conv2D(256, 3, strides=(1,1), activation = 'relu'))
model.add(BatchNormalization())
model.add(Conv2D(256, 3, strides=(1,1), activation = 'relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2), strides=2))
model.add(BatchNormalization())
model.add(Conv2D(512, 3, strides=(1,1), activation = 'relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(BatchNormalization())
model.add(Flatten())
model.add(Dense(128, activation = 'relu'))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(32, activation = 'relu'))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(1, activation = 'sigmoid'))
model.compile(loss = "binary_crossentropy", optimizer = 'adam', metrics = ['accuracy'])
#model.summary()
earlyStop = EarlyStopping(monitor = 'val_accuracy', min_delta = 0.0001, patience = 50, restore_best_weights = True)
model.fit(
train_gen,
workers=8,
epochs= 250,
validation_data=val_gen,
callbacks=earlyStop,
verbose=2)
model.save(f'F:/Storage/TrainedVersions/YiffModel{Version}')
Вывод:
2020-11-13 09:37:16.194295: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library cudart64_110.dll
2020-11-13 09:37:18.603135: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
2020-11-13 09:37:18.603871: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library nvcuda.dll
2020-11-13 09:37:18.627727: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1724] Found device 0 with properties:
pciBusID: 0000:01:00.0 name: GeForce RTX 3070 computeCapability: 8.6
coreClock: 1.725GHz coreCount: 46 deviceMemorySize: 8.00GiB deviceMemoryBandwidth: 417.29GiB/s
2020-11-13 09:37:18.628118: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library cudart64_110.dll
2020-11-13 09:37:18.643771: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library cublas64_11.dll
2020-11-13 09:37:18.643991: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library cublasLt64_11.dll
2020-11-13 09:37:18.648605: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library cufft64_10.dll
2020-11-13 09:37:18.649976: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library curand64_10.dll
2020-11-13 09:37:18.660679: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library cusolver64_10.dll
2020-11-13 09:37:18.664522: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library cusparse64_11.dll
2020-11-13 09:37:18.665411: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library cudnn64_8.dll
2020-11-13 09:37:18.665724: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1866] Adding visible gpu devices: 0
2020-11-13 09:37:18.666681: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2020-11-13 09:37:18.667597: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1724] Found device 0 with properties:
pciBusID: 0000:01:00.0 name: GeForce RTX 3070 computeCapability: 8.6
coreClock: 1.725GHz coreCount: 46 deviceMemorySize: 8.00GiB deviceMemoryBandwidth: 417.29GiB/s
2020-11-13 09:37:18.668060: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library cudart64_110.dll
2020-11-13 09:37:18.668326: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library cublas64_11.dll
2020-11-13 09:37:18.668641: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library cublasLt64_11.dll
2020-11-13 09:37:18.668905: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library cufft64_10.dll
2020-11-13 09:37:18.669169: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library curand64_10.dll
2020-11-13 09:37:18.669409: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library cusolver64_10.dll
2020-11-13 09:37:18.669644: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library cusparse64_11.dll
2020-11-13 09:37:18.669857: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library cudnn64_8.dll
2020-11-13 09:37:18.670141: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1866] Adding visible gpu devices: 0
2020-11-13 09:37:19.291798: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1265] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-11-13 09:37:19.292056: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1271] 0
2020-11-13 09:37:19.292207: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1284] 0: N
2020-11-13 09:37:19.292500: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1410] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 6553 MB memory) -> physical GPU (device: 0, name: GeForce RTX 3070, pci bus id: 0000:01:00.0, compute capability: 8.6)
2020-11-13 09:37:19.293799: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
WARNING:tensorflow:From e:PYTHONYiffMinerTrainYIFF.py:18: The name tf.keras.backend.set_session is deprecated. Please use tf.compat.v1.keras.backend.set_session instead.
2020-11-13 09:37:19.310193: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1724] Found device 0 with properties:
pciBusID: 0000:01:00.0 name: GeForce RTX 3070 computeCapability: 8.6
coreClock: 1.725GHz coreCount: 46 deviceMemorySize: 8.00GiB deviceMemoryBandwidth: 417.29GiB/s
2020-11-13 09:37:19.310564: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library cudart64_110.dll
2020-11-13 09:37:19.310818: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library cublas64_11.dll
2020-11-13 09:37:19.311052: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library cublasLt64_11.dll
2020-11-13 09:37:19.311288: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library cufft64_10.dll
2020-11-13 09:37:19.311478: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library curand64_10.dll
2020-11-13 09:37:19.311714: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library cusolver64_10.dll
2020-11-13 09:37:19.311924: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library cusparse64_11.dll
2020-11-13 09:37:19.312137: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library cudnn64_8.dll
2020-11-13 09:37:19.312375: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1866] Adding visible gpu devices: 0
2020-11-13 09:37:19.312561: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1265] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-11-13 09:37:19.312777: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1271] 0
2020-11-13 09:37:19.312905: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1284] 0: N
2020-11-13 09:37:19.313141: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1410] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 6553 MB memory) -> physical GPU (device: 0, name: GeForce RTX 3070, pci bus id: 0000:01:00.0, compute capability: 8.6)
2020-11-13 09:37:19.313565: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
2020-11-13 09:37:20.174517: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:126] None of the MLIR optimization passes are enabled (registered 2)
2020-11-13 09:37:22.133669: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library cudnn64_8.dll
2020-11-13 09:37:23.042617: E tensorflow/stream_executor/cuda/cuda_dnn.cc:349] Could not create cudnn handle: CUDNN_STATUS_ALLOC_FAILED
2020-11-13 09:37:23.043316: E tensorflow/stream_executor/cuda/cuda_dnn.cc:349] Could not create cudnn handle: CUDNN_STATUS_ALLOC_FAILED
2020-11-13 09:37:23.043512: W tensorflow/core/framework/op_kernel.cc:1763] OP_REQUIRES failed at conv_ops_fused_impl.h:697 : Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
C:Userscircuanaconda3envstf2python.exe
Found 18 images belonging to 2 classes.
Found 43 images belonging to 2 classes.
Epoch 1/250
Traceback (most recent call last):
File "e:PYTHONYiffMinerTrainYIFF.py", line 89, in <module>
model.fit(
File "C:Userscircuanaconda3envstf2libsite-packagestensorflowpythonkerasenginetraining.py", line 1103, in fit
tmp_logs = self.train_function(iterator)
File "C:Userscircuanaconda3envstf2libsite-packagestensorflowpythoneagerdef_function.py", line 784, in __call__
result = self._call(*args, **kwds)
File "C:Userscircuanaconda3envstf2libsite-packagestensorflowpythoneagerdef_function.py", line 844, in _call
return self._stateless_fn(*args, **kwds)
File "C:Userscircuanaconda3envstf2libsite-packagestensorflowpythoneagerfunction.py", line 2971, in __call__
return graph_function._call_flat(
File "C:Userscircuanaconda3envstf2libsite-packagestensorflowpythoneagerfunction.py", line 1947, in _call_flat
return self._build_call_outputs(self._inference_function.call(
File "C:Userscircuanaconda3envstf2libsite-packagestensorflowpythoneagerfunction.py", line 556, in call
outputs = execute.execute(
File "C:Userscircuanaconda3envstf2libsite-packagestensorflowpythoneagerexecute.py", line 59, in quick_execute
tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
tensorflow.python.framework.errors_impl.UnknownError: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
[[node sequential/conv2d/Relu (defined at e:PYTHONYiffMinerTrainYIFF.py:89) ]] [Op:__inference_train_function_3249]
Function call stack:
train_function
2020-11-13 09:37:23.152884: W tensorflow/core/kernels/data/generator_dataset_op.cc:107] Error occurred when finalizing GeneratorDataset iterator: Failed precondition: Python interpreter state is not initialized. The process may be terminated.
[[{{node PyFunc}}]]
Ответ №1:
Включите увеличение объема памяти для вашего графического процессора.
for device in tf.config.experimental.list_physical_devices("GPU"):
tf.config.experimental.set_memory_growth(device, True)