Crash when trying to run RetinaNet

Hi,

I tried running the beta RetinaNet notebook, but ran into an error in Cell 4.
The same data set works fine for YOLOv2, so I think that this should not be an issue.

Note that in Cell 3.3, I changed the the location of checkpoints_path, as this was not actually stored in model_path, but directly in the content folder:

#checkpoints_path = os.path.join(model_path,'checkpoint')
  checkpoints_path = '/content/ssd_resnet50_v1_fpn_640x640_coco17_tpu-8/checkpoint'

Help would be very much appreciated!

This is the error message I get:

InternalError                             Traceback (most recent call last)
<ipython-input-7-a129353f211e> in <module>()
     22 start = time.time()
     23 
---> 24 train(pretrained_model, verbose = Verbose)
     25 
     26 # Displaying the time elapsed for training

11 frames
<ipython-input-1-0daf2db5b026> in train(model, verbose)
    742 
    743 def train(model, verbose = True):
--> 744   train_image_tensors, gt_classes_one_hot_tensors, gt_box_tensors = prepare_data_to_train(augmented_training_source, df_anno, gt_boxes, gt_classes)
    745   print('Done training data preprocessing.')
    746 

<ipython-input-1-0daf2db5b026> in prepare_data_to_train(training_img_path, df, data_gt_boxes, data_gt_classes)
    729     img = cv2.cvtColor(img,cv2.COLOR_GRAY2RGB)
    730     train_image_tensors.append(tf.expand_dims(tf.convert_to_tensor(
--> 731       img, dtype=tf.float32), axis=0))
    732 
    733     predicted_classes = np.zeros(shape=[data_gt_boxes[index].shape[0]], dtype=np.int32)

/usr/local/lib/python3.7/dist-packages/tensorflow/python/util/dispatch.py in wrapper(*args, **kwargs)
    204     """Call target, and fall back on dispatchers if there is a TypeError."""
    205     try:
--> 206       return target(*args, **kwargs)
    207     except (TypeError, ValueError):
    208       # Note: convert_to_eager_tensor currently raises a ValueError, not a

/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/ops.py in convert_to_tensor_v2_with_dispatch(value, dtype, dtype_hint, name)
   1429   """
   1430   return convert_to_tensor_v2(
-> 1431       value, dtype=dtype, dtype_hint=dtype_hint, name=name)
   1432 
   1433 

/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/ops.py in convert_to_tensor_v2(value, dtype, dtype_hint, name)
   1439       name=name,
   1440       preferred_dtype=dtype_hint,
-> 1441       as_ref=False)
   1442 
   1443 

/usr/local/lib/python3.7/dist-packages/tensorflow/python/profiler/trace.py in wrapped(*args, **kwargs)
    161         with Trace(trace_name, **trace_kwargs):
    162           return func(*args, **kwargs)
--> 163       return func(*args, **kwargs)
    164 
    165     return wrapped

/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/ops.py in convert_to_tensor(value, dtype, name, as_ref, preferred_dtype, dtype_hint, ctx, accepted_result_types)
   1564 
   1565     if ret is None:
-> 1566       ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
   1567 
   1568     if ret is NotImplemented:

/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/tensor_conversion_registry.py in _default_conversion_function(***failed resolving arguments***)
     50 def _default_conversion_function(value, dtype, name, as_ref):
     51   del as_ref  # Unused.
---> 52   return constant_op.constant(value, dtype, name=name)
     53 
     54 

/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/constant_op.py in constant(value, dtype, shape, name)
    270   """
    271   return _constant_impl(value, dtype, shape, name, verify_shape=False,
--> 272                         allow_broadcast=True)
    273 
    274 

/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/constant_op.py in _constant_impl(value, dtype, shape, name, verify_shape, allow_broadcast)
    281       with trace.Trace("tf.constant"):
    282         return _constant_eager_impl(ctx, value, dtype, shape, verify_shape)
--> 283     return _constant_eager_impl(ctx, value, dtype, shape, verify_shape)
    284 
    285   g = ops.get_default_graph()

/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/constant_op.py in _constant_eager_impl(ctx, value, dtype, shape, verify_shape)
    306 def _constant_eager_impl(ctx, value, dtype, shape, verify_shape):
    307   """Creates a constant on the current device."""
--> 308   t = convert_to_eager_tensor(value, ctx, dtype)
    309   if shape is None:
    310     return t

/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/constant_op.py in convert_to_eager_tensor(value, ctx, dtype)
    104       dtype = dtypes.as_dtype(dtype).as_datatype_enum
    105   ctx.ensure_initialized()
--> 106   return ops.EagerTensor(value, ctx.device_name, dtype)
    107 
    108 

InternalError: Failed copying input tensor from /job:localhost/replica:0/task:0/device:CPU:0 to /job:localhost/replica:0/task:0/device:GPU:0 in order to run _EagerConst: Dst tensor is not initialized.