tensorflow / tensorflow/models

mask rcnn inception resenet v2 error in train model

Open
#9,527 5 comments 2 reactions 2 assignees View on GitHub

@jch1 is already working on this.

Since Feb 7, 2021.

models:research:odapi type:bug
Dominant language
Python
Stars
77.7k
Forks
44.8k
PR merge metrics
No merged PRs in 30d

Description

anybody face this issue while training mask rcnn inception resnet v2?
here is my pipeline.config

`Mask R-CNN with Inception Resnet v2 (no atrous)
Sync-trained on COCO (with 8 GPUs) with batch size 16 (1024x1024 resolution)
Initialized from Imagenet classification checkpoint
TF2-Compatible, Not TPU-Compatible

Achieves XXX mAP on COCO

model {
faster_rcnn {
number_of_stages: 3
num_classes: 3
image_resizer {
fixed_shape_resizer {
height: 1024
width: 1024
# pad_to_max_dimension: true
}
}
feature_extractor {
type: 'faster_rcnn_inception_resnet_v2_keras'
}
first_stage_anchor_generator {
grid_anchor_generator {
scales: [0.25, 0.5, 1.0, 2.0]
aspect_ratios: [0.5, 1.0, 2.0]
height_stride: 16
width_stride: 16
}
}
first_stage_box_predictor_conv_hyperparams {
op: CONV
regularizer {
l2_regularizer {
weight: 0.0
}
}
initializer {
truncated_normal_initializer {
stddev: 0.01
}
}
}
first_stage_nms_score_threshold: 0.0
first_stage_nms_iou_threshold: 0.7
first_stage_max_proposals: 300
first_stage_localization_loss_weight: 2.0
first_stage_objectness_loss_weight: 1.0
initial_crop_size: 17
maxpool_kernel_size: 1
maxpool_stride: 1
second_stage_box_predictor {
mask_rcnn_box_predictor {
use_dropout: false
dropout_keep_probability: 1.0
fc_hyperparams {
op: FC
regularizer {
l2_regularizer {
weight: 0.0
}
}
initializer {
variance_scaling_initializer {
factor: 1.0
uniform: true
mode: FAN_AVG
}
}
}
mask_height: 33
mask_width: 33
mask_prediction_conv_depth: 0
mask_prediction_num_conv_layers: 4
conv_hyperparams {
op: CONV
regularizer {
l2_regularizer {
weight: 0.0
}
}
initializer {
truncated_normal_initializer {
stddev: 0.01
}
}
}
predict_instance_masks: true
}
}
second_stage_post_processing {
batch_non_max_suppression {
score_threshold: 0.0
iou_threshold: 0.6
max_detections_per_class: 100
max_total_detections: 100
}
score_converter: SOFTMAX
}
second_stage_localization_loss_weight: 2.0
second_stage_classification_loss_weight: 1.0
second_stage_mask_prediction_loss_weight: 4.0
resize_masks: false
}
}

train_config: {
batch_size: 5
num_steps: 200000
optimizer {
momentum_optimizer: {
learning_rate: {
cosine_decay_learning_rate {
learning_rate_base: 0.008
total_steps: 200000
warmup_learning_rate: 0.0
warmup_steps: 5000
}
}
momentum_optimizer_value: 0.9
}
use_moving_average: false
}
gradient_clipping_by_norm: 10.0
fine_tune_checkpoint: "path/to/mymodel/checkpoint"
data_augmentation_options {
random_horizontal_flip {
}
}
}

train_input_reader: {
tf_record_input_reader {
input_path: "annotations/train.record"
}
load_instance_masks: true
mask_type: PNG_MASKS
}

eval_config: {
metrics_set: "coco_detection_metrics"
metrics_set: "coco_mask_metrics"
eval_instance_masks: true
use_moving_averages: false
batch_size: 1
include_metrics_per_category: true
}

eval_input_reader: {
label_map_path: "annotations/label_map.pbtxt"
shuffle: false
num_epochs: 1
tf_record_input_reader {
input_path: "annotations/test.record"
}
load_instance_masks: true
mask_type: PNG_MASKS
}`

but im getting this error
raceback (most recent call last): File "model_main_tf2.py", line 113, in <module> tf.compat.v1.app.run() File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/platform/app.py", line 40, in run _run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef) File "/usr/local/lib/python3.6/dist-packages/absl/app.py", line 300, in run _run_main(main, args) File "/usr/local/lib/python3.6/dist-packages/absl/app.py", line 251, in _run_main sys.exit(main(argv)) File "model_main_tf2.py", line 110, in main record_summaries=FLAGS.record_summaries) File "/usr/local/lib/python3.6/dist-packages/object_detection-0.1-py3.6.egg/object_detection/model_lib_v2.py", line 566, in train_loop unpad_groundtruth_tensors) File "/usr/local/lib/python3.6/dist-packages/object_detection-0.1-py3.6.egg/object_detection/model_lib_v2.py", line 344, in load_fine_tune_checkpoint features, labels = iter(input_dataset).next() File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/distribute/input_lib.py", line 645, in next return self.__next__() File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/distribute/input_lib.py", line 649, in __next__ return self.get_next() File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/distribute/input_lib.py", line 694, in get_next self._iterators[i].get_next_as_list_static_shapes(new_name)) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/distribute/input_lib.py", line 1474, in get_next_as_list_static_shapes return self._iterator.get_next() File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/data/ops/multi_device_iterator_ops.py", line 581, in get_next result.append(self._device_iterators[i].get_next()) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/data/ops/iterator_ops.py", line 825, in get_next return self._next_internal() File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/data/ops/iterator_ops.py", line 764, in _next_internal return structure.from_compatible_tensor_list(self._element_spec, ret) File "/usr/lib/python3.6/contextlib.py", line 99, in __exit__ self.gen.throw(type, value, traceback) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/context.py", line 2105, in execution_mode executor_new.wait() File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/executor.py", line 67, in wait pywrap_tfe.TFE_ExecutorWaitForAllPendingNodes(self._handle) tensorflow.python.framework.errors_impl.InvalidArgumentError: indices[0] = 0 is not in [0, 0) [[{{node GatherV2_7}}]] [[MultiDeviceIteratorGetNextFromShard]] [[RemoteCall]]

Contributor guide

Open the contributing guide

First steps

  1. Read the whole issue, then the project's contributing guide.
  2. Comment on the issue to say you are picking it up — it saves two people doing the same work.
  3. Fork the repository and make your change on a branch.
  4. Open a pull request that references the issue number.

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.