All objects disappear when restoring scenes in ALFRED environment

Hi, I'm having issues with restoring object positions in AI2Thor. I'm using the `ThorEnv` class provided with the [alfred](https://github.com/askforalfred/alfred/blob/master/env/thor_env.py) benchmark. When I use this class's `restore_scene` all objects disappear from the room. This behavior seems to hold across scenes.

I get the object poses, object toggles, and dirty and empty states from the data provided with alfred. Example:

`object_poses`:

```
[{'objectName': 'Pan_fee30bc3',
'position': {'x': -0.698599458, 'y': 0.931220651, 'z': 0.23935467},
'rotation': {'x': 0.0, 'y': 0.0, 'z': 0.0}},
{'objectName': 'Potato_ccc73403',
'position': {'x': 1.02802265, 'y': 1.81412673, 'z': -1.23613048},
'rotation': {'x': 0.0, 'y': 270.000183, 'z': 0.0}},
{'objectName': 'Potato_ccc73403',
'position': {'x': 0.905203938, 'y': 0.893499136, 'z': 1.349407},
'rotation': {'x': 0.0, 'y': 270.0, 'z': 0.0}},
{'objectName': 'CreditCard_0304788b',
'position': {'x': 0.7540258, 'y': 0.492195338, 'z': -0.56567806},
'rotation': {'x': 0.0, 'y': 270.0, 'z': 0.0}},
{'objectName': 'CreditCard_0304788b',
'position': {'x': 1.02947235, 'y': 0.908296, 'z': -0.03420466},
'rotation': {'x': 0.0, 'y': 0.0, 'z': 0.0}},
{'objectName': 'ButterKnife_b86b8dfc',
'position': {'x': 0.94426924, 'y': 0.9113421, 'z': -0.5555908},
'rotation': {'x': 0.0, 'y': 0.0, 'z': 0.0}},
{'objectName': 'Fork_1427a484',
'position': {'x': 0.773862958, 'y': 0.911848545, 'z': 0.13959074},
'rotation': {'x': 0.0, 'y': 0.0, 'z': 0.0}},
{'objectName': 'Spoon_4fda088e',
'position': {'x': -1.10427678, 'y': 0.9346294, 'z': -0.118192375},
'rotation': {'x': 0.0, 'y': 0.0, 'z': 0.0}},
{'objectName': 'Spoon_4fda088e',
'position': {'x': 0.8349252, 'y': 0.496328741, 'z': -0.607139},
'rotation': {'x': 0.0, 'y': 270.0, 'z': 0.0}},
{'objectName': 'Spatula_ef7cb958',
'position': {'x': 0.915824652, 'y': 0.7174152, 'z': -0.7315219},
'rotation': {'x': 0.0, 'y': 0.0, 'z': 0.0}},
{'objectName': 'Spatula_ef7cb958',
'position': {'x': 1.02947235, 'y': 0.92629987, 'z': -0.03420466},
'rotation': {'x': 0.0, 'y': 180.0, 'z': 0.0}},
{'objectName': 'Cup_12782efe',
'position': {'x': 1.10209739, 'y': 1.6634922, 'z': -0.6571665},
'rotation': {'x': -7.062251e-30, 'y': 270.0, 'z': 0.0}},
{'objectName': 'Cup_12782efe',
'position': {'x': 0.966, 'y': 1.13210487, 'z': 1.34940743},
'rotation': {'x': 0.0, 'y': 270.0, 'z': 0.0}},
{'objectName': 'Plate_f4832794',
'position': {'x': 1.082977, 'y': 2.21146345, 'z': -1.08218741},
'rotation': {'x': -7.062251e-30, 'y': 270.0, 'z': 0.0}},
{'objectName': 'SoapBottle_8542e5a1',
'position': {'x': -1.22995138, 'y': 0.9338691, 'z': -0.905403852},
'rotation': {'x': 0.0, 'y': 0.0, 'z': 0.0}},
{'objectName': 'SoapBottle_8542e5a1',
'position': {'x': -0.86559397, 'y': 0.177220911, 'z': 0.5759999},
'rotation': {'x': 0.0, 'y': 180.000061, 'z': 0.0}},
{'objectName': 'SaltShaker_49f13c52',
'position': {'x': 0.9608768, 'y': 0.104643464, 'z': -2.05269957},
'rotation': {'x': 0.0, 'y': 0.0, 'z': 0.0}},
{'objectName': 'PepperShaker_ae82d920',
'position': {'x': -0.7724334, 'y': 0.9298061, 'z': -1.3088007},
'rotation': {'x': 0.0, 'y': 0.0, 'z': 0.0}},
{'objectName': 'PepperShaker_ae82d920',
'position': {'x': 1.14810014, 'y': 1.66259885, 'z': -0.27343297},
'rotation': {'x': -7.062251e-30, 'y': 270.0, 'z': 0.0}},
{'objectName': 'Lettuce_5e722375',
'position': {'x': 0.773862958, 'y': 0.988640249, 'z': -0.381795347},
'rotation': {'x': 0.0, 'y': 0.0, 'z': 0.0}},
{'objectName': 'Bread_c3a328b9',
'position': {'x': 0.94426924, 'y': 0.992455661, 'z': 0.13959074},
'rotation': {'x': 0.0, 'y': 0.0, 'z': 0.0}},
{'objectName': 'Mug_63934229',
'position': {'x': 0.9032305, 'y': 0.9428, 'z': -2.11038733},
'rotation': {'x': 0.0, 'y': 300.131073, 'z': 0.0}},
{'objectName': 'Mug_63934229',
'position': {'x': -0.6247653, 'y': 0.928499937, 'z': -1.15414691},
'rotation': {'x': 0.0, 'y': 0.0, 'z': 0.0}},
{'objectName': 'Apple_560cdd24',
'position': {'x': 1.08759236, 'y': 1.69327343, 'z': 1.54022264},
'rotation': {'x': 0.0, 'y': 270.0, 'z': 0.0}},
{'objectName': 'Statue_e620c55b',
'position': {'x': -1.816, 'y': 0.871, 'z': 2.754},
'rotation': {'x': 0.0, 'y': 180.0, 'z': 0.0}},
{'objectName': 'Glassbottle_dc08ca77',
'position': {'x': 0.688659847, 'y': 0.9120167, 'z': -0.729386151},
'rotation': {'x': 0.0, 'y': 0.0, 'z': 0.0}},
{'objectName': 'Bread_c3a328b9',
'position': {'x': 0.9052038, 'y': 1.48296094, 'z': 1.06318474},
'rotation': {'x': 0.0, 'y': 270.0, 'z': 0.0}},
{'objectName': 'CreditCard_0304788b',
'position': {'x': 0.915824652, 'y': 0.699411333, 'z': -0.690061},
'rotation': {'x': 0.0, 'y': 270.0, 'z': 0.0}},
{'objectName': 'CellPhone_ef488591',
'position': {'x': 0.773862958, 'y': 0.9082095, 'z': -0.381795347},
'rotation': {'x': 0.0, 'y': 0.0, 'z': 0.0}},
{'objectName': 'Potato_ccc73403',
'position': {'x': -1.0072031, 'y': 0.207337171, 'z': 0.64120543},
'rotation': {'x': 0.0, 'y': 180.000061, 'z': 0.0}},
{'objectName': 'Knife_f88e84d1',
'position': {'x': -0.624766, 'y': 0.9570276, 'z': -0.06722319},
'rotation': {'x': 0.0, 'y': 0.0, 'z': 0.0}},
{'objectName': 'Fork_1427a484',
'position': {'x': -0.697999954, 'y': 0.7809668, 'z': -0.5730868},
'rotation': {'x': -1.40334191e-14, 'y': 179.999954, 'z': 1.24043245e-20}},
{'objectName': 'Pan_fee30bc3',
'position': {'x': 0.8484889, 'y': 0.9462139, 'z': -1.08015645},
'rotation': {'x': 0.0, 'y': 0.0, 'z': 0.0}},
{'objectName': 'Lettuce_5e722375',
'position': {'x': -0.753279746, 'y': 0.8577969, 'z': -0.7457817},
'rotation': {'x': 1.21328534e-20, 'y': 89.99995, 'z': 1.40334191e-14}},
{'objectName': 'Plate_f4832794',
'position': {'x': 1.08297729, 'y': 2.21146345, 'z': -1.62671876},
'rotation': {'x': -7.062251e-30, 'y': 270.0, 'z': 0.0}},
{'objectName': 'Apple_560cdd24',
'position': {'x': 0.965999961, 'y': 1.45207322, 'z': 1.444815},
'rotation': {'x': 0.0, 'y': 270.0, 'z': 0.0}},
{'objectName': 'Pot_54bd4d72',
'position': {'x': 1.0776, 'y': 0.950000048, 'z': -1.5042},
'rotation': {'x': 0.0, 'y': 0.0, 'z': 0.0}},
{'objectName': 'Spatula_ef7cb958',
'position': {'x': -0.6427199, 'y': 0.791556, 'z': -0.918475866},
'rotation': {'x': 1.40334208e-14,
'y': -4.82967334e-05,
'z': -1.18128613e-20}},
{'objectName': 'PaperTowelRoll_412e7506',
'position': {'x': -0.7724334, 'y': 1.03778529, 'z': -1.54078138},
'rotation': {'x': 0.0, 'y': 0.0, 'z': 0.0}},
{'objectName': 'Tomato_451cbeb9',
'position': {'x': 1.02679634, 'y': 1.68378913, 'z': 1.25400007},
'rotation': {'x': 0.0, 'y': 270.0, 'z': 0.0}},
{'objectName': 'Egg_fe6a8905',
'position': {'x': -1.05440617, 'y': 0.216895491, 'z': 0.5107947},
'rotation': {'x': 0.0, 'y': 180.000061, 'z': 0.0}},
{'objectName': 'SaltShaker_49f13c52',
'position': {'x': 0.811699748, 'y': 0.117171228, 'z': 0.37439996},
'rotation': {'x': -7.062251e-30, 'y': 270.0, 'z': 0.0}},
{'objectName': 'PepperShaker_ae82d920',
'position': {'x': -0.477098227, 'y': 0.9298061, 'z': -0.14386791},
'rotation': {'x': 0.0, 'y': 0.0, 'z': 0.0}},
{'objectName': 'Vase_0c323360',
'position': {'x': -1.423, 'y': 0.169, 'z': 2.621},
'rotation': {'x': 0.0, 'y': 0.0, 'z': 0.0}},
{'objectName': 'ButterKnife_b86b8dfc',
'position': {'x': -0.697999656, 'y': 0.7806329, 'z': -0.8321287},
'rotation': {'x': 1.21328534e-20, 'y': 89.99995, 'z': 1.40334191e-14}},
{'objectName': 'SoapBottle_8542e5a1',
'position': {'x': 0.773862958, 'y': 0.911669135, 'z': -1.79397738},
'rotation': {'x': 0.0, 'y': 0.0, 'z': 0.0}},
{'objectName': 'Vase_aa254d2e',
'position': {'x': 1.11467552, 'y': 0.912376046, 'z': -0.381795347},
'rotation': {'x': 0.0, 'y': 0.0, 'z': 0.0}},
{'objectName': 'Bowl_6f54d42d',
'position': {'x': 1.19987857, 'y': 0.9146208, 'z': -1.79397738},
'rotation': {'x': 0.0, 'y': 0.0, 'z': 0.0}},
{'objectName': 'DishSponge_f11e4d33',
'position': {'x': -1.57190788, 'y': 0.88139534, 'z': 2.69243169},
'rotation': {'x': 0.0, 'y': 0.0, 'z': 0.0}},
{'objectName': 'Spoon_4fda088e',
'position': {'x': -0.6247653, 'y': 0.9346293, 'z': -1.46345448},
'rotation': {'x': 0.0, 'y': 0.0, 'z': 0.0}},
{'objectName': 'Cup_12782efe',
'position': {'x': 1.19400287, 'y': 1.6634922, 'z': -1.9616164},
'rotation': {'x': -7.062251e-30, 'y': 270.0, 'z': 0.0}},
{'objectName': 'Mug_63934229',
'position': {'x': 0.7836112, 'y': 1.12990534, 'z': 1.06318474},
'rotation': {'x': 0.0, 'y': 270.0, 'z': 0.0}}]
```
`object_toggles`:
```
[]
```

`dirty_and_empty`:
```
False
```

The `restore_scene` method from the `ThorEnv` class:

```python
def restore_scene(self, object_poses, object_toggles, dirty_and_empty):
'''
restore object locations and states
'''
super().step(**dict(
action='Initialize',
gridSize=constants.AGENT_STEP_SIZE / constants.RECORD_SMOOTHING_FACTOR,
cameraY=constants.CAMERA_HEIGHT_OFFSET,
renderImage=constants.RENDER_IMAGE,
renderDepthImage=constants.RENDER_DEPTH_IMAGE,
renderClassImage=constants.RENDER_CLASS_IMAGE,
renderObjectImage=constants.RENDER_OBJECT_IMAGE,
visibility_distance=constants.VISIBILITY_DISTANCE,
makeAgentsVisible=False,
))
if len(object_toggles) > 0:
print(object_toggles)
for obj in object_toggles:
# super().step(**(dict(action='SetObjectToggles', objectToggles=object_toggles)))
objects = [objj for objj in self.last_event.metadata["objects"] if obj["objectType"] == obj["objectType"] and "toggleable" in obj and obj["toggleable"]]
if obj["isOn"]:
for objj in objects:
super().step(action='ToggleObjectOn', objectId=objj["objectId"])
else:
for objj in objects:
super().step(action='ToggleObjectOff', objectId=objj["objectId"])

if dirty_and_empty:
super().step(**dict(action='SetStateOfAllObjects',
StateChange="CanBeDirty",
forceAction=True))
super().step(**dict(action='SetStateOfAllObjects',
StateChange="CanBeFilled",
forceAction=False))
super().step(**(dict(action='SetObjectPoses', objectPoses=object_poses)))
```

I'm using AI2Thor version 3.5.1, I'm thinking this may be the cause since ALFRED requires version 2.1.0.

Example empty room (`FloorPlan308`):