OPTE port not cleaned up on Propolis panic

When analyzing an instance that failed to start we noticed two issues:

1. The VM panicked (and is tracked in https://github.com/oxidecomputer/propolis/issues/875)
2. There was a second attempt to start the instance but it looks to have failed prior to propolis starting

This issue is focused on the second point. Looking at the database we see the following for the two failures:

725e8aab-0fc8-4773-974e-a8394fc3b425 | 2025-03-04 14:50:24.074707+00 | 2025-03-04 14:50:39.952477+00 | ca0c0155-d912-45b4-9108-a67b62840eb3 | 2025-03-04 14:50:39.289311+00 |                3 | ac1d392c-ff2c-445f-b9d1-afabde391df4 | fd00:1122:3344:101::1:34f |         12400 | destroyed
3fe66236-b48c-4384-a689-de7f389199dc | 2025-03-04 14:51:00.670226+00 | 2025-03-04 14:51:08.548218+00 | ca0c0155-d912-45b4-9108-a67b62840eb3 | 2025-03-04 14:51:01.549313+00 |                3 | eaad4483-ea93-48d1-99d3-bf43267b9eb7 | fd00:1122:3344:109::1:31c |         12400 | destroyed

725e8aab-0fc8-4773-974e-a8394fc3b425 is the panic, and 3fe66236-b48c-4384-a689-de7f389199dc is the second failure.

We did not get a propolis core nor any propolis logs, which suggests that we did not get to the point of creating the zone. Looking at the sled-agent logs for the zone id we see:

BRM42220015 # grep -E "3fe66236-b48c-4384-a689-de7f389199dc" `svcs -L sled-agent` | looker
14:51:01.459Z INFO SledAgent (InstanceManager): ensuring instance is registered
    file = sled-agent/src/instance_manager.rs:555
    hardware = InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(64), memory: ByteCount(274877906944), hostname: Hostname("carrot") }, nics: [NetworkInterface { id: 2cd4ec99-18d5-4579-9519-1364b82c2cc9, kind: Instance { id: ca0c0155-d912-45b4-9108-a67b62840eb3 }, name: Name("net0"), ip: 172.30.0.7, mac: MacAddr(MacAddr6([168, 64, 37, 253, 208, 66])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6357461), primary: true, slot: 1, transit_ips: [] }], source_nat: SourceNatConfig { ip: 45.154.216.142, first_port: 32768, last_port: 49151 }, ephemeral_ip: None, floating_ips: [45.154.216.122], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1c81b5b3-596a-46fe-a4c6-ac97a7d5b2e2, kind: Instance { id: e350ec3e-3600-4fc2-931c-34132709d297 }, name: Name("net0"), ip: 172.30.0.9, mac: MacAddr(MacAddr6([168, 64, 37, 254, 232, 199])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6357461), primary: true, slot: 1, transit_ips: [] }], filter_hosts: Some([]), filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Tcp, Udp]), action: Allow, priority: VpcFirewallRulePriority(0) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1c81b5b3-596a-46fe-a4c6-ac97a7d5b2e2, kind: Instance { id: e350ec3e-3600-4fc2-931c-34132709d297 }, name: Name("net0"), ip: 172.30.0.9, mac: MacAddr(MacAddr6([168, 64, 37, 254, 232, 199])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6357461), primary: true, slot: 1, transit_ips: [] }], filter_hosts: Some([]), filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(0) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1c81b5b3-596a-46fe-a4c6-ac97a7d5b2e2, kind: Instance { id: e350ec3e-3600-4fc2-931c-34132709d297 }, name: Name("net0"), ip: 172.30.0.9, mac: MacAddr(MacAddr6([168, 64, 37, 254, 232, 199])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6357461), primary: true, slot: 1, transit_ips: [] }, NetworkInterface { id: 2cd4ec99-18d5-4579-9519-1364b82c2cc9, kind: Instance { id: ca0c0155-d912-45b4-9108-a67b62840eb3 }, name: Name("net0"), ip: 172.30.0.7, mac: MacAddr(MacAddr6([168, 64, 37, 253, 208, 66])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6357461), primary: true, slot: 1, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1c81b5b3-596a-46fe-a4c6-ac97a7d5b2e2, kind: Instance { id: e350ec3e-3600-4fc2-931c-34132709d297 }, name: Name("net0"), ip: 172.30.0.9, mac: MacAddr(MacAddr6([168, 64, 37, 254, 232, 199])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6357461), primary: true, slot: 1, transit_ips: [] }, NetworkInterface { id: 2cd4ec99-18d5-4579-9519-1364b82c2cc9, kind: Instance { id: ca0c0155-d912-45b4-9108-a67b62840eb3 }, name: Name("net0"), ip: 172.30.0.7, mac: MacAddr(MacAddr6([168, 64, 37, 253, 208, 66])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6357461), primary: true, slot: 1, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(6357461)), Vpc(Vni(6357461))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1c81b5b3-596a-46fe-a4c6-ac97a7d5b2e2, kind: Instance { id: e350ec3e-3600-4fc2-931c-34132709d297 }, name: Name("net0"), ip: 172.30.0.9, mac: MacAddr(MacAddr6([168, 64, 37, 254, 232, 199])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6357461), primary: true, slot: 1, transit_ips: [] }, NetworkInterface { id: 2cd4ec99-18d5-4579-9519-1364b82c2cc9, kind: Instance { id: ca0c0155-d912-45b4-9108-a67b62840eb3 }, name: Name("net0"), ip: 172.30.0.7, mac: MacAddr(MacAddr6([168, 64, 37, 253, 208, 66])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6357461), primary: true, slot: 1, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1c81b5b3-596a-46fe-a4c6-ac97a7d5b2e2, kind: Instance { id: e350ec3e-3600-4fc2-931c-34132709d297 }, name: Name("net0"), ip: 172.30.0.9, mac: MacAddr(MacAddr6([168, 64, 37, 254, 232, 199])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6357461), primary: true, slot: 1, transit_ips: [] }], filter_hosts: Some([]), filter_ports: Some([L4PortRange { first: L4Port(12375), last: L4Port(12375) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(0) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 8.8.8.8], host_domain: None, search_domains: [] }, disks: [InstanceDisk { disk_id: b89e38e3-ac9e-4ecd-ade3-611912b79c39, slot: 0, read_only: false, vcr_json: .., name: "carrot-boot" }, InstanceDisk { disk_id: e776a05c-eeb6-4e78-bf10-680083f9ad03, slot: 1, read_only: false, vcr_json: .., name: "carrot-data" }, InstanceDisk { disk_id: 60ad6d06-4fb1-4240-b6c1-6675ff039a80, slot: 2, read_only: false, vcr_json: .., name: "carrot-dump" }], boot_settings: None, cloud_init_bytes: Some(..) }
    instance_id = ca0c0155-d912-45b4-9108-a67b62840eb3
    metadata = InstanceMetadata { silo_id: a92911f2-f144-41bf-ac52-da7c7d5e84d9, project_id: ae90ba27-b4f1-4341-a4c7-4cababe1b3a0 }
    migration_id = None
    propolis_addr = [fd00:1122:3344:109::1:31c]:12400
    propolis_id = 3fe66236-b48c-4384-a689-de7f389199dc
    vmm_runtime = VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2025-03-04T14:51:01.458489236Z }
14:51:01.459Z INFO SledAgent (InstanceManager): registering new instance
    file = sled-agent/src/instance_manager.rs:588
    instance_id = ca0c0155-d912-45b4-9108-a67b62840eb3
    migration_id = None
    propolis_id = 3fe66236-b48c-4384-a689-de7f389199dc
14:51:01.459Z INFO SledAgent (InstanceManager): initializing new Instance
    file = sled-agent/src/instance.rs:1565
    instance_id = ca0c0155-d912-45b4-9108-a67b62840eb3
    migration_id = None
    propolis_id = 3fe66236-b48c-4384-a689-de7f389199dc
    state = InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(64), memory: ByteCount(274877906944), hostname: Hostname("carrot") }, nics: [NetworkInterface { id: 2cd4ec99-18d5-4579-9519-1364b82c2cc9, kind: Instance { id: ca0c0155-d912-45b4-9108-a67b62840eb3 }, name: Name("net0"), ip: 172.30.0.7, mac: MacAddr(MacAddr6([168, 64, 37, 253, 208, 66])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6357461), primary: true, slot: 1, transit_ips: [] }], source_nat: SourceNatConfig { ip: 45.154.216.142, first_port: 32768, last_port: 49151 }, ephemeral_ip: None, floating_ips: [45.154.216.122], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1c81b5b3-596a-46fe-a4c6-ac97a7d5b2e2, kind: Instance { id: e350ec3e-3600-4fc2-931c-34132709d297 }, name: Name("net0"), ip: 172.30.0.9, mac: MacAddr(MacAddr6([168, 64, 37, 254, 232, 199])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6357461), primary: true, slot: 1, transit_ips: [] }], filter_hosts: Some([]), filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Tcp, Udp]), action: Allow, priority: VpcFirewallRulePriority(0) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1c81b5b3-596a-46fe-a4c6-ac97a7d5b2e2, kind: Instance { id: e350ec3e-3600-4fc2-931c-34132709d297 }, name: Name("net0"), ip: 172.30.0.9, mac: MacAddr(MacAddr6([168, 64, 37, 254, 232, 199])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6357461), primary: true, slot: 1, transit_ips: [] }], filter_hosts: Some([]), filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(0) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1c81b5b3-596a-46fe-a4c6-ac97a7d5b2e2, kind: Instance { id: e350ec3e-3600-4fc2-931c-34132709d297 }, name: Name("net0"), ip: 172.30.0.9, mac: MacAddr(MacAddr6([168, 64, 37, 254, 232, 199])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6357461), primary: true, slot: 1, transit_ips: [] }, NetworkInterface { id: 2cd4ec99-18d5-4579-9519-1364b82c2cc9, kind: Instance { id: ca0c0155-d912-45b4-9108-a67b62840eb3 }, name: Name("net0"), ip: 172.30.0.7, mac: MacAddr(MacAddr6([168, 64, 37, 253, 208, 66])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6357461), primary: true, slot: 1, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1c81b5b3-596a-46fe-a4c6-ac97a7d5b2e2, kind: Instance { id: e350ec3e-3600-4fc2-931c-34132709d297 }, name: Name("net0"), ip: 172.30.0.9, mac: MacAddr(MacAddr6([168, 64, 37, 254, 232, 199])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6357461), primary: true, slot: 1, transit_ips: [] }, NetworkInterface { id: 2cd4ec99-18d5-4579-9519-1364b82c2cc9, kind: Instance { id: ca0c0155-d912-45b4-9108-a67b62840eb3 }, name: Name("net0"), ip: 172.30.0.7, mac: MacAddr(MacAddr6([168, 64, 37, 253, 208, 66])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6357461), primary: true, slot: 1, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(6357461)), Vpc(Vni(6357461))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1c81b5b3-596a-46fe-a4c6-ac97a7d5b2e2, kind: Instance { id: e350ec3e-3600-4fc2-931c-34132709d297 }, name: Name("net0"), ip: 172.30.0.9, mac: MacAddr(MacAddr6([168, 64, 37, 254, 232, 199])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6357461), primary: true, slot: 1, transit_ips: [] }, NetworkInterface { id: 2cd4ec99-18d5-4579-9519-1364b82c2cc9, kind: Instance { id: ca0c0155-d912-45b4-9108-a67b62840eb3 }, name: Name("net0"), ip: 172.30.0.7, mac: MacAddr(MacAddr6([168, 64, 37, 253, 208, 66])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6357461), primary: true, slot: 1, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1c81b5b3-596a-46fe-a4c6-ac97a7d5b2e2, kind: Instance { id: e350ec3e-3600-4fc2-931c-34132709d297 }, name: Name("net0"), ip: 172.30.0.9, mac: MacAddr(MacAddr6([168, 64, 37, 254, 232, 199])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(6357461), primary: true, slot: 1, transit_ips: [] }], filter_hosts: Some([]), filter_ports: Some([L4PortRange { first: L4Port(12375), last: L4Port(12375) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(0) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 8.8.8.8], host_domain: None, search_domains: [] }, disks: [InstanceDisk { disk_id: b89e38e3-ac9e-4ecd-ade3-611912b79c39, slot: 0, read_only: false, vcr_json: .., name: "carrot-boot" }, InstanceDisk { disk_id: e776a05c-eeb6-4e78-bf10-680083f9ad03, slot: 1, read_only: false, vcr_json: .., name: "carrot-data" }, InstanceDisk { disk_id: 60ad6d06-4fb1-4240-b6c1-6675ff039a80, slot: 2, read_only: false, vcr_json: .., name: "carrot-dump" }], boot_settings: None, cloud_init_bytes: Some(..) }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2025-03-04T14:51:01.458489236Z }, propolis_addr: [fd00:1122:3344:109::1:31c]:12400, migration_id: None }
14:51:01.460Z INFO SledAgent (dropshot (SledAgent)): request completed
    file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867
    latency_us = 1094
    local_addr = [fd00:1122:3344:109::1]:12345
    method = PUT
    remote_addr = [fd00:1122:3344:111::3]:54630
    req_id = 0b43fed0-3f41-4c25-b694-6bbd6eb8965e
    response_code = 200
    uri = /vmms/3fe66236-b48c-4384-a689-de7f389199dc
14:51:01.548Z INFO SledAgent (dropshot (SledAgent)): request completed
    file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867
    latency_us = 134
    local_addr = [fd00:1122:3344:109::1]:12345
    method = PUT
    remote_addr = [fd00:1122:3344:111::3]:62282
    req_id = 1c41ec87-4339-40a3-9e08-bc195f8b4983
    response_code = 200
    uri = /vmms/3fe66236-b48c-4384-a689-de7f389199dc/state
14:51:01.549Z ERRO SledAgent (InstanceManager): vmm setup failed: Err(Opte(Opte(CommandError(CreateXde, MacExists { port: "opte17", vni: Vni { inner: 6357461 }, mac: MacAddr { inner: A8:40:25:FD:D0:42 } }))))
    file = sled-agent/src/instance.rs:1890
    instance_id = ca0c0155-d912-45b4-9108-a67b62840eb3
    propolis_id = 3fe66236-b48c-4384-a689-de7f389199dc
14:51:07.873Z INFO SledAgent (dropshot (SledAgent)): request completed
    file = /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867
    latency_us = 522
    local_addr = [fd00:1122:3344:109::1]:12345
    method = GET
    remote_addr = [fd00:1122:3344:111::3]:35290
    req_id = 29abd320-d8b3-4409-9c3e-1b1a30481575
    response_code = 200
    uri = /vmms/3fe66236-b48c-4384-a689-de7f389199dc/state