action_info_hash_key_to_awaited_action out of sync

  2025-03-21T16:25:52.555398Z ERROR nativelink_scheduler::memory_awaited_action_db: action_info_hash_key_to_awaited_action out of sync, it should have had the unique_key, new_awaited_action: AwaitedAction { version: AwaitedActionVersion(3), action_info: ActionInfo { command_digest: DigestInfo("2480a83f765c31df8343465c42d6ea7361de446a9023b26edca463f0f16a3e55-1900"), input_root_digest: DigestInfo("e48f114a106cd85e089e055963fb5966ba84174f75aa64a911baf3107b5dd14b-166"), timeout: 18446744073709551615.999999999s, platform_properties: {"OSFamily": "macos"}, priority: 0, load_timestamp: SystemTime { tv_sec: 0, tv_nsec: 0 }, insert_timestamp: SystemTime { tv_sec: 1742574351, tv_nsec: 655849641 }, unique_qualifier: Cachable(ActionUniqueKey { instance_name: "main", digest_function: Blake3, digest: DigestInfo("3b9e97329b2931dac6489b915b4cd03e8fad9d6708e1d7acb3083f9552d6ba65-167") }) }, operation_id: Uuid(4a72fa78-a18a-4db8-96dc-3047b27d5de4), sort_key: AwaitedActionSortKey(9223372039407168752), last_worker_updated_timestamp: SystemTime { tv_sec: 1742574352, tv_nsec: 555387625 }, last_client_keepalive_timestamp: SystemTime { tv_sec: 1742574351, tv_nsec: 659251161 }, worker_id: Some(1f004e5c-41a9-60bc-973a-b4a5b8d74be9), state: ActionState { stage: Completed(ActionResult { output_files: [FileInfo { name_or_path: Path("bazel-out/darwin_arm64-opt-exec/bin/external/rules_rust++i2+rules_rust_ctve__icu_normalizer_data-1.5.0/libicu_normalizer_data-1274090407.rlib"), digest: DigestInfo("da455a762126ddcedc45fb180c672dff9dd22c34acf43cb8af728611cf237ec4-288176"), is_executable: false }], output_folders: [], output_directory_symlinks: [], output_file_symlinks: [], exit_code: 0, stdout_digest: DigestInfo("af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262-0"), stderr_digest: DigestInfo("af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262-0"), execution_metadata: ExecutionMetadata { worker: "1f004e5c-41a9-60bc-973a-b4a5b8d74be9", queued_timestamp: SystemTime { tv_sec: 1742574351, tv_nsec: 655849641 }, worker_start_timestamp: SystemTime { tv_sec: 1742574351, tv_nsec: 632879000 }, worker_completed_timestamp: SystemTime { tv_sec: 1742574352, tv_nsec: 494261000 }, input_fetch_start_timestamp: SystemTime { tv_sec: 1742574351, tv_nsec: 681078000 }, input_fetch_completed_timestamp: SystemTime { tv_sec: 1742574352, tv_nsec: 116648000 }, execution_start_timestamp: SystemTime { tv_sec: 1742574352, tv_nsec: 116649000 }, execution_completed_timestamp: SystemTime { tv_sec: 1742574352, tv_nsec: 419753000 }, output_upload_start_timestamp: SystemTime { tv_sec: 1742574352, tv_nsec: 419761000 }, output_upload_completed_timestamp: SystemTime { tv_sec: 1742574352, tv_nsec: 494261000 } }, server_logs: {}, error: None, message: "" }), client_operation_id: Uuid(4a72fa78-a18a-4db8-96dc-3047b27d5de4), action_digest: DigestInfo("3b9e97329b2931dac6489b915b4cd03e8fad9d6708e1d7acb3083f9552d6ba65-167") }, maybe_origin_metadata: None, attempts: 0 }, action_key: ActionUniqueKey { instance_name: "main", digest_function: Blake3, digest: DigestInfo("3b9e97329b2931dac6489b915b4cd03e8fad9d6708e1d7acb3083f9552d6ba65-167") }
    at nativelink-scheduler/src/memory_awaited_action_db.rs:563
    in nativelink_service::worker_api_server::execution_response with request: ExecuteResult { worker_id: "1f004e5c-41a9-60bc-973a-b4a5b8d74be9", instance_name: "main", operation_id: "4a72fa78-a18a-4db8-96dc-3047b27d5de4", result: Some(ExecuteResponse(ExecuteResponse { result: Some(ActionResult { output_files: [OutputFile { path: "bazel-out/darwin_arm64-opt-exec/bin/external/rules_rust++i2+rules_rust_ctve__icu_normalizer_data-1.5.0/libicu_normalizer_data-1274090407.rlib", digest: Some(Digest { hash: "da455a762126ddcedc45fb180c672dff9dd22c34acf43cb8af728611cf237ec4", size_bytes: 288176 }), is_executable: false, contents: b"", node_properties: None }], output_file_symlinks: [], output_symlinks: [], output_directories: [], output_directory_symlinks: [], exit_code: 0, stdout_raw: b"", stdout_digest: Some(Digest { hash: "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", size_bytes: 0 }), stderr_raw: b"", stderr_digest: Some(Digest { hash: "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", size_bytes: 0 }), execution_metadata: Some(ExecutedActionMetadata { worker: "1f004e5c-41a9-60bc-973a-b4a5b8d74be9", queued_timestamp: Some(Timestamp { seconds: 1742574351, nanos: 655849641 }), worker_start_timestamp: Some(Timestamp { seconds: 1742574351, nanos: 632879000 }), worker_completed_timestamp: Some(Timestamp { seconds: 1742574352, nanos: 494261000 }), input_fetch_start_timestamp: Some(Timestamp { seconds: 1742574351, nanos: 681078000 }), input_fetch_completed_timestamp: Some(Timestamp { seconds: 1742574352, nanos: 116648000 }), execution_start_timestamp: Some(Timestamp { seconds: 1742574352, nanos: 116649000 }), execution_completed_timestamp: Some(Timestamp { seconds: 1742574352, nanos: 419753000 }), virtual_execution_duration: Some(Duration { seconds: 0, nanos: 303104000 }), output_upload_start_timestamp: Some(Timestamp { seconds: 1742574352, nanos: 419761000 }), output_upload_completed_timestamp: Some(Timestamp { seconds: 1742574352, nanos: 494261000 }), auxiliary_metadata: [] }) }), cached_result: false, status: Some(Status { code: 0, message: "", details: [] }), server_logs: {}, message: "" })) }
    in nativelink_util::task::http_executor
    in nativelink::services::http_connection with remote_addr: 192.168.191.222:61210, socket_addr: 0.0.0.0:50061