Cannot compile m5stack-atom-echo voice assistant example

### The problem

Trying to compile the m5stack-atom voice assistant example using an aarch64-based container on a Raspberry Pi 4B running bookworm fails.

### Which version of ESPHome has the issue?

ESPHome 2023.11.6

### What type of installation are you using?

Docker

### Which version of Home Assistant has the issue?

_No response_

### What platform are you using?

ESP32-IDF

### Board

m5stack-atom

### Component causing the issue

_No response_

### Example YAML snippet

```yaml
---
esphome:
name: family-room-echo
friendly_name: family-room-echo
project:
name: m5stack.atom-echo-voice-assistant
version: "1.0"
min_version: 2023.11.1

esp32:
board: m5stack-atom
framework:
type: esp-idf

logger:
api:
ota:

dashboard_import:
package_import_url: github://esphome/firmware/voice-assistant/m5stack-atom-echo.yaml@main

wifi:
on_connect:
- delay: 5s # Gives time for improv results to be transmitted
- ble.disable:
on_disconnect:
- ble.enable:
ap:

improv_serial:

esp32_improv:
authorizer: none

button:
- platform: factory_reset
id: factory_reset_btn
name: Factory reset

i2s_audio:
i2s_lrclk_pin: GPIO33
i2s_bclk_pin: GPIO19

microphone:
- platform: i2s_audio
id: echo_microphone
i2s_din_pin: GPIO23
adc_type: external
pdm: true

speaker:
- platform: i2s_audio
id: echo_speaker
i2s_dout_pin: GPIO22
dac_type: external
mode: mono

voice_assistant:
id: va
microphone: echo_microphone
speaker: echo_speaker
noise_suppression_level: 2
auto_gain: 31dBFS
volume_multiplier: 2.0
vad_threshold: 3
on_listening:
- light.turn_on:
id: led
blue: 100%
red: 0%
green: 0%
effect: "Slow Pulse"
on_stt_vad_end:
- light.turn_on:
id: led
blue: 100%
red: 0%
green: 0%
effect: "Fast Pulse"
on_tts_start:
- light.turn_on:
id: led
blue: 100%
red: 0%
green: 0%
brightness: 100%
effect: none
on_end:
- delay: 100ms
- wait_until:
not:
speaker.is_playing:
- script.execute: reset_led
on_error:
- light.turn_on:
id: led
red: 100%
green: 0%
blue: 0%
brightness: 100%
effect: none
- delay: 1s
- script.execute: reset_led
on_client_connected:
- if:
condition:
switch.is_on: use_wake_word
then:
- voice_assistant.start_continuous:
- script.execute: reset_led
on_client_disconnected:
- if:
condition:
switch.is_on: use_wake_word
then:
- voice_assistant.stop:
- light.turn_off: led

binary_sensor:
- platform: gpio
pin:
number: GPIO39
inverted: true
name: Button
disabled_by_default: true
entity_category: diagnostic
id: echo_button
on_multi_click:
- timing:
- ON for at least 250ms
- OFF for at least 50ms
then:
- if:
condition:
switch.is_off: use_wake_word
then:
- if:
condition: voice_assistant.is_running
then:
- voice_assistant.stop:
- script.execute: reset_led
else:
- voice_assistant.start:
else:
- voice_assistant.stop
- delay: 1s
- script.execute: reset_led
- script.wait: reset_led
- voice_assistant.start_continuous:
- timing:
- ON for at least 10s
then:
- button.press: factory_reset_btn

light:
- platform: esp32_rmt_led_strip
id: led
name: None
disabled_by_default: true
entity_category: config
pin: GPIO27
default_transition_length: 0s
chipset: SK6812
num_leds: 1
rgb_order: grb
rmt_channel: 0
effects:
- pulse:
name: "Slow Pulse"
transition_length: 250ms
update_interval: 250ms
min_brightness: 50%
max_brightness: 100%
- pulse:
name: "Fast Pulse"
transition_length: 100ms
update_interval: 100ms
min_brightness: 50%
max_brightness: 100%

script:
- id: reset_led
then:
- if:
condition:
- switch.is_on: use_wake_word
- switch.is_on: use_listen_light
then:
- light.turn_on:
id: led
red: 100%
green: 89%
blue: 71%
brightness: 60%
effect: none
else:
- light.turn_off: led

switch:
- platform: template
name: Use wake word
id: use_wake_word
optimistic: true
restore_mode: RESTORE_DEFAULT_ON
entity_category: config
on_turn_on:
- lambda: id(va).set_use_wake_word(true);
- if:
condition:
not:
- voice_assistant.is_running
then:
- voice_assistant.start_continuous
- script.execute: reset_led
on_turn_off:
- voice_assistant.stop
- lambda: id(va).set_use_wake_word(false);
- script.execute: reset_led
- platform: template
name: Use Listen Light
id: use_listen_light
optimistic: true
restore_mode: RESTORE_DEFAULT_ON
entity_category: config
on_turn_on:
- script.execute: reset_led
on_turn_off:
- script.execute: reset_led

external_components:
- source: github://pr#5230
components:
- esp_adf
refresh: 0s

esp_adf:
```

### Anything in the logs that might be useful for us?

```txt
INFO ESPHome 2023.11.6
INFO Reading configuration config/family-room-echo.yaml...
INFO Updating https://github.com/esphome/esphome.git@pull/5230/head
INFO Generating C++ source...
INFO Cloning https://github.com/espressif/esp-adf@v2.5
INFO Initialising submodules (components/esp-sr, components/esp-adf-libs) for https://github.com/espressif/esp-adf@v2.5
INFO Cloning https://github.com/espressif/esp-dsp@v1.2.0
INFO Compiling app...
Processing family-room-echo (board: m5stack-atom; framework: espidf; platform: platformio/espressif32@5.4.0)
--------------------------------------------------------------------------------
HARDWARE: ESP32 240MHz, 320KB RAM, 4MB Flash
- framework-espidf @ 3.40405.230623 (4.4.5)
- tool-cmake @ 3.16.9
- tool-ninja @ 1.10.2
- toolchain-esp32ulp @ 2.35.0-20220830
- toolchain-xtensa-esp32 @ 8.4.0+2021r2-patch5
Creating a new virtual environment for IDF Python dependencies
AssertionError: Error: Failed to create a proper virtual environment. Missing the `pip` binary!:
File "/usr/local/lib/python3.11/dist-packages/platformio/builder/main.py", line 173:
env.SConscript("$BUILD_SCRIPT")
File "/config/.esphome/platformio/packages/tool-scons/scons-local-4.5.2/SCons/Script/SConscript.py", line 598:
return _SConscript(self.fs, *files, **subst_kw)
File "/config/.esphome/platformio/packages/tool-scons/scons-local-4.5.2/SCons/Script/SConscript.py", line 285:
exec(compile(scriptdata, scriptname, 'exec'), call_stack[-1].globals)
File "/config/.esphome/platformio/platforms/espressif32@5.4.0/builder/main.py", line 312:
target_elf = env.BuildProgram()
File "/config/.esphome/platformio/packages/tool-scons/scons-local-4.5.2/SCons/Util/envs.py", line 242:
return self.method(*nargs, **kwargs)
File "/usr/local/lib/python3.11/dist-packages/platformio/builder/tools/piobuild.py", line 61:
env.ProcessProgramDeps()
File "/config/.esphome/platformio/packages/tool-scons/scons-local-4.5.2/SCons/Util/envs.py", line 242:
return self.method(*nargs, **kwargs)
File "/usr/local/lib/python3.11/dist-packages/platformio/builder/tools/piobuild.py", line 121:
env.BuildFrameworks(env.get("PIOFRAMEWORK"))
File "/config/.esphome/platformio/packages/tool-scons/scons-local-4.5.2/SCons/Util/envs.py", line 242:
return self.method(*nargs, **kwargs)
File "/usr/local/lib/python3.11/dist-packages/platformio/builder/tools/piobuild.py", line 342:
SConscript(env.GetFrameworkScript(name), exports="env")
File "/config/.esphome/platformio/packages/tool-scons/scons-local-4.5.2/SCons/Script/SConscript.py", line 662:
return method(*args, **kw)
File "/config/.esphome/platformio/packages/tool-scons/scons-local-4.5.2/SCons/Script/SConscript.py", line 598:
return _SConscript(self.fs, *files, **subst_kw)
File "/config/.esphome/platformio/packages/tool-scons/scons-local-4.5.2/SCons/Script/SConscript.py", line 285:
exec(compile(scriptdata, scriptname, 'exec'), call_stack[-1].globals)
File "/config/.esphome/platformio/platforms/espressif32@5.4.0/builder/frameworks/espidf.py", line 1232:
ensure_python_venv_available()
File "/config/.esphome/platformio/platforms/espressif32@5.4.0/builder/frameworks/espidf.py", line 1208:
_create_venv(venv_dir)
File "/config/.esphome/platformio/platforms/espressif32@5.4.0/builder/frameworks/espidf.py", line 1200:
assert os.path.isfile(
========================= [FAILED] Took 15.95 seconds =========================
```

### Additional information

_No response_