Investigate high inference latency

Ported from https://github.com/flutter/genui/issues/70 — originally opened by [jacobsimionato](https://github.com/jacobsimionato) on 2025-08-05.

<-->

Gen UI is feeling less responsive than I hoped!

Some data for a typical request

## First request, shows filter chips etc:
- Latency = 10573ms
- Output tokens = 722
- Prompt tokens = 6800

## Second request, to generate itinerary:
- Latency = 19491ms
- Output tokens = 2248
- Prompt tokens = 7256

## Actual example data:

Full logs

```
[AiLoggingSeverity.info] ****** Performing Inference ******
{
"role": "user",
"parts": [
{
"text": "Trip to New York"
}
]
}
With functions:
'provideFinalOutput',
[AiLoggingSeverity.info] ****** Completed Inference ******
Latency = 10573ms}
Output tokens = 722
Prompt tokens = 6800
[AiLoggingSeverity.info] ****** Gen UI Output ******.
{
"actions": [
{
"surfaceId": "new_york_trip_planning_surface",
"definition": {
"widgets": [
{
"id": "main_column",
"widget": {
"Column": {
"spacing": 16,
"mainAxisAlignment": "start",
"crossAxisAlignment": "start",
"children": [
"nyc_travel_carousel",
"intro_text",
"nyc_filter_group"
]
}
}
},
{
"widget": {
"text": {
"text": "Tell me more about your ideal trip to New York City!"
}
},
"id": "intro_text"
},
{
"widget": {
"travelCarousel": {
"items": [
{
"title": "Culture & Museums",
"imageChild": "image_culture"
},
{
"imageChild": "image_food",
"title": "Foodie Adventure"
},
{
"title": "Central Park Retreat",
"imageChild": "image_park"
},
{
"imageChild": "image_broadway",
"title": "Broadway & Entertainment"
}
]
}
},
"id": "nyc_travel_carousel"
},
{
"id": "image_culture",
"widget": {
"image": {
"fit": "cover",
"url": "https://genui-demo-stockimages.web.app/Brighton,_the_front_and_the_chain_pier_seen_in_the_distance.jpg"
}
}
},
{
"id": "image_food",
"widget": {
"image": {
"fit": "cover",
"url": "https://genui-demo-stockimages.web.app/Monte_Carlo_Casino_seaside_facade_before_1878_-_Bonillo_2004_p113.jpg"
}
}
},
{
"widget": {
"image": {
"url": "https://genui-demo-stockimages.web.app/Leaf_1_web.jpg",
"fit": "cover"
}
},
"id": "image_park"
},
{
"widget": {
"image": {
"url": "https://genui-demo-stockimages.web.app/The_promenade,_Blackpool,_Lancashire,_England,_ca._1898.jpg",
"fit": "cover"
}
},
"id": "image_broadway"
},
{
"id": "nyc_filter_group",
"widget": {
"filterChipGroup": {
"submitLabel": "Find My Trip",
"children": [
"budget_chip",
"activity_chip",
"duration_chip",
"travelers_chip"
]
}
}
},
{
"id": "budget_chip",
"widget": {
"optionsFilterChip": {
"chipLabel": "Budget",
"options": [
"Economy",
"Mid-range",
"Luxury"
]
}
}
},
{
"widget": {
"optionsFilterChip": {
"options": [
"Sightseeing",
"Art & Culture",
"Food & Drink",
"Outdoors",
"Shopping",
"Nightlife"
],
"chipLabel": "Activity Type"
}
},
"id": "activity_chip"
},
{
"widget": {
"optionsFilterChip": {
"chipLabel": "Duration",
"options": [
"Weekend (2-3 days)",
"Short Trip (4-5 days)",
"Week-long (6-7 days)"
]
}
},
"id": "duration_chip"
},
{
"id": "travelers_chip",
"widget": {
"optionsFilterChip": {
"options": [
"Solo",
"Couple",
"Family",
"Friends"
],
"chipLabel": "Travelers"
}
}
}
],
"root": "main_column"
},
"action": "add"
}
]
}}
[AiLoggingSeverity.info] ****** Performing Inference ******
{
"role": "user",
"parts": [
{
"text": "Trip to New York"
}
]
}
{
"role": "function",
"parts": [
{
"functionResponse": {
"name": "activity_chip",
"response": {
"surfaceId": "new_york_trip_planning_surface",
"widgetId": "activity_chip",
"eventType": "filterOptionSelected",
"timestamp": "2025-08-05T03:51:40.439795Z",
"value": "Art & Culture"
}
}
}
]
}
{
"role": "function",
"parts": [
{
"functionResponse": {
"name": "duration_chip",
"response": {
"surfaceId": "new_york_trip_planning_surface",
"widgetId": "duration_chip",
"eventType": "filterOptionSelected",
"timestamp": "2025-08-05T03:51:41.820525Z",
"value": "Short Trip (4-5 days)"
}
}
}
]
}
{
"role": "function",
"parts": [
{
"functionResponse": {
"name": "travelers_chip",
"response": {
"surfaceId": "new_york_trip_planning_surface",
"widgetId": "travelers_chip",
"eventType": "filterOptionSelected",
"timestamp": "2025-08-05T03:51:43.772607Z",
"value": "Couple"
}
}
}
]
}
{
"role": "function",
"parts": [
{
"functionResponse": {
"name": "budget_chip",
"response": {
"surfaceId": "new_york_trip_planning_surface",
"widgetId": "budget_chip",
"eventType": "filterOptionSelected",
"timestamp": "2025-08-05T03:51:45.388522Z",
"value": "Mid-range"
}
}
}
]
}
{
"role": "function",
"parts": [
{
"functionResponse": {
"name": "nyc_filter_group",
"response": {
"surfaceId": "new_york_trip_planning_surface",
"widgetId": "nyc_filter_group",
"eventType": "submit",
"timestamp": "2025-08-05T03:51:46.125136Z"
}
}
}
]
}
{
"role": "user",
"parts": [
{
"text": "The user has interacted with the UI surface named \"new_york_trip_planning_surface\". Consolidate the UI events and update the UI accordingly. Respond with an updated UI definition. You may update any of the surfaces, or delete them if they are no longer needed."
}
]
}
With functions:
'provideFinalOutput',
[AiLoggingSeverity.info] ****** Completed Inference ******
Latency = 19491ms}
Output tokens = 2248
Prompt tokens = 7256
[AiLoggingSeverity.info] ****** Gen UI Output ******.
{
"actions": [
{
"action": "update",
"definition": {
"root": "main_column",
"widgets": [
{
"widget": {
"Column": {
"mainAxisAlignment": "start",
"children": [
"filter_chips_column",
"nyc_itinerary",
"trailhead_buttons"
],
"crossAxisAlignment": "start",
"spacing": 24
}
},
"id": "main_column"
},
{
"widget": {
"Column": {
"spacing": 16,
"children": [
"prompt_text",
"nyc_filter_group"
],
"mainAxisAlignment": "start",
"crossAxisAlignment": "start"
}
},
"id": "filter_chips_column"
},
{
"id": "prompt_text",
"widget": {
"text": {
"text": "Tell me more about your ideal trip to New York City:"
}
}
},
{
"widget": {
"filterChipGroup": {
"children": [
"activity_chip",
"duration_chip",
"travelers_chip",
"budget_chip"
],
"submitLabel": "Generate Itinerary"
}
},
"id": "nyc_filter_group"
},
{
"widget": {
"optionsFilterChip": {
"chipLabel": "Activity",
"options": [
"Art & Culture",
"Outdoors & Nature",
"Food & Drink",
"Shopping",
"Nightlife"
],
"iconChild": "sports_icon"
}
},
"id": "activity_chip"
},
{
"id": "sports_icon",
"widget": {
"image": {
"url": "https://genui-demo-stockimages.web.app/Sports_icon.png"
}
}
},
{
"id": "duration_chip",
"widget": {
"optionsFilterChip": {
"chipLabel": "Duration",
"options": [
"Weekend Getaway (2-3 days)",
"Short Trip (4-5 days)",
"Week-long (6-7 days)",
"Extended Stay (1+ week)"
]
}
}
},
{
"widget": {
"optionsFilterChip": {
"chipLabel": "Travelers",
"options": [
"Solo",
"Couple",
"Family",
"Friends"
]
}
},
"id": "travelers_chip"
},
{
"id": "budget_chip",
"widget": {
"optionsFilterChip": {
"chipLabel": "Budget",
"options": [
"Budget-friendly",
"Mid-range",
"Luxury"
]
}
}
},
{
"id": "nyc_itinerary",
"widget": {
"itinerary_with_details": {
"subheading": "A 4-day cultural and artistic journey for two.",
"title": "New York City: Art & Culture Getaway",
"child": "itinerary_column",
"imageChild": "nyc_art_image"
}
}
},
{
"widget": {
"image": {
"fit": "cover",
"url": "https://genui-demo-stockimages.web.app/The_promenade,_Blackpool,_Lancashire,_England,_ca._1898.jpg"
}
},
"id": "nyc_art_image"
},
{
"id": "itinerary_column",
"widget": {
"Column": {
"mainAxisAlignment": "start",
"children": [
"day1_header",
"day1_item1",
"day1_item2",
"day2_header",
"day2_item1",
"day2_item2",
"day3_header",
"day3_item1",
"day3_item2",
"day4_header",
"day4_item1",
"day4_item2"
],
"crossAxisAlignment": "start",
"spacing": 16
}
}
},
{
"widget": {
"sectionHeader": {
"title": "Day 1: Museum Masterpieces & Broadway Magic",
"subtitle": "Immerse yourselves in classic art and a quintessential Broadway experience."
}
},
"id": "day1_header"
},
{
"id": "day1_item1",
"widget": {
"itinerary_item": {
"imageChild": "day1_item1_image",
"detailText": "Spend your morning exploring the vast collections of The Metropolitan Museum of Art, from ancient Egyptian artifacts to European paintings. Afterwards, take a leisurely stroll through Central Park, just adjacent to the museum.",
"title": "The Met & Central Park",
"subtitle": "Morning & Afternoon"
}
}
},
{
"widget": {
"image": {
"fit": "cover",
"url": "https://genui-demo-stockimages.web.app/Green_sand_closeup_1.jpg"
}
},
"id": "day1_item1_image"
},
{
"id": "day1_item2",
"widget": {
"itinerary_item": {
"subtitle": "Evening",
"imageChild": "day1_item2_image",
"title": "Broadway Show",
"detailText": "Enjoy an early dinner in the Theater District before catching a world-class Broadway show. Choose from a variety of musicals and plays for an unforgettable evening."
}
}
},
{
"widget": {
"image": {
"url": "https://genui-demo-stockimages.web.app/Brighton,_the_front_and_the_chain_pier_seen_in_the_distance.jpg",
"fit": "cover"
}
},
"id": "day1_item2_image"
},
{
"widget": {
"sectionHeader": {
"subtitle": "Discover modern art and the vibrant energy of Times Square.",
"title": "Day 2: Modern Art & Midtown Lights"
}
},
"id": "day2_header"
},
{
"id": "day2_item1",
"widget": {
"itinerary_item": {
"detailText": "Start your day at the Museum of Modern Art (MoMA), home to masterpieces by Picasso, Van Gogh, Warhol, and more. Allow ample time to explore its extensive collections.",
"subtitle": "Morning & Afternoon",
"imageChild": "day2_item1_image",
"title": "Museum of Modern Art (MoMA)"
}
}
},
{
"widget": {
"image": {
"fit": "cover",
"url": "https://genui-demo-stockimages.web.app/Monte_Carlo_Casino_seaside_facade_before_1878_-_Bonillo_2004_p113.jpg"
}
},
"id": "day2_item1_image"
},
{
"id": "day2_item2",
"widget": {
"itinerary_item": {
"title": "Times Square & Hell's Kitchen Dinner",
"subtitle": "Evening",
"imageChild": "day2_item2_image",
"detailText": "Experience the dazzling lights and bustling atmosphere of Times Square. Afterwards, enjoy a delightful dinner in Hell's Kitchen, known for its diverse culinary scene."
}
}
},
{
"widget": {
"image": {
"url": "https://genui-demo-stockimages.web.app/Aerial_view_of_dunefield,_White_Sands_National_Park,_New_Mexico,_United_States.png",
"fit": "cover"
}
},
"id": "day2_item2_image"
},
{
"id": "day3_header",
"widget": {
"sectionHeader": {
"title": "Day 3: High Line, Chelsea & Greenwich Village",
"subtitle": "Explore urban parks, vibrant markets, and contemporary art in Chelsea and Greenwich Village."
}
}
},
{
"id": "day3_item1",
"widget": {
"itinerary_item": {
"title": "The High Line & Chelsea Market",
"subtitle": "Morning & Afternoon",
"detailText": "Walk along The High Line, an elevated urban park built on historic freight rail lines, offering unique views of the city. Then, head to Chelsea Market for lunch and explore its array of food vendors and shops.",
"imageChild": "day3_item1_image"
}
}
},
{
"id": "day3_item1_image",
"widget": {
"image": {
"fit": "cover",
"url": "https://genui-demo-stockimages.web.app/Checkerboard_forest_in_Idaho.jpg"
}
}
},
{
"id": "day3_item2",
"widget": {
"itinerary_item": {
"subtitle": "Afternoon & Evening",
"title": "Whitney Museum & Greenwich Village",
"imageChild": "day3_item2_image",
"detailText": "Visit the Whitney Museum of American Art for its focus on 20th and 21st-century American art. Conclude your day by wandering through the charming streets of Greenwich Village, known for its bohemian history and lively atmosphere."
}
}
},
{
"id": "day3_item2_image",
"widget": {
"image": {
"fit": "cover",
"url": "https://genui-demo-stockimages.web.app/The_Queen_Mary's_Army_Auxiliary_Corps_during_the_First_World_War,_France_Q11512.jpg"
}
}
},
{
"id": "day4_header",
"widget": {
"sectionHeader": {
"title": "Day 4: Brooklyn Culture & Iconic Views",
"subtitle": "Explore Brooklyn's cultural side and iconic landmarks."
}
}
},
{
"widget": {
"itinerary_item": {
"title": "Brooklyn Museum & DUMBO",
"imageChild": "day4_item1_image",
"detailText": "Cross into Brooklyn to visit the Brooklyn Museum, one of the oldest and largest art museums in the United States. Afterwards, head to DUMBO, a trendy waterfront neighborhood with stunning views of the Manhattan skyline.",
"subtitle": "Morning & Afternoon"
}
},
"id": "day4_item1"
},
{
"widget": {
"image": {
"fit": "cover",
"url": "https://genui-demo-stockimages.web.app/Carters_Beach-_Sand.jpg"
}
},
"id": "day4_item1_image"
},
{
"id": "day4_item2",
"widget": {
"itinerary_item": {
"imageChild": "day4_item2_image",
"subtitle": "Afternoon",
"detailText": "Enjoy a scenic walk across the Brooklyn Bridge, taking in panoramic views of Manhattan and Brooklyn. This is a perfect way to conclude your cultural New York adventure.",
"title": "Brooklyn Bridge Walk"
}
}
},
{
"id": "day4_item2_image",
"widget": {
"image": {
"fit": "cover",
"url": "https://genui-demo-stockimages.web.app/Man_o'war_cove_near_lulworth_dorset_arp.jpg"
}
}
},
{
"widget": {
"trailhead": {
"topics": [
"Best Restaurants in NYC",
"Nightlife in NYC",
"Outdoor Activities in NYC",
"Shopping Districts in NYC"
]
}
},
"id": "trailhead_buttons"
}
]
},
"surfaceId": "new_york_trip_planning_surface"
}
]
}}
```

---
_{↴ Ported from [flutter/genui#70](https://github.com/flutter/genui/issues/70) — originally opened by [jacobsimionato](https://github.com/jacobsimionato) on 2025-08-05.
Original labels: P3}

---
### 6 comment(s) from the original issue

**[jacobsimionato](https://github.com/jacobsimionato)** commented on 2025-08-05:

I tried turning off thinking in Gemini flash. Curiously, it seemed to speed up the first step (even relative to the small output) but not really help the second step.

# Filter chips etc
flutter: Latency = 5742ms
flutter: Output tokens = 616
flutter: Prompt tokens = 6799

# Generate itinerary
flutter: Latency = 27265ms
flutter: Output tokens = 2500
flutter: Prompt tokens = 7090

---

**[jacobsimionato](https://github.com/jacobsimionato)** commented on 2025-08-05:

Tried remove the long list of image URLs from the prompt input, but it also didn't seem to help so much.

## Filter chips
- Latency = 5970ms
- Output tokens = 499
- Prompt tokens = 3453

## Itinerary
- Latency = 19606ms
- Output tokens = 2177
- Prompt tokens = 3805

---

**[jacobsimionato](https://github.com/jacobsimionato)** commented on 2025-08-05:

Removing about half the catalog items to simplify the output schema:

## Filter chips
flutter: Latency = 4574ms
flutter: Output tokens = 319
flutter: Prompt tokens = 2538

## Itinerary
flutter: Latency = 22150ms
flutter: Output tokens = 2331
flutter: Prompt tokens = 2805

---

**[sethladd](https://github.com/sethladd)** commented on 2025-09-25:

Do we still plan to do this @jacobsimionato ?

---

**[jacobsimionato](https://github.com/jacobsimionato)** commented on 2025-09-26:

Yes absolutely. This is an ongoing strand of work, and I'd like to keep this issue open to track our investigations and directions. The three main things I think we should try in order of priority are:
1. Implement streaming, so at least some content arrives quickly
2. Try to get this working with smaller models e.g. Gemini Flash Lite and verify the reliability is still good. I see some encouraging results here with my A2UI evaluations. A possible pattern for agents could be to use a small model to triage user interactions and only delegate to a larger model for more complex requests.
3. Try experimenting with the format to make it more efficient in terms of output tokens to see if we can optimize here.

---

**[jacobsimionato](https://github.com/jacobsimionato)** commented on 2025-09-26:

Re the modes, I'm seeing really promising eval data from Flash Lite.

For simple UI generation, it seems to have similar reliability to Flash and be ~25% faster, and much cheaper too of course.

```
----------------------------------------
Model: gemini-2.5-pro (thinking: 1000)
----------------------------------------
Prompt Name Avg Latency (ms) Failed Runs Tool Error Runs
-----------------------------------------------------------------------------------------------
dogBreedGenerator 22839ms
loginForm 13380ms
productGallery 12530ms
settingsPage 15422ms
streamHeader 4251ms
dataModelUpdate 7730ms
uiRoot 8204ms
animalKingdomExplorer 43162ms
recipeCard 16191ms
musicPlayer 12492ms
weatherForecast 20331ms
surveyForm 15891ms 1 / 5
flightBooker 15772ms 1 / 5
dashboard 15027ms
contactCard 13802ms
calendarEventCreator 25366ms
checkoutPage 14617ms
socialMediaPost 16461ms
eCommerceProductPage 21480ms
interactiveDashboard 19534ms 3 / 5
travelItinerary 27920ms
-----------------------------------------------------------------------------------------------
Total failed runs: 5 / 105

----------------------------------------
Model: gemini-2.5-flash (thinking: 0)
----------------------------------------
Prompt Name Avg Latency (ms) Failed Runs Tool Error Runs
-----------------------------------------------------------------------------------------------
dogBreedGenerator 6736ms 1 / 5
loginForm 3713ms
productGallery 3150ms
settingsPage 4928ms
streamHeader 2417ms
dataModelUpdate 2204ms
uiRoot 2241ms
animalKingdomExplorer 15077ms
recipeCard 5048ms
musicPlayer 123013ms 2 / 5 2 / 5
weatherForecast 7399ms
surveyForm 4448ms
flightBooker 4467ms
dashboard 4640ms
contactCard 3363ms
calendarEventCreator 5475ms
checkoutPage 4577ms
socialMediaPost 4454ms
eCommerceProductPage 6647ms
interactiveDashboard 5544ms
travelItinerary 9181ms
-----------------------------------------------------------------------------------------------
Total failed runs: 3 / 105

----------------------------------------
Model: gemini-2.5-flash-lite (thinking: 0)
----------------------------------------
Prompt Name Avg Latency (ms) Failed Runs Tool Error Runs
-----------------------------------------------------------------------------------------------
dogBreedGenerator 4804ms
loginForm 3608ms
productGallery 2707ms
settingsPage 3653ms
streamHeader 1945ms
dataModelUpdate 2454ms
uiRoot 2011ms
animalKingdomExplorer 10773ms
recipeCard 4380ms
musicPlayer 21160ms 1 / 5
weatherForecast 3142ms
surveyForm 3626ms
flightBooker 3472ms
dashboard 3533ms
contactCard 3510ms
calendarEventCreator 3741ms
checkoutPage 3055ms
socialMediaPost 4127ms
eCommerceProductPage 4913ms
interactiveDashboard 4478ms 1 / 5
travelItinerary 6608ms
-----------------------------------------------------------------------------------------------
Total failed runs: 2 / 105
```

---