googleapis / googleapis/python-genai

Exclusion of Tools Are Not Respected for Computer Use in Gemini 3 Flash Preview and Gemini 2.5 Computer Use

Aperta
#1,885 2 commenti 0 reazioni 1 assegnatario Rivendicata da @kkorpal Vedi su GitHub
priority: p2 type: bug
Lingua principale
Python
Stelle
4k
Fork
1k
Merge medio
2g 12h
PR unite (30g)
41

Descrizione

When passing in the `excluded_predefined_functions` field, the excluded tools are not respected for ComputerUse models (`gemini-2.5-computer-use-preview-10-2025` and `gemini-3-flash-preview`.

#### Environment details

- Programming language: Python
- OS: macOS
- Language runtime version: 3.13 Python
- Package version: 1.56.0

#### Steps to reproduce

Run this script:
```
"""Minimal test to verify if Gemini exclusions work."""

import os
import base64
from pathlib import Path

# Load env
from dotenv import load_dotenv
load_dotenv(Path(__file__).parent / ".env")
# load_dotenv(Path(__file__).parent / ".env.local")

from google import genai
from google.genai import types

# Initialize client
api_key = os.getenv("GOOGLE_API_KEY")
client = genai.Client(api_key=api_key)

# All browser-specific functions we want to exclude
excluded_functions = [
"open_web_browser",
"search",
"navigate",
"go_forward",
"go_back",
"scroll_document", # Keep only coordinate-based scroll_at
]

print("=" * 60)
print("Testing Gemini Computer Use Exclusions")
print("=" * 60)
print(f"\nExcluded functions: {excluded_functions}")

# Config with exclusions
config = types.GenerateContentConfig(
tools=[
types.Tool(
computer_use=types.ComputerUse(
environment=types.Environment.ENVIRONMENT_BROWSER,
excluded_predefined_functions=excluded_functions,
)
)
],
)

# Test 1: Without screenshot - expect navigate/open_web_browser (maybe acceptable)
print("\n--- Test 1: Without screenshot ---")
contents = [
types.Content(
role="user",
parts=[types.Part(text="Go to github.com/trycua/cua")]
)
]

response = client.models.generate_content(
model="gemini-2.5-computer-use-preview-10-2025",
contents=contents,
config=config,
)

for part in response.candidates[0].content.parts:
if hasattr(part, 'function_call') and part.function_call:
print(f" Function called: {part.function_call.name}")
print(f" Args: {dict(part.function_call.args)}")
if part.function_call.name in excluded_functions:
print(f" ❌ EXCLUSION NOT RESPECTED!")
else:
print(f" ✅ Function allowed")

# Test 2: With a dummy screenshot - should NOT use navigate
print("\n--- Test 2: With screenshot (simulating browser already open) ---")

# Create a simple test image (1x1 red pixel PNG)
# In real test, you'd use an actual screenshot
import struct
import zlib

def create_minimal_png():
"""Create a minimal valid PNG image."""
# PNG signature
signature = b'\x89PNG\r\n\x1a\n'

# IHDR chunk (image header)
width = 100
height = 100
bit_depth = 8
color_type = 2 # RGB
ihdr_data = struct.pack('>IIBBBBB', width, height, bit_depth, color_type, 0, 0, 0)
ihdr_crc = zlib.crc32(b'IHDR' + ihdr_data) & 0xffffffff
ihdr_chunk = struct.pack('>I', 13) + b'IHDR' + ihdr_data + struct.pack('>I', ihdr_crc)

# IDAT chunk (image data) - simple gray image
raw_data = b''
for y in range(height):
raw_data += b'\x00' # filter byte
for x in range(width):
raw_data += b'\x80\x80\x80' # gray RGB

compressed = zlib.compress(raw_data)
idat_crc = zlib.crc32(b'IDAT' + compressed) & 0xffffffff
idat_chunk = struct.pack('>I', len(compressed)) + b'IDAT' + compressed + struct.pack('>I', idat_crc)

# IEND chunk
iend_crc = zlib.crc32(b'IEND') & 0xffffffff
iend_chunk = struct.pack('>I', 0) + b'IEND' + struct.pack('>I', iend_crc)

return signature + ihdr_chunk + idat_chunk + iend_chunk

test_image = create_minimal_png()
print(f" Test image size: {len(test_image)} bytes")

contents_with_image = [
types.Content(
role="user",
parts=[
types.Part(text="Go to github.com/trycua/cua. Click on the Issues tab."),
types.Part.from_bytes(data=test_image, mime_type="image/png"),
]
)
]

response2 = client.models.generate_content(
model="gemini-2.5-computer-use-preview-10-2025",
contents=contents_with_image,
config=config,
)

for part in response2.candidates[0].content.parts:
if hasattr(part, 'function_call') and part.function_call:
print(f" Function called: {part.function_call.name}")
print(f" Args: {dict(part.function_call.args)}")
if part.function_call.name in excluded_functions:
print(f" ❌ EXCLUSION NOT RESPECTED!")
else:
print(f" ✅ Function allowed")

print("\n" + "=" * 60)
print("Test complete")
print("=" * 60)
```

and you should see that the model is invoking excluded tools

Thanks!

Guida per i contributori

Apri la guida per i contributori

Valutazione

Questa issue non è ancora stata valutata.

Ricevi le nuove issue nella tua casella

Un breve riepilogo di issue GitHub adatte ai principianti.