support for deeply nested models

hey there,

I realize that it is possible to define a nested pydantic model for my vector db. However, when I add a second level of nested fields, I get RustPanic: rust future panicked: unknown error error.

what I want to achieve is to add an optional metadata field which could be empty depending on the document type.

here is an example to demonstrate the issue.

from datetime import datetime
from zoneinfo import ZoneInfo
from typing import Optional, Dict, Any, Union

from pydantic import BaseModel, Field
from lancedb.pydantic import LanceModel, Vector
import lancedb
from lancedb.embeddings import get_registry

db = lancedb.connect("/tmp/db")

registry = get_registry()
embedding_function = registry.get("openai").create(
    model="text-embedding-ada-002",
)



class HeaderMetadata(BaseModel):
    """Metadata for document headers"""
    level: int = Field(..., description="Header level (1-4)")
    title: str = Field(..., description="Header title")

class DocumentChunkMetadata(BaseModel):
    """Metadata for the document chunk"""
    headers: Optional[HeaderMetadata] = Field(default=None, description="Headers from level 1 to 4")
    page: int = Field(description="Page number")

class DocumentChunk(LanceModel):
    """Main document chunk model"""
    parent_id: str = Field(description="Parent document ID")
    chunk_id: str = Field(description="Unique chunk ID")
    content: str = embedding_function.SourceField()
    contentVector: Vector(embedding_function.ndims()) = embedding_function.VectorField()
    file_name: str = Field( description="Original file name")
    file_type: str = Field( description="File type (pdf, ipynb, etc.)")
    uploaded_at: datetime = Field(default_factory=datetime.utcnow, description="Upload timestamp")
    chunk_size_bytes: int = Field(description="Size of the chunk in bytes")

    metadata: DocumentChunkMetadata = Field(description="Chunk metadata including headers")


table = db.create_table("documents_1", schema=DocumentChunk, mode="overwrite")
table.create_fts_index("content", use_tantivy=False)




content1 = "hello world"
content2 = "goodby world"

data = [{
       "parent_id": "doc_123",
        "chunk_id": "chunk_1",
        "content": content1,
        "contentVector": embeddings.get_embedding(content1)
        "file_name": "document.pdf",
        "file_type": "pdf",
        "uploaded_at": datetime.utcnow(),
        "chunk_size_bytes": 1024,

        "metadata": {
        
        "page": 1,
     
}},
{
       "parent_id": "doc_123",
        "chunk_id": "chunk_2",
        "content": content2,
        "contentVector": embeddings.get_embedding(content2),
        "file_name": "document.pdf",
        "file_type": "pdf",
        "uploaded_at": datetime.utcnow(),
        "chunk_size_bytes": 1024,

        "metadata": {
            "page": 2,
     
}},
]

table.add(data)

results = table.search("greetings").limit(1).to_pydantic(DocumentChunk) ---> RAISE ERROR

if I remove the headers field from the DocumentChunkMetadata everything work just fine.

this is the error log

lancedb\\table.py:3512\u001b[39m, in \u001b[36mAsyncTable._execute_query\u001b[39m\u001b[34m(self, query, batch_size, timeout)\u001b[39m\n\u001b[32m   3499\u001b[39m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34m_execute_query\u001b[39m(\n\u001b[32m   3500\u001b[39m     \u001b[38;5;28mself\u001b[39m,\n\u001b[32m   3501\u001b[39m     query: Query,\n\u001b[32m   (...)\u001b[39m\u001b[32m   3507\u001b[39m     \u001b[38;5;66;03m# query to the async version of the query and run that here. This is only\u001b[39;00m\n\u001b[32m   3508\u001b[39m     \u001b[38;5;66;03m# used for that code path right now.\u001b[39;00m\n\u001b[32m   3510\u001b[39m     async_query = \u001b[38;5;28mself\u001b[39m._sync_query_to_async(query)\n\u001b[32m-> \u001b[39m\u001b[32m3512\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m async_query.to_batches(\n\u001b[32m   3513\u001b[39m         max_batch_length=batch_size, timeout=timeout\n\u001b[32m   3514\u001b[39m     )\n\n\u001b[36mFile \u001b[39m\u001b

\lancedb\\query.py:2785\u001b[39m, in \u001b[36mAsyncVectorQuery.to_batches\u001b[39m\u001b[34m(self, max_batch_length, timeout)\u001b[39m\n\u001b[32m   2778\u001b[39m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mto_batches\u001b[39m(\n\u001b[32m   2779\u001b[39m     \u001b[38;5;28mself\u001b[39m,\n\u001b[32m   2780\u001b[39m     *,\n\u001b[32m   2781\u001b[39m     max_batch_length: Optional[\u001b[38;5;28mint\u001b[39m] = \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[32m   2782\u001b[39m     timeout: Optional[timedelta] = \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[32m   2783\u001b[39m ) -> AsyncRecordBatchReader:\n\u001b[32m   2784\u001b[39m     reader = \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28msuper\u001b[39m().to_batches(timeout=timeout)\n\u001b[32m-> \u001b[39m\u001b[32m2785\u001b[39m     results = pa.Table.from_batches(\u001b[38;5;28;01mawait\u001b[39;00m reader.read_all(), reader.schema)\n\u001b[32m   2786\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m._reranker:\n\u001b[32m   2787\u001b[39m         results = \u001b[38;5;28mself\u001b[39m._reranker.rerank_vector(\u001b[38;5;28mself\u001b[39m._query_string, results)\n\n\u001b[36mFile \u001b[39m\u001b

\\arrow.py:48\u001b[39m, in \u001b[36mAsyncRecordBatchReader.read_all\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m     40\u001b[39m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mread_all\u001b[39m(\u001b[38;5;28mself\u001b[39m) -> List[pa.RecordBatch]:\n\u001b[32m     41\u001b[39m \u001b[38;5;250m    \u001b[39m\u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m     42\u001b[39m \u001b[33;03m    Read all the record batches from the stream\u001b[39;00m\n\u001b[32m     43\u001b[39m \n\u001b[32m   (...)\u001b[39m\u001b[32m     46\u001b[39m \u001b[33;03m    If there are a lot of results this may consume a lot of memory\u001b[39;00m\n\u001b[32m     47\u001b[39m \u001b[33;03m    \"\"\"\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m48\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m [batch \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mfor\u001b[39;00m batch \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m]\n\n\u001b[36mFile \u001b[39m\u001b

\\lancedb\\arrow.py:48\u001b[39m, in \u001b[36m<listcomp>\u001b[39m\u001b[34m(.0)\u001b[39m\n\u001b[32m     40\u001b[39m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mread_all\u001b[39m(\u001b[38;5;28mself\u001b[39m) -> List[pa.RecordBatch]:\n\u001b[32m     41\u001b[39m \u001b[38;5;250m    \u001b[39m\u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m     42\u001b[39m \u001b[33;03m    Read all the record batches from the stream\u001b[39;00m\n\u001b[32m     43\u001b[39m \n\u001b[32m   (...)\u001b[39m\u001b[32m     46\u001b[39m \u001b[33;03m    If there are a lot of results this may consume a lot of memory\u001b[39;00m\n\u001b[32m     47\u001b[39m \u001b[33;03m    \"\"\"\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m48\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m [batch \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mfor\u001b[39;00m batch \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m]\n\n\u001b[36mFile \u001b[39m\u001b


\Lib\\site-packages\\lancedb\\arrow.py:54\u001b[39m, in \u001b[36mAsyncRecordBatchReader.__anext__\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m     53\u001b[39m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34m__anext__\u001b[39m(\u001b[38;5;28mself\u001b[39m) -> pa.RecordBatch:\n\u001b[32m---> \u001b[39m\u001b[32m54\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m._inner.\u001b[34m__anext__\u001b[39m()\n\n\u001b[31mRustPanic\u001b[39m: rust future panicked: unknown error"
```

Start with the Python paths named in the traceback: lancedb/table.py, lancedb/query.py, and lancedb/arrow.py, then trace the to_pydantic(DocumentChunk) result path using the supplied nested-model reproducer. Done means an optional second-level Pydantic model can be queried and converted without a RustPanic, while the existing non-nested case continues to work.