docling-project / docling-project/docling
Docling is unable to extract handwritten details from scanned form screenshots; only Customer ID is recognized correctly
- Dominant language
- Python
- Stars
- 66.4k
- Forks
- 4.8k
- Avg merge
- 2d 21h
- Merged PRs (30d)
- 84
Description
### Bug
When using Docling to process a screenshot of a filled PDF form (handwritten), only the Customer ID is extracted correctly. Other fields, such as ABHA number and contact details, are not recognized or are partially incorrect.
The issue occurs when using granite-docling-258M model on PNG images of scanned forms.
...
### Steps to reproduce
Take a screenshot of a filled PDF form with handwritten text.
Save the screenshot as PNG (e.g., ScreenShotLIC.png).
Run the following Jupyter notebook code:
```
# Granite Docling Benchmark (Image-only, with token)
# =========================================================
import time
import torch
from transformers import AutoModelForVision2Seq, AutoProcessor
from PIL import Image
from IPython.display import display, HTML
import warnings
warnings.filterwarnings("ignore")
from transformers import logging
logging.set_verbosity_error()
import os
os.environ["OMP_NUM_THREADS"] = "4"
torch.set_num_threads(4)
# -------------------------------
# Model and documents setup
# -------------------------------
model_name = "ibm-granite/granite-docling-258M"
documents = [
{
"file_path": "/home/ibm-user/ScreenShotLIC.png", # update path if needed
"extra_info": {"doc_id": "img1", "source": "manual"}
}
]
# Must include placeholder
question = (
"Here is a document: \n"
"Question: what is contact details ?"
)
devices_to_test = ["cpu", "nnpa"]
results = []
# -------------------------------
# Run inference
# -------------------------------
for device_type in devices_to_test:
try:
if device_type == "nnpa":
import torch_nnpa
device = torch_nnpa.device()
device_label = "NNPA"
else:
device = torch.device("cpu")
device_label = "CPU"
display(HTML(f"
Running on: {device_label}
")) # Load model + processor
start_pipe = time.time()
model = AutoModelForVision2Seq.from_pretrained(
model_name,
trust_remote_code=True
).to(device)
processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
pipeline_loading_time = time.time() - start_pipe
for doc in documents:
file_path = doc["file_path"]
# Load image
img = Image.open(file_path).convert("RGB")
img = img.resize((512, 512))
# Inference
start = time.time()
inputs = processor(
text=[question], # includes
images=[img], # same length as tokens
return_tensors="pt"
).to(device)
#outputs = model.generate(**inputs, max_new_tokens=4, do_sample=False, num_beams=1)
outputs = model.generate(**inputs, max_new_tokens=32)
avg_inference_time = time.time() - start
response = processor.batch_decode(outputs, skip_special_tokens=True)[0]
tokens_generated = len(response.split())
tokens_per_sec = tokens_generated / avg_inference_time if avg_inference_time > 0 else 0
results.append({
"device": device_label,
"doc_id": doc["extra_info"].get("doc_id", ""),
"pipeline_loading_time": pipeline_loading_time,
"inference_time": avg_inference_time,
"tokens_generated": tokens_generated,
"tokens_per_sec": tokens_per_sec,
"response": response
})
except Exception as ex:
print(f" Error on {device_label}: {ex}")
# -------------------------------
# Styled CPU vs NNPA Comparison
# -------------------------------
metrics = ["pipeline_loading_time", "inference_time", "tokens_generated", "tokens_per_sec"]
labels = ["Pipeline Load Time (s)", "Avg Inference Time (s)", "Tokens Generated", "Tokens per second"]
if len(results) > 1:
cpu_result = next((r for r in results if r["device"] == "CPU"), {})
nnpa_result = next((r for r in results if r["device"] == "NNPA"), {})
comparison_html = """
CPU vs NNPA Performance Comparison
"""
row_styles = ["background-color: #ffffff;", "background-color: #f9f9f9;"]
for i, (label, metric) in enumerate(zip(labels, metrics)):
cpu_val_raw = cpu_result.get(metric)
nnpa_val_raw = nnpa_result.get(metric)
cpu_val = f"{cpu_val_raw:.4f}" if isinstance(cpu_val_raw, (int, float)) else "N/A"
nnpa_val = f"{nnpa_val_raw:.4f}" if isinstance(nnpa_val_raw, (int, float)) else "N/A"
improvement_str = "N/A"
improvement_color = "#333"
if isinstance(cpu_val_raw, (int, float)) and isinstance(nnpa_val_raw, (int, float)) and cpu_val_raw > 0 and nnpa_val_raw > 0:
if metric == "inference_time":
speedup = cpu_val_raw / nnpa_val_raw
improvement_str = f"{speedup:.2f}× faster"
improvement_color = "green" if speedup > 1 else "red"
elif metric == "tokens_per_sec":
speedup = nnpa_val_raw / cpu_val_raw
improvement_str = f"{speedup:.2f}× faster"
improvement_color = "green" if speedup > 1 else "red"
else:
improvement = ((nnpa_val_raw - cpu_val_raw) / cpu_val_raw) * 100
improvement_str = f"{improvement:.1f}%"
improvement_color = "green" if improvement > 0 else "red"
row_style = row_styles[i % 2]
comparison_html += f"""
"""
comparison_html += "
Metric
CPU
NNPA
Improvement
{label}
{cpu_val}
{nnpa_val}
{improvement_str}
"
# Add generated responses for clarity
comparison_html += f"""
Generated Answer (NNPA): {nnpa_result.get("response", "N/A")}
"""
display(HTML(comparison_html))
else:
display(HTML("
Not enough results to compare CPU and NNPA.
"))```
Additional Notes / Environment:
Model: ibm-granite/granite-docling-258M
File type: PNG screenshot of a filled form
Devices tested: CPU and NNPA
Image preprocessing: Resizing to 512×512 or 1024×1024
Docling version: (provide version if installed)
...
### Docling version
latest version with transformers
...
### Python version
python 10
...
Contributor guide
Assessment
This issue has not been assessed yet.