[Bug]: AttributeError: 'DynamicCacheWithRepeat' object has no attribute 'key_cache'

Describe the bug

AttributeError: 'DynamicCacheWithRepeat' object has no attribute 'key_cache'

from transformers import pipeline
from minference import MInference

pipe = pipeline("text-generation", model="/home/xx/workspace/model_dir/Qwen2.5-1.5B-Instruct", torch_dtype="auto", devi
...: ce_map="auto")
...:
...: # Patch MInference Module,
...: # If you use the local path, please use the model_name from HF when initializing MInference.
...: model_name = "Qwen/Qwen2.5-7B-Instruct"
In [2]: minference_patch = MInference(attn_type="minference", model_name=model_name, kv_type="quest")
...: pipe.model = minference_patch(pipe.model)
...:
...: pipe('你好', max_length=10)

Truncation was not explicitly activated but max_length is provided a specific value, please use truncation=True to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to truncation.
Both max_new_tokens (=256) and max_length(=10) seem to have been set. max_new_tokens will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)

AttributeError Traceback (most recent call last)
Cell In[3], line 1
----> 1 pipe('你好', max_length=10)

File ~/anaconda3/lib/python3.13/site-packages/transformers/pipelines/text_generation.py:332, in TextGenerationPipeline.call(self, text_inputs, **kwargs)
330 else:
331 return super().call(list(chats), **kwargs)
--> 332 return super().call(text_inputs, **kwargs)

File ~/anaconda3/lib/python3.13/site-packages/transformers/pipelines/base.py:1467, in Pipeline.call(self, inputs, num_workers, batch_size, *args, **kwargs)
1459 return next(
1460 iter(
1461 self.get_iterator(
(...)
1464 )
1465 )
1466 else:
-> 1467 return self.run_single(inputs, preprocess_params, forward_params, postprocess_params)

File ~/anaconda3/lib/python3.13/site-packages/transformers/pipelines/base.py:1474, in Pipeline.run_single(self, inputs, preprocess_params, forward_params, postprocess_params)
1472 def run_single(self, inputs, preprocess_params, forward_params, postprocess_params):
1473 model_inputs = self.preprocess(inputs, **preprocess_params)
-> 1474 model_outputs = self.forward(model_inputs, **forward_params)
1475 outputs = self.postprocess(model_outputs, **postprocess_params)
1476 return outputs

File ~/anaconda3/lib/python3.13/site-packages/transformers/pipelines/base.py:1374, in Pipeline.forward(self, model_inputs, **forward_params)
1372 with inference_context():
1373 model_inputs = self._ensure_tensor_on_device(model_inputs, device=self.device)
-> 1374 model_outputs = self._forward(model_inputs, **forward_params)
1375 model_outputs = self._ensure_tensor_on_device(model_outputs, device=torch.device("cpu"))
1376 else:

File ~/anaconda3/lib/python3.13/site-packages/transformers/pipelines/text_generation.py:432, in TextGenerationPipeline._forward(self, model_inputs, **generate_kwargs)
429 if "generation_config" not in generate_kwargs:
430 generate_kwargs["generation_config"] = self.generation_config
--> 432 output = self.model.generate(input_ids=input_ids, attention_mask=attention_mask, **generate_kwargs)
434 if isinstance(output, ModelOutput):
435 generated_sequence = output.sequences

File ~/anaconda3/lib/python3.13/site-packages/torch/utils/_contextlib.py:120, in context_decorator..decorate_context(*args, **kwargs)
117 @functools.wraps(func)
118 def decorate_context(*args, **kwargs):
119 with ctx_factory():
--> 120 return func(*args, **kwargs)

File ~/anaconda3/lib/python3.13/site-packages/transformers/generation/utils.py:2539, in GenerationMixin.generate(self, inputs, generation_config, logits_processor, stopping_criteria, prefix_allowed_tokens_fn, synced_gpus, assistant_model, streamer, negative_prompt_ids, negative_prompt_attention_mask, use_model_defaults, custom_generate, **kwargs)
2528 return GenerationMixin.generate(
2529 self,
2530 inputs,
(...)
2534 **kwargs,
2535 )
2537 elif generation_mode in (GenerationMode.SAMPLE, GenerationMode.GREEDY_SEARCH):
2538 # 11. run sample (it degenerates to greedy search when generation_config.do_sample=False)
-> 2539 result = self._sample(
2540 input_ids,
2541 logits_processor=prepared_logits_processor,
2542 stopping_criteria=prepared_stopping_criteria,
2543 generation_config=generation_config,
2544 synced_gpus=synced_gpus,
2545 streamer=streamer,
2546 **model_kwargs,
2547 )
2549 elif generation_mode in (GenerationMode.BEAM_SAMPLE, GenerationMode.BEAM_SEARCH):
2550 # 11. run beam sample
2551 result = self._beam_search(
2552 input_ids,
2553 logits_processor=prepared_logits_processor,
(...)
2557 **model_kwargs,
2558 )

File ~/anaconda3/lib/python3.13/site-packages/transformers/generation/utils.py:2835, in GenerationMixin._sample(self, input_ids, logits_processor, stopping_criteria, generation_config, synced_gpus, streamer, **model_kwargs)
2833 this_peer_finished = False
2834 unfinished_sequences = torch.ones(batch_size, dtype=torch.long, device=input_ids.device)
-> 2835 model_kwargs = self._get_initial_cache_position(cur_len, input_ids.device, model_kwargs)
2837 model_forward = self.call
2838 compile_forward = self._valid_auto_compile_criteria(model_kwargs, generation_config)

File ~/anaconda3/lib/python3.13/site-packages/transformers/generation/utils.py:1810, in GenerationMixin._get_initial_cache_position(self, seq_length, device, model_kwargs)
1808 past_length = cache[0][0].shape[2]
1809 elif hasattr(cache, "get_seq_length"):
-> 1810 past_length = cache.get_seq_length()
1812 cache_position = cache_position[past_length:]
1814 model_kwargs["cache_position"] = cache_position

File ~/anaconda3/lib/python3.13/site-packages/minference/modules/kvcompression.py:439, in DynamicCacheWithRepeat.get_seq_length(self, layer_idx)
438 def get_seq_length(self, layer_idx=0):
--> 439 if len(self.key_cache) <= layer_idx:
440 return 0
441 return self._seen_tokens

AttributeError: 'DynamicCacheWithRepeat' object has no attribute 'key_cache'

In [4]:

Steps to reproduce

No response

Expected Behavior

No response

Logs

No response

Additional Information

No response

Start at minference/modules/kvcompression.py:439 and inspect DynamicCacheWithRepeat.get_seq_length together with how that cache is initialized. Re-run the supplied Transformers pipeline example with the Qwen model and kv_type="quest" to confirm the failure. Done means text generation completes without the missing key_cache AttributeError.