LangChain & ColBERT API Mismatch

1.
```sh
---------------------------------------------------------------------------
ModuleNotFoundError Traceback (most recent call last)
Cell In[16], [line 1](vscode-notebook-cell:?execution_count=16&line=1)
----> [1](vscode-notebook-cell:?execution_count=16&line=1) from ragatouille import RAGPretrainedModel
2 RAG = RAGPretrainedModel.from_pretrained("colbert-ir/colbertv2.0")

File ~/mamba/envs/llm/lib/python3.12/site-packages/ragatouille/__init__.py:21
14 warnings.warn(
15 _FUTURE_MIGRATION_WARNING_MESSAGE,
16 UserWarning,
17 stacklevel=2 # Ensures the warning points to the user's import line
18 )
20 __version__ = "0.0.9post2"
---> [21](https://file+.vscode-resource.vscode-cdn.net/Users/visomopokoa8/Documents/GitHub/codex-assistant/knowledge-base/10%20-%20Learning/90%20-%20Practices/91%20-%20LLMs%20%26%20Agents/21%20-%20rag/~/mamba/envs/llm/lib/python3.12/site-packages/ragatouille/__init__.py:21) from .RAGPretrainedModel import RAGPretrainedModel
22 from .RAGTrainer import RAGTrainer
24 __all__ = ["RAGPretrainedModel", "RAGTrainer"]

File ~/mamba/envs/llm/lib/python3.12/site-packages/ragatouille/RAGPretrainedModel.py:5
2 from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, TypeVar, Union
3 from uuid import uuid4
----> [5](https://file+.vscode-resource.vscode-cdn.net/Users/visomopokoa8/Documents/GitHub/codex-assistant/knowledge-base/10%20-%20Learning/90%20-%20Practices/91%20-%20LLMs%20%26%20Agents/21%20-%20rag/~/mamba/envs/llm/lib/python3.12/site-packages/ragatouille/RAGPretrainedModel.py:5) from langchain.retrievers.document_compressors.base import BaseDocumentCompressor
6 from langchain_core.retrievers import BaseRetriever
8 from ragatouille.data.corpus_processor import CorpusProcessor

ModuleNotFoundError: No module named 'langchain.retrievers'
```

**Fix:**
```python
File ragatouille/RAGPretrainedModel.py:5

from langchain.retrievers.document_compressors.base import BaseDocumentCompressor
--> from langchain_classic.retrievers.document_compressors.base import BaseDocumentCompressor
```

---

2.
```sh
---------------------------------------------------------------------------
ModuleNotFoundError Traceback (most recent call last)
Cell In[17], line 1
----> 1 from ragatouille import RAGPretrainedModel
2 RAG = RAGPretrainedModel.from_pretrained("colbert-ir/colbertv2.0")

File ~/mamba/envs/llm/lib/python3.12/site-packages/ragatouille/__init__.py:21
14 warnings.warn(
15 _FUTURE_MIGRATION_WARNING_MESSAGE,
16 UserWarning,
17 stacklevel=2 # Ensures the warning points to the user's import line
18 )
20 __version__ = "0.0.9post2"
---> 21 from .RAGPretrainedModel import RAGPretrainedModel
22 from .RAGTrainer import RAGTrainer
24 __all__ = ["RAGPretrainedModel", "RAGTrainer"]

File ~/mamba/envs/llm/lib/python3.12/site-packages/ragatouille/RAGPretrainedModel.py:10
8 from ragatouille.data.corpus_processor import CorpusProcessor
9 from ragatouille.data.preprocessors import llama_index_sentence_splitter
---> 10 from ragatouille.integrations import (
11 RAGatouilleLangChainCompressor,
12 RAGatouilleLangChainRetriever,
13 )
14 from ragatouille.models import ColBERT, LateInteractionModel
17 class RAGPretrainedModel:

File ~/mamba/envs/llm/lib/python3.12/site-packages/ragatouille/integrations/__init__.py:1
----> 1 from ragatouille.integrations._langchain import (
2 RAGatouilleLangChainCompressor,
3 RAGatouilleLangChainRetriever,
4 )
6 __all__ = ["RAGatouilleLangChainRetriever", "RAGatouilleLangChainCompressor"]

File ~/mamba/envs/llm/lib/python3.12/site-packages/ragatouille/integrations/_langchain.py:3
1 from typing import Any, List, Optional, Sequence
----> 3 from langchain.retrievers.document_compressors.base import BaseDocumentCompressor
4 from langchain_core.callbacks.manager import CallbackManagerForRetrieverRun, Callbacks
5 from langchain_core.documents import Document

ModuleNotFoundError: No module named 'langchain.retrievers'
```

**Fix:**
```python
File ragatouille/integrations/_langchain.py:3

from langchain.retrievers.document_compressors.base import BaseDocumentCompressor
--> from langchain_classic.retrievers.document_compressors.base import BaseDocumentCompressor
```

---

3.
```sh
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
Cell In[18], line 2
1 from ragatouille import RAGPretrainedModel
----> 2 RAG = RAGPretrainedModel.from_pretrained("colbert-ir/colbertv2.0")

File ~/mamba/envs/llm/lib/python3.12/site-packages/ragatouille/RAGPretrainedModel.py:71, in RAGPretrainedModel.from_pretrained(cls, pretrained_model_name_or_path, n_gpu, verbose, index_root)
59 """Load a ColBERT model from a pre-trained checkpoint.
60
61 Parameters:
(...) 68 cls (RAGPretrainedModel): The current instance of RAGPretrainedModel, with the model initialised.
69 """
70 instance = cls()
---> 71 instance.model = ColBERT(
72 pretrained_model_name_or_path, n_gpu, index_root=index_root, verbose=verbose
73 )
74 return instance

File ~/mamba/envs/llm/lib/python3.12/site-packages/ragatouille/models/colbert.py:84, in ColBERT.__init__(self, pretrained_model_name_or_path, n_gpu, index_name, verbose, load_from_index, training_mode, index_root, **kwargs)
81 self.config.root = self.index_root
83 if not training_mode:
---> 84 self.inference_ckpt = Checkpoint(
85 self.checkpoint, colbert_config=self.config
86 )
87 self.base_model_max_tokens = (
88 self.inference_ckpt.bert.config.max_position_embeddings
89 ) - 4
91 self.run_context = Run().context(self.run_config)

File ~/mamba/envs/llm/lib/python3.12/site-packages/colbert/modeling/checkpoint.py:75, in Checkpoint.__init__(self, name, colbert_config, verbose)
74 def __init__(self, name, colbert_config=None, verbose: int = 3):
---> 75 super().__init__(name, colbert_config)
76 assert self.training is False
78 self.verbose = verbose

File ~/mamba/envs/llm/lib/python3.12/site-packages/colbert/modeling/colbert.py:21, in ColBERT.__init__(self, name, colbert_config)
20 def __init__(self, name='bert-base-uncased', colbert_config=None):
---> 21 super().__init__(name, colbert_config)
22 self.use_gpu = colbert_config.total_visible_gpus > 0
24 ColBERT.try_load_torch_extensions(self.use_gpu)

File ~/mamba/envs/llm/lib/python3.12/site-packages/colbert/modeling/base_colbert.py:36, in BaseColBERT.__init__(self, name_or_path, colbert_config)
31 HF_ColBERT = class_factory(self.name)
33 # assert self.name is not None
34 # HF_ColBERT = class_factory(self.name)
---> 36 self.model = HF_ColBERT.from_pretrained(name_or_path, colbert_config=self.colbert_config)
37 self.model.to(DEVICE)
38 self.raw_tokenizer = AutoTokenizer.from_pretrained(name_or_path)

File ~/mamba/envs/llm/lib/python3.12/site-packages/colbert/modeling/hf_colbert.py:133, in class_factory..HF_ColBERT.from_pretrained(cls, name_or_path, colbert_config)
129 obj.base = base
131 return obj
--> 133 obj = super().from_pretrained(name_or_path, colbert_config=colbert_config)
134 obj.base = name_or_path
136 return obj

File ~/mamba/envs/llm/lib/python3.12/site-packages/transformers/modeling_utils.py:4297, in PreTrainedModel.from_pretrained(cls, pretrained_model_name_or_path, config, cache_dir, ignore_mismatched_sizes, force_download, local_files_only, token, revision, use_safetensors, weights_only, fusion_config, disable_mmap, *model_args, **kwargs)
4279 load_config = LoadStateDictConfig(
4280 pretrained_model_name_or_path=pretrained_model_name_or_path,
4281 ignore_mismatched_sizes=ignore_mismatched_sizes,
(...) 4294 disable_mmap=disable_mmap,
4295 )
4296 loading_info, disk_offload_index = cls._load_pretrained_model(model, state_dict, checkpoint_files, load_config)
-> 4297 loading_info = cls._finalize_model_loading(model, load_config, loading_info)
4298 model.eval() # Set model in evaluation mode to deactivate Dropout modules by default
4299 model.set_use_kernels(use_kernels, kernel_config)

File ~/mamba/envs/llm/lib/python3.12/site-packages/transformers/modeling_utils.py:4453, in PreTrainedModel._finalize_model_loading(model, load_config, loading_info)
4449 model.mark_tied_weights_as_initialized(loading_info)
4451 # Move missing (and potentially mismatched) keys and non-persistent buffers back to their expected device from
4452 # meta device (because they were not moved when loading the weights as they were not in the loaded state dict)
-> 4453 model._move_missing_keys_from_meta_to_device(
4454 loading_info.missing_and_mismatched(),
4455 load_config.device_map,
4456 load_config.device_mesh,
4457 load_config.hf_quantizer,
4458 )
4460 # Correctly initialize the missing (and potentially mismatched) keys (all parameters without the `_is_hf_initialized` flag)
4461 model._initialize_missing_keys(load_config.is_quantized)

File ~/mamba/envs/llm/lib/python3.12/site-packages/transformers/modeling_utils.py:4736, in PreTrainedModel._move_missing_keys_from_meta_to_device(self, missing_keys, device_map, device_mesh, hf_quantizer)
4731 return
4733 # The tied weight keys are in the "missing" usually, but they should not be moved (they will be tied anyway)
4734 # This is especially important because if they are moved, they will lose the `_is_hf_initialized` flag, and they
4735 # will be re-initialized for nothing (which can be quite long)
-> 4736 for key in missing_keys - self.all_tied_weights_keys.keys():
4737 param = self.get_parameter_or_buffer(key)
4738 param_device = get_device(device_map, key, valid_torch_device=True)

File ~/mamba/envs/llm/lib/python3.12/site-packages/torch/nn/modules/module.py:1967, in Module.__getattr__(self, name)
1965 if name in modules:
1966 return modules[name]
-> 1967 raise AttributeError(
1968 f"'{type(self).__name__}' object has no attribute '{name}'"
1969 )

AttributeError: 'HF_ColBERT' object has no attribute 'all_tied_weights_keys'
```

**Fix:**
```python
File colbert/modeling/hf_colbert.py:91-99

def class_factory(name_or_path):
...
class HF_ColBERT(pretrained_class_object):
"""
Shallow wrapper around HuggingFace transformers. All new parameters should be defined at this level.

This makes sure `{from,save}_pretrained` and `init_weights` are applied to new parameters correctly.
"""
_keys_to_ignore_on_load_unexpected = [r"cls"]

def __init__(self, config, colbert_config):
...

-->

def class_factory(name_or_path):
...
class HF_ColBERT(pretrained_class_object):
"""
Shallow wrapper around HuggingFace transformers. All new parameters should be defined at this level.

This makes sure `{from,save}_pretrained` and `init_weights` are applied to new parameters correctly.
"""
_keys_to_ignore_on_load_unexpected = [r"cls"]

# Required by newer transformers
all_tied_weights_keys = {}

# Keep compatible with code paths expecting tied-weight keys
_tied_weights_keys = set()

def __init__(self, config, colbert_config):
...
```

4.
```sh
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
Cell In[1], line 2
1 from ragatouille import RAGPretrainedModel
----> 2 RAG = RAGPretrainedModel.from_pretrained("colbert-ir/colbertv2.0")

File ~/mamba/envs/llm/lib/python3.12/site-packages/ragatouille/RAGPretrainedModel.py:71, in RAGPretrainedModel.from_pretrained(cls, pretrained_model_name_or_path, n_gpu, verbose, index_root)
59 """Load a ColBERT model from a pre-trained checkpoint.
60
61 Parameters:
(...) 68 cls (RAGPretrainedModel): The current instance of RAGPretrainedModel, with the model initialised.
69 """
70 instance = cls()
---> 71 instance.model = ColBERT(
72 pretrained_model_name_or_path, n_gpu, index_root=index_root, verbose=verbose
73 )
74 return instance

File ~/mamba/envs/llm/lib/python3.12/site-packages/ragatouille/models/colbert.py:84, in ColBERT.__init__(self, pretrained_model_name_or_path, n_gpu, index_name, verbose, load_from_index, training_mode, index_root, **kwargs)
81 self.config.root = self.index_root
83 if not training_mode:
---> 84 self.inference_ckpt = Checkpoint(
85 self.checkpoint, colbert_config=self.config
86 )
87 self.base_model_max_tokens = (
88 self.inference_ckpt.bert.config.max_position_embeddings
89 ) - 4
91 self.run_context = Run().context(self.run_config)

File ~/mamba/envs/llm/lib/python3.12/site-packages/colbert/modeling/checkpoint.py:75, in Checkpoint.__init__(self, name, colbert_config, verbose)
74 def __init__(self, name, colbert_config=None, verbose: int = 3):
---> 75 super().__init__(name, colbert_config)
76 assert self.training is False
78 self.verbose = verbose

File ~/mamba/envs/llm/lib/python3.12/site-packages/colbert/modeling/colbert.py:21, in ColBERT.__init__(self, name, colbert_config)
20 def __init__(self, name='bert-base-uncased', colbert_config=None):
---> 21 super().__init__(name, colbert_config)
22 self.use_gpu = colbert_config.total_visible_gpus > 0
24 ColBERT.try_load_torch_extensions(self.use_gpu)

File ~/mamba/envs/llm/lib/python3.12/site-packages/colbert/modeling/base_colbert.py:36, in BaseColBERT.__init__(self, name_or_path, colbert_config)
31 HF_ColBERT = class_factory(self.name)
33 # assert self.name is not None
34 # HF_ColBERT = class_factory(self.name)
---> 36 self.model = HF_ColBERT.from_pretrained(name_or_path, colbert_config=self.colbert_config)
37 self.model.to(DEVICE)
38 self.raw_tokenizer = AutoTokenizer.from_pretrained(name_or_path)

File ~/mamba/envs/llm/lib/python3.12/site-packages/colbert/modeling/hf_colbert.py:139, in class_factory..HF_ColBERT.from_pretrained(cls, name_or_path, colbert_config)
135 obj.base = base
137 return obj
--> 139 obj = super().from_pretrained(name_or_path, colbert_config=colbert_config)
140 obj.base = name_or_path
142 return obj

File ~/mamba/envs/llm/lib/python3.12/site-packages/transformers/modeling_utils.py:4297, in PreTrainedModel.from_pretrained(cls, pretrained_model_name_or_path, config, cache_dir, ignore_mismatched_sizes, force_download, local_files_only, token, revision, use_safetensors, weights_only, fusion_config, disable_mmap, *model_args, **kwargs)
4279 load_config = LoadStateDictConfig(
4280 pretrained_model_name_or_path=pretrained_model_name_or_path,
4281 ignore_mismatched_sizes=ignore_mismatched_sizes,
(...) 4294 disable_mmap=disable_mmap,
4295 )
4296 loading_info, disk_offload_index = cls._load_pretrained_model(model, state_dict, checkpoint_files, load_config)
-> 4297 loading_info = cls._finalize_model_loading(model, load_config, loading_info)
4298 model.eval() # Set model in evaluation mode to deactivate Dropout modules by default
4299 model.set_use_kernels(use_kernels, kernel_config)

File ~/mamba/envs/llm/lib/python3.12/site-packages/transformers/modeling_utils.py:4467, in PreTrainedModel._finalize_model_loading(model, load_config, loading_info)
4464 model.tie_weights(missing_keys=loading_info.missing_keys, recompute_mapping=False)
4466 # Adjust missing and unexpected keys
-> 4467 model._adjust_missing_and_unexpected_keys(loading_info)
4468 finally:
4469 log_state_dict_report(
4470 model=model,
4471 pretrained_model_name_or_path=load_config.pretrained_model_name_or_path,
(...) 4474 logger=logger,
4475 )

File ~/mamba/envs/llm/lib/python3.12/site-packages/transformers/modeling_utils.py:4805, in PreTrainedModel._adjust_missing_and_unexpected_keys(self, loading_info)
4802 additional_unexpected_patterns.add(r"(^|\.)position_ids$")
4804 missing_patterns = self._keys_to_ignore_on_load_missing or set()
-> 4805 unexpected_patterns = (self._keys_to_ignore_on_load_unexpected or set()) | additional_unexpected_patterns
4806 ignore_missing_regex, ignore_unexpected_regex = None, None
4807 if len(missing_patterns) > 0:

TypeError: unsupported operand type(s) for |: 'list' and 'set'
```

**Fix:**
```python
File colbert/modeling/hf_colbert.py:97

_keys_to_ignore_on_load_unexpected = [r"cls"]
--> _keys_to_ignore_on_load_unexpected = {r"cls"}

Reproduce the reported import and model-loading failures in the Python 3.12 environment, then inspect ragatouille/RAGPretrainedModel.py, ragatouille/integrations/_langchain.py, and colbert/modeling/hf_colbert.py. Verify that the LangChain imports and newer Transformers compatibility paths work together and that loading colbert-ir/colbertv2.0 completes without the shown errors.

Written by the indexing model from the issue text.