The results of topic modeling on colab are unstable

Have you searched existing issues? 🔎

• I have searched and found no existing issues

Desribe the bug

After using GPU for topic modeling on colab, it was found that even if a fixed random seed was used, when the same data was used for topic modeling again the next day, different topics would be obtained (for example: the quantity decreased, from 20 to 2, or 20 to 15).
Will the results of bertopic modeling vary with different Gpus (could it be a problem of random seed generation at the bottom layer?) ? This issue was not detected when using the CPU before. It would be highly appreciated if anyone knew the reason or the solution.

Reproduction

import os
from openai import OpenAI
import openai
from bertopic.representation import OpenAI
from bertopic import BERTopic
from umap import UMAP
from bertopic import BERTopic
from umap import UMAP

from bertopic import BERTopic
from sentence_transformers import models,SentenceTransformer
from bertopic.representation import KeyBERTInspired

from sklearn.feature_extraction.text import CountVectorizer
from hdbscan import HDBSCAN
import pandas as pd

import logging
from typing import List, Mapping, Any, Union, Callable, Tuple
import os
import requests
from openai import OpenAI
from bertopic.representation import BaseRepresentation
from bertopic import BERTopic
from umap import UMAP
from sentence_transformers import SentenceTransformer
from hdbscan import HDBSCAN
from http import HTTPStatus
import pandas as pd
from scipy.sparse import csr_matrix
from tqdm import tqdm
import time

# 配置日志
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# 初始化OpenAI客户端
client = OpenAI(
    api_key='MY_KEY',
    base_url="https://madmodel.cs.tsinghua.edu.cn/v1"
)

# 自定义OpenAIRepresentation以适应DashScope的API路径
class DashScopeOpenAIRepresentation(BaseRepresentation):
    def __init__(
        self,
        client: OpenAI,
        model: str = "DeepSeek-R1-671B",
        prompt: str = None,
        system_prompt: str = None,
        generator_kwargs: Mapping[str, Any] = {},
        delay_in_seconds: float = None,
        exponential_backoff: bool = False,
        nr_docs: int = 20,
        diversity: float = None,
        doc_length: int = None,
        tokenizer: Union[str, Callable] = None,
        **kwargs,
    ):
        super().__init__()
        self.client = client
        self.model = model

        if prompt is None:
            self.prompt = DEFAULT_CHAT_PROMPT
        else:
            self.prompt = prompt

        if system_prompt is None:
            self.system_prompt = DEFAULT_SYSTEM_PROMPT
        else:
            self.system_prompt = system_prompt

        self.default_prompt_ = DEFAULT_CHAT_PROMPT
        self.default_system_prompt_ = DEFAULT_SYSTEM_PROMPT
        self.delay_in_seconds = delay_in_seconds
        self.exponential_backoff = exponential_backoff
        self.nr_docs = nr_docs
        self.diversity = diversity
        self.doc_length = doc_length
        self.tokenizer = tokenizer
        validate_truncate_document_parameters(self.tokenizer, self.doc_length)

        self.prompts_ = []

        self.generator_kwargs = generator_kwargs
        if self.generator_kwargs.get("model"):
            self.model = self.generator_kwargs.get("model")
            del self.generator_kwargs["model"]
        if self.generator_kwargs.get("prompt"):
            del self.generator_kwargs["prompt"]
        if not self.generator_kwargs.get("stop"):
            self.generator_kwargs["stop"] = ["\n"]

    def extract_topics(
        self,
        topic_model,
        documents: pd.DataFrame,
        c_tf_idf: csr_matrix,
        topics: Mapping[str, List[Tuple[str, float]]],
    ) -> Mapping[str, List[Tuple[str, float]]]:
        """Extract topics.

        Arguments:
            topic_model: A BERTopic model
            documents: All input documents
            c_tf_idf: The topic c-TF-IDF representation
            topics: The candidate topics as calculated with c-TF-IDF

        Returns:
            updated_topics: Updated topic representations
        """
        # Extract the top n representative documents per topic
        repr_docs_mappings, _, _, _ = topic_model._extract_representative_docs(
            c_tf_idf, documents, topics, 500, self.nr_docs, self.diversity
        )

        # Generate using OpenAI's Language Model
        updated_topics = {}
        manual_labels = []
        for topic, docs in tqdm(repr_docs_mappings.items(), disable=not topic_model.verbose):
            truncated_docs = [truncate_document(topic_model, self.doc_length, self.tokenizer, doc) for doc in docs]
            prompt = self._create_prompt(truncated_docs, topic, topics)
            self.prompts_.append(prompt)

            # Log the prompt being sent to the model
            logger.info(f"Prompt sent to model:\n{prompt}")

            # Delay
            if self.delay_in_seconds:
                time.sleep(self.delay_in_seconds)

            messages = [
                {"role": "system", "content": self.system_prompt},
                {"role": "user", "content": prompt},
            ]
            kwargs = {
                "model": self.model,
                "messages": messages,
                "stream": True,
                **self.generator_kwargs,
            }
            if self.exponential_backoff:
                response = chat_completions_with_backoff(self.client, **kwargs)
            else:
                response = self.client.chat.completions.create(**kwargs)

            from openai.types.chat.chat_completion_chunk import ChatCompletionChunk

            # Collect the streamed response
            content_parts = []
            for chunk in response:
                logger.info(f"Chunk received: {chunk}")

                # 检查chunk是否为ChatCompletionChunk对象
                if not isinstance(chunk, ChatCompletionChunk):
                    logger.error(f"Invalid chunk type. Expected ChatCompletionChunk, got {type(chunk)}")
                    continue

                try:
                    # 访问choices属性
                    choices = chunk.choices
                    if len(choices) == 0:
                        logger.warning("No choices in chunk. Skipping this chunk.")
                        continue

                    # 访问第一个Choice对象的delta和content
                    delta = choices[0].delta
                    if not hasattr(delta, 'content') or delta.content is None:
                        logger.warning("No content in delta. Skipping this chunk.")
                        continue

                    delta_content = delta.content
                    content_parts.append(delta_content)
                    print(delta_content, end='', flush=True)  # Print each part immediately

                except Exception as e:
                    logger.error(f"Error processing chunk: {e}")
                    continue

            # 组装最终内容
            content = ''.join(content_parts).strip()
            logger.info(f"Generated content: '{content}'")

            # Check whether content was actually generated
            if content:
                label = content
                if label.lower().startswith("topic: "):
                    label = label.replace("topic: ", "").strip()
                else:
                    label = "Unexpected format: " + label
            else:
                label = "No label returned"

            manual_labels.append(label)
            updated_topics[topic] = [(label, 1)]

        return updated_topics

    def _create_prompt(self, docs, topic, topics):
        keywords = list(zip(*topics[topic]))[0]

        # Use the Default Chat Prompt
        if self.prompt == DEFAULT_CHAT_PROMPT:
            prompt = self.prompt.replace("[KEYWORDS]", ", ".join(keywords))
            prompt = self._replace_documents(prompt, docs)

        # Use a custom prompt that leverages keywords, documents or both using
        # custom tags, namely [KEYWORDS] and [DOCUMENTS] respectively
        else:
            prompt = self.prompt
            if "[KEYWORDS]" in prompt:
                prompt = prompt.replace("[KEYWORDS]", ", ".join(keywords))
            if "[DOCUMENTS]" in prompt:
                prompt = self._replace_documents(prompt, docs)

        return prompt

    @staticmethod
    def _replace_documents(prompt, docs):
        to_replace = ""
        for doc in docs:
            to_replace += f"- {doc}\n"
        prompt = prompt.replace("[DOCUMENTS]", to_replace)
        return prompt


def chat_completions_with_backoff(client, **kwargs):
    return retry_with_exponential_backoff(
        client.chat.completions.create,
        errors=(requests.exceptions.RequestException,),
    )(**kwargs)


def retry_with_exponential_backoff(func, errors, max_retries=10, backoff_factor=0.5):
    retries = 0
    while retries < max_retries:
        try:
            return func(**kwargs)
        except errors as e:
            logger.warning(f"Encountered an error: {e}. Retrying...")
            time.sleep(backoff_factor * (2 ** retries))
            retries += 1
    raise Exception("Max retries exceeded.")


def validate_truncate_document_parameters(tokenizer, doc_length):
    if doc_length is not None and doc_length <= 0:
        raise ValueError("doc_length must be greater than 0.")
    if tokenizer is not None and not callable(tokenizer):
        raise ValueError("tokenizer must be a callable function.")


def truncate_document(topic_model, doc_length, tokenizer, document):
    """Truncate a document to a specified length."""
    if tokenizer is not None:
        tokens = tokenizer(document)
        truncated_tokens = tokens[:doc_length]
        truncated_doc = topic_model.embedding_model.tokenizer.convert_tokens_to_string(truncated_tokens)
    else:
        truncated_doc = document[:doc_length]
    return truncated_doc


DEFAULT_CHAT_PROMPT = """You will extract a short topic label from given documents.
Here are two examples of topics you created before:

# Example 1
Sample texts from this topic:
- Traditional diets in most cultures were primarily plant-based with a little meat on top, but with the rise of industrial style meat production and factory farming, meat has become a staple food.
- Meat, but especially beef, is the worst food in terms of emissions.
- Eating meat doesn't make you a bad person, not eating meat doesn't make you a good one.

The topic is described by the following keywords: 'meat, beef, eat, eating, emissions, steak, food, health, processed, chicken'.

topic: Environmental impacts of eating meat

# Example 2
Sample texts from this topic:
- I have ordered the product weeks ago but it still has not arrived!
- The website mentions that it only takes a couple of days to deliver but I still have not received mine.
- I got a message stating that I received the monitor but that is not true!
- It took a month longer to deliver than was advised...

topic: Shipping and delivery issues

# Your task
Sample texts from this topic:
[DOCUMENTS]

The topic is described by the following keywords: '[KEYWORDS]'.

Based on the information above, extract a short topic label (three words at most) in the following format:
topic: <topic_label>
"""

DEFAULT_SYSTEM_PROMPT = "You are an assistant that extracts high-level topics from texts."

main_representation = KeyBERTInspired()

aspect_model1 = DashScopeOpenAIRepresentation(
    client=OpenAI(
    # 若没有配置环境变量，请用百炼API Key将下行替换为：api_key="sk-xxx",
    api_key="",
    base_url=""),
    model="qwen-turbo-2025-04-28",
    delay_in_seconds=5
)
# Additional ways of representing a topic


aspect_model2 = DashScopeOpenAIRepresentation(
    client=OpenAI(
    # 若没有配置环境变量，请用百炼API Key将下行替换为：api_key="sk-xxx",
    api_key="",
    base_url=""),
    model="deepseek-chat",
    delay_in_seconds=5
)
aspect_model3 = DashScopeOpenAIRepresentation(
    client=OpenAI(
    # 若没有配置环境变量，请用百炼API Key将下行替换为：api_key="sk-xxx",
    api_key="",
    base_url=""),
    model="qwen-plus-2025-04-28",
    delay_in_seconds=5
)




# Function: Topics - Create
def topics_creation(self, stop_words = ['en'], rmv_custom_words = [], embeddings = False,type = 'abstract',model ='/root/all-MiniLM-L6-v2',yearlist = []):
    umap_model = UMAP(n_neighbors = 15, n_components = 5, min_dist = 0.0, metric = 'cosine', random_state = 1001)
    #vectorizer_model = CountVectorizer(stop_words="english", min_df=2)


    # Add all models together to be run in a single `fit`
    representation_model = {
    "Main": main_representation,
    'qwen-turbo': aspect_model1,
    "deepseek-chat":  aspect_model2,
    'qwen-plus': aspect_model3
    }

    hdbscan_model = HDBSCAN(min_cluster_size=100, metric='euclidean', cluster_selection_method='eom', prediction_data=True)
    if (embeddings ==  False):
        self.topic_model = BERTopic(umap_model = umap_model, calculate_probabilities = True)
    else:
        if model =='/root/all-MiniLM-L6-v2':
            sentence_model   = SentenceTransformer(model)
            self.topic_model = BERTopic(umap_model = umap_model, calculate_probabilities = True, embedding_model = sentence_model,top_n_words=30,n_gram_range =(1,2),nr_topics="auto")
        else:
            word_embedding_model = models.Transformer(model)

            # Apply mean pooling to get one fixed sized sentence vector
            pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension(),
                                        pooling_mode = "mean")
            normalize = models.Normalize()
            sentence_model = SentenceTransformer(modules=[word_embedding_model, pooling_model,normalize])
            # Create your representation model
            #representation_model = KeyBERTInspired()
            self.topic_model = BERTopic(umap_model = umap_model, calculate_probabilities = True, embedding_model = sentence_model,hdbscan_model = hdbscan_model,representation_model =  representation_model,n_gram_range =(1,2))

    self.topic_corpus       = self.clear_text(self.data[type], stop_words = stop_words, lowercase = True, rmv_accents = True, rmv_special_chars = True, rmv_numbers = True, rmv_custom_words = rmv_custom_words, verbose = False)
    print(len((self.topic_corpus)))
    for i in range(len(self.topic_corpus)):
        try:
            (self.topic_corpus).remove('')
        except:
            yearlist.append( self.data['year'][i])
            pass

    print(len((self.topic_corpus)))

    self.topics, self.probs = self.topic_model.fit_transform(self.topic_corpus)
    self.topic_info         = self.topic_model.get_topic_info()
    print(self.topic_info)
    return self

yearlist = []
for i in range(len(bib_list)):
  topics_creation(bib_list[i], stop_words = ['en'], embeddings = True,type = 'abstract', model = "allenai/scibert_scivocab_uncased",yearlist = yearlist)

BERTopic Version

v0.17.0

Start with the provided Colab reproduction and compare repeated seeded runs using CPU and GPU with the same data. Inspect the BERTopic, UMAP, HDBSCAN, and sentence-transformers configuration shown, then determine whether the differing topic counts are reproducible and identify the relevant source of nondeterminism. Done means documenting the cause and a verified reproducibility result or fix.