c_tf_idf update in evolution_tuning (in reference to Discussion #2349)

Have you searched existing issues? 🔎

• I have searched and found no existing issues

Desribe the bug

When applying evolutionary tuning in topics over time, the c_tf_idf[current_overlap_idx] is not updated after the averaging operation. This results in the evolutionary tuning not operating as intended.

Reproduction

pip install bertopic

import re
import math
import joblib
import inspect
import collections
import numpy as np
import pandas as pd
import scipy.sparse as sp

from tqdm import tqdm
from pathlib import Path
from packaging import version
from tempfile import TemporaryDirectory
from collections import defaultdict, Counter
from scipy.sparse import csr_matrix
from scipy.cluster import hierarchy as sch

# Typing
import sys

if sys.version_info >= (3, 8):
    from typing import Literal
else:
    from typing_extensions import Literal
from typing import List, Tuple, Union, Mapping, Any, Callable, Iterable

# Models
try:
    from hdbscan import HDBSCAN

    HAS_HDBSCAN = True
except (ImportError, ModuleNotFoundError):
    HAS_HDBSCAN = False
    from sklearn.cluster import HDBSCAN as SK_HDBSCAN

from sklearn.preprocessing import normalize
from sklearn import __version__ as sklearn_version
from sklearn.cluster import AgglomerativeClustering
from sklearn.decomposition import PCA
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer

# Prepare data
trump = pd.read_csv('https://drive.google.com/uc?export=download&id=1xRKHaP-QwACMydlDnyFPEaFdtskJuBa6')
trump.text = trump.apply(lambda row: re.sub(r"http\S+", "", row.text).lower(), 1)
trump.text = trump.apply(lambda row: " ".join(filter(lambda x:x[0]!="@", row.text.split())), 1)
trump.text = trump.apply(lambda row: " ".join(re.sub("[^a-zA-Z]+", " ", row.text).split()), 1)
trump = trump.loc[(trump.isRetweet == "f") & (trump.text != ""), :]
timestamps = trump.date.to_list()
tweets = trump.text.to_list()

from bertopic import BERTopic

topic_model = BERTopic(verbose=True)
topics, probs = topic_model.fit_transform(tweets)

# The next section is from the topics_over_time() function, separated so we can check the c-tf-idf values before running the averaging operation

selected_topics = topics
docs = tweets
documents = pd.DataFrame({"Document": docs, "Topic": selected_topics, "Timestamps": timestamps})
c_tf_idf = topic_model.c_tf_idf_
global_c_tf_idf = normalize(c_tf_idf, axis = 1, norm = "l1", copy=False)
all_topics=sorted(list(documents.Topic.unique()))
all_topics_indices = {topic: index for index, topic in enumerate(all_topics)}
documents = documents.sort_values("Timestamps")
timestamps = documents.Timestamps.unique()
selection = documents.loc[documents.Timestamps == timestamps[0], :]
documents_per_topic = selection.groupby(["Topic"], as_index = False).agg({"Document": " ".join, "Timestamps": "count"}
            )
c_tf_idf, words = topic_model._c_tf_idf(documents_per_topic, fit=False)
c_tf_idf = normalize(c_tf_idf, axis=1, norm="l1", copy=False)

#set previous time slice data
previous_topics = sorted(list(documents_per_topic.Topic.values))
previous_c_tf_idf = c_tf_idf.copy()

#set current time slice data
selection = documents.loc[documents.Timestamps == timestamps[1], :]
documents_per_topic = selection.groupby(["Topic"], as_index=False).agg(
                {"Document": " ".join, "Timestamps": "count"}
            )
c_tf_idf, words = topic_model._c_tf_idf(documents_per_topic, fit=False)
c_tf_idf = normalize(c_tf_idf, axis=1, norm="l1", copy=False)
current_topics = sorted(list(documents_per_topic.Topic.values))

overlapping_topics = sorted(list(set(previous_topics).intersection(set(current_topics))))
current_overlap_idx = [current_topics.index(topic) for topic in overlapping_topics]
previous_overlap_idx = [
                    previous_topics.index(topic)  # noqa: F821
                    for topic in overlapping_topics
                ]

# Checking the values before running the operation
words_per_topic_pre = topic_model._extract_words_per_topic(words, selection, c_tf_idf, calculate_aspects = False)
words_per_topic_pre[7] #number here may change depending on the run

# Execute current averaging operation
c_tf_idf[current_overlap_idx] = (
                    (
                        c_tf_idf[current_overlap_idx] + previous_c_tf_idf[previous_overlap_idx]  # noqa: F821
                    )
                    / 2.0
                )

# Check values after run. Did they change?
words_per_topic = topic_model._extract_words_per_topic(words, selection, c_tf_idf, calculate_aspects = False)
words_per_topic[7]

BERTopic Version

0.17.0

Start with the topics_over_time() path and the _c_tf_idf call shown in the reproduction; run the supplied two-time-slice example and compare extracted words before and after averaging. Trace where the averaged c_tf_idf is consumed, then add a regression check that the overlapping topic output reflects the averaging.