fix: preserve c-TF-IDF idf diagonal in delete_topics (#2530)

bertopic/_bertopic.py

@@ -2219,11 +2219,9 @@ def delete_topics(
             if hasattr(self, "custom_labels_") and self.custom_labels_ is not None:
                 self.custom_labels_[-1] = ""
 
-            # Initialize ctfidf model diagonal for -1 topic (ones) if it exists
-            if hasattr(self, "ctfidf_model") and self.ctfidf_model is not None:
-                n_features = self.ctfidf_model._idf_diag.shape[1]
-                outlier_diag = sp.csr_matrix(([1.0], ([0], [0])), shape=(1, n_features))
-                self.ctfidf_model._idf_diag = sp.vstack([outlier_diag, self.ctfidf_model._idf_diag])
+            # NOTE: `ctfidf_model._idf_diag` is a (n_features, n_features) diagonal matrix
+            # over the vocabulary and is independent of the number of topics, so it must
+            # not be modified when adding the -1 topic.
 
             # Initialize topic aspects for -1 topic (empty dict for each aspect) if they exist
             if hasattr(self, "topic_aspects_") and self.topic_aspects_ is not None:
@@ -2305,10 +2303,10 @@ def delete_topics(
             mask = np.array([topic not in topics_to_delete for topic in range(matrix.shape[0])])
             setattr(self, attr, matrix[mask])
 
-        # Update ctfidf model to remove deleted topics if it exists
-        if hasattr(self, "ctfidf_model") and self.ctfidf_model is not None:
-            mask = np.array([topic not in topics_to_delete for topic in range(self.ctfidf_model._idf_diag.shape[0])])
-            self.ctfidf_model._idf_diag = self.ctfidf_model._idf_diag[mask]
+        # NOTE: `ctfidf_model._idf_diag` is a (n_features, n_features) diagonal matrix over
+        # the vocabulary. Deleting topics does not change the vocabulary, so it is left
+        # unchanged here; masking it along the topic axis would corrupt its shape and break
+        # any later `ctfidf_model.transform` call (e.g. `topics_over_time`).
 
     def reduce_topics(
         self,

tests/test_reduction/test_delete.py

@@ -57,3 +57,34 @@ def test_delete(model, request):
         assert mapped_labels == topic_model.topics_[950:]
     else:
         assert mapped_labels == topic_model.topics_
+
+
+@pytest.mark.parametrize(
+    "model",
+    [
+        ("kmeans_pca_topic_model"),
+        ("base_topic_model"),
+    ],
+)
+def test_delete_topics_preserves_ctfidf_idf_diag(model, documents, request):
+    """`delete_topics` must not corrupt the c-TF-IDF idf diagonal (see #2530).
+
+    `ctfidf_model._idf_diag` is a (n_features, n_features) matrix over the vocabulary
+    and is independent of the number of topics. Deleting topics used to mutate it along
+    the topic axis, leaving it non-square and breaking any later `ctfidf_model.transform`
+    call, such as the one in `topics_over_time`.
+    """
+    topic_model = copy.deepcopy(request.getfixturevalue(model))
+    idf_shape = topic_model.ctfidf_model._idf_diag.shape
+
+    topic_model.delete_topics([1, 2])
+
+    # The idf diagonal must remain the same square vocabulary-sized matrix
+    assert idf_shape[0] == idf_shape[1]
+    assert topic_model.ctfidf_model._idf_diag.shape == idf_shape
+
+    # `topics_over_time` relies on `ctfidf_model.transform` and must not raise
+    timestamps = [i % 10 for i in range(len(documents))]
+    topics_over_time = topic_model.topics_over_time(documents, timestamps)
+    assert topics_over_time.Frequency.sum() == len(documents)
+    assert set(topics_over_time.Topic.unique()) == set(topic_model.topics_)