Fix weighted/macro AUC aggregation in rank metrics

calculator/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/calculation/RankMetrics.kt

@@ -147,13 +147,24 @@ fun calculateROC(
     val rocPoints: MutableList<DoubleArray> = ArrayList()
     rocPoints.add(doubleArrayOf(0.0, 0.0)) // Start at (0, 0)
 
-    for (pair in relevanceIsTPList) {
-        if (pair.second) {
-            truePositives++
-        } else {
-            falsePositives++
+    var index = 0
+    while (index < relevanceIsTPList.size) {
+        val currentRelevance = relevanceIsTPList[index].first
+        var tpInGroup = 0
+        var fpInGroup = 0
+
+        while (index < relevanceIsTPList.size && relevanceIsTPList[index].first == currentRelevance) {
+            if (relevanceIsTPList[index].second) {
+                tpInGroup++
+            } else {
+                fpInGroup++
+            }
+            index++
         }
 
+        truePositives += tpInGroup
+        falsePositives += fpInGroup
+
         val tpr = truePositives.toDouble() / totalPositives
         val fpr = falsePositives.toDouble() / totalNegatives
 

calculator/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/internal/RankMetricsCalculatorImpl.kt

@@ -34,9 +34,14 @@ internal class RankMetricsCalculatorImpl : RankMetricsCalculator {
         return listOf(macroAverage, weightedAverage)
     }
 
-    private fun calculateMacroAverage(singleRankMetricsResults: List<SingleRankMetricsResult>): AggregatedRankMetricsResult {
-        return calculateWeightedAverage(singleRankMetricsResults, singleRankMetricsResults.map { 1 }, AggregationType.MACRO_AVERAGE)
-    }
+    private fun calculateMacroAverage(singleRankMetricsResults: List<SingleRankMetricsResult>): AggregatedRankMetricsResult =
+        calculateWeightedAverage(
+            singleRankMetricsResults,
+            singleRankMetricsResults.map {
+                1
+            },
+            AggregationType.MACRO_AVERAGE
+        )
 
     private fun calculateWeightedAverage(
         singleRankMetricsResults: List<SingleRankMetricsResult>,
@@ -60,6 +65,7 @@ internal class RankMetricsCalculatorImpl : RankMetricsCalculator {
 
         map /= sumOfWeights
         lag /= sumOfWeights
+        auc /= sumOfWeights
 
         return if (singleRankMetricsResults.all { it.auc == null }) {
             AggregatedRankMetricsResult(type, map, lag, null, singleRankMetricsResults, weights)

calculator/src/test/kotlin/edu/kit/kastel/mcse/ardoco/metrics/calculation/RankMetricsTest.kt

@@ -0,0 +1,108 @@
+package edu.kit.kastel.mcse.ardoco.metrics.calculation
+
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Test
+
+class RankMetricsTest {
+    @Test
+    fun calculateAucShouldReturnOneForPerfectRanking() {
+        val auc =
+            calculateAUC(
+                rankedResults = listOf(listOf("tp1", "tp2", "fp1", "fp2")),
+                rankedRelevances = listOf(listOf(0.9, 0.8, 0.7, 0.6)),
+                groundTruth = setOf("tp1", "tp2"),
+                biggerIsMoreSimilar = true
+            )
+
+        assertEquals(1.0, auc, 1e-9)
+    }
+
+    @Test
+    fun calculateAucShouldReturnZeroForWorstRanking() {
+        val auc =
+            calculateAUC(
+                rankedResults = listOf(listOf("tp1", "tp2", "fp1", "fp2")),
+                rankedRelevances = listOf(listOf(0.6, 0.7, 0.8, 0.9)),
+                groundTruth = setOf("tp1", "tp2"),
+                biggerIsMoreSimilar = true
+            )
+
+        assertEquals(0.0, auc, 1e-9)
+    }
+
+    @Test
+    fun calculateAucShouldTreatTiedScoresAsHalfCorrect() {
+        val auc =
+            calculateAUC(
+                rankedResults = listOf(listOf("tp", "fp")),
+                rankedRelevances = listOf(listOf(0.5, 0.5)),
+                groundTruth = setOf("tp"),
+                biggerIsMoreSimilar = true
+            )
+
+        assertEquals(0.5, auc, 1e-9)
+    }
+
+    @Test
+    fun calculateAucShouldSupportAscendingScores() {
+        val auc =
+            calculateAUC(
+                rankedResults = listOf(listOf("tp1", "tp2", "fp1", "fp2")),
+                rankedRelevances = listOf(listOf(0.1, 0.2, 0.3, 0.4)),
+                groundTruth = setOf("tp1", "tp2"),
+                biggerIsMoreSimilar = false
+            )
+
+        assertEquals(1.0, auc, 1e-9)
+    }
+
+    // Tests based on internet examples (Google Developers ML Crash Course and related references):
+    // "If every positive example is ranked higher than every negative example, AUC = 1.0"
+    @Test
+    fun calculateAucShouldReturnOneForThreePositivesAboveThreeNegatives() {
+        // Positives: 0.9, 0.8, 0.7 — Negatives: 0.3, 0.2, 0.1 (exact internet example)
+        val auc =
+            calculateAUC(
+                rankedResults = listOf(listOf("tp1", "tp2", "tp3", "fp1", "fp2", "fp3")),
+                rankedRelevances = listOf(listOf(0.9, 0.8, 0.7, 0.3, 0.2, 0.1)),
+                groundTruth = setOf("tp1", "tp2", "tp3"),
+                biggerIsMoreSimilar = true
+            )
+
+        assertEquals(1.0, auc, 1e-9)
+    }
+
+    // "If every positive example is ranked lower than every negative example, AUC = 0.0"
+    @Test
+    fun calculateAucShouldReturnZeroForThreePositivesBelowThreeNegatives() {
+        // Positives: 0.1, 0.2, 0.3 — Negatives: 0.7, 0.8, 0.9 (exact internet example)
+        val auc =
+            calculateAUC(
+                rankedResults = listOf(listOf("tp1", "tp2", "tp3", "fp1", "fp2", "fp3")),
+                rankedRelevances = listOf(listOf(0.1, 0.2, 0.3, 0.7, 0.8, 0.9)),
+                groundTruth = setOf("tp1", "tp2", "tp3"),
+                biggerIsMoreSimilar = true
+            )
+
+        assertEquals(0.0, auc, 1e-9)
+    }
+
+    // "Half of the positive-negative pairs are correctly ranked, half are not" → AUC = 0.5
+    // Non-tie partial example:
+    // Scores: tp1=0.8, tp2=0.2, fp1=0.9, fp2=0.1
+    // Pairs: (tp1=0.8 vs fp1=0.9) → fp wins (0), (tp1=0.8 vs fp2=0.1) → tp wins (1),
+    //        (tp2=0.2 vs fp1=0.9) → fp wins (0), (tp2=0.2 vs fp2=0.1) → tp wins (1)
+    // AUC = (0+1+0+1)/4 = 0.5
+    @Test
+    fun calculateAucShouldReturnHalfForMixedRankingWithHalfConcordantPairs() {
+        val auc =
+            calculateAUC(
+                rankedResults = listOf(listOf("tp1", "tp2", "fp1", "fp2")),
+                rankedRelevances = listOf(listOf(0.8, 0.2, 0.9, 0.1)),
+                groundTruth = setOf("tp1", "tp2"),
+                biggerIsMoreSimilar = true
+            )
+
+        assertEquals(0.5, auc, 1e-9)
+    }
+}

calculator/src/test/kotlin/edu/kit/kastel/mcse/ardoco/metrics/internal/RankMetricsCalculatorImplTest.kt

@@ -0,0 +1,27 @@
+package edu.kit.kastel.mcse.ardoco.metrics.internal
+
+import edu.kit.kastel.mcse.ardoco.metrics.RankMetricsCalculator
+import edu.kit.kastel.mcse.ardoco.metrics.result.AggregationType
+import edu.kit.kastel.mcse.ardoco.metrics.result.SingleRankMetricsResult
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Assertions.assertNotNull
+import org.junit.jupiter.api.Test
+
+class RankMetricsCalculatorImplTest {
+    @Test
+    fun calculateAveragesShouldAverageAucWithWeights() {
+        val results =
+            listOf(
+                SingleRankMetricsResult(map = 0.2, lag = 0.1, auc = 0.2, groundTruthSize = 1),
+                SingleRankMetricsResult(map = 0.8, lag = 0.3, auc = 0.6, groundTruthSize = 3)
+            )
+
+        val weightedAverage =
+            RankMetricsCalculator.Instance
+                .calculateAverages(results, listOf(1, 3))
+                .first { it.type == AggregationType.WEIGHTED_AVERAGE }
+
+        assertNotNull(weightedAverage.auc)
+        assertEquals(0.5, weightedAverage.auc!!, 1e-9)
+    }
+}