diff --git a/calculator/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/calculation/RankMetrics.kt b/calculator/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/calculation/RankMetrics.kt index e94e647..32f308d 100644 --- a/calculator/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/calculation/RankMetrics.kt +++ b/calculator/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/calculation/RankMetrics.kt @@ -147,13 +147,24 @@ fun calculateROC( val rocPoints: MutableList = ArrayList() rocPoints.add(doubleArrayOf(0.0, 0.0)) // Start at (0, 0) - for (pair in relevanceIsTPList) { - if (pair.second) { - truePositives++ - } else { - falsePositives++ + var index = 0 + while (index < relevanceIsTPList.size) { + val currentRelevance = relevanceIsTPList[index].first + var tpInGroup = 0 + var fpInGroup = 0 + + while (index < relevanceIsTPList.size && relevanceIsTPList[index].first == currentRelevance) { + if (relevanceIsTPList[index].second) { + tpInGroup++ + } else { + fpInGroup++ + } + index++ } + truePositives += tpInGroup + falsePositives += fpInGroup + val tpr = truePositives.toDouble() / totalPositives val fpr = falsePositives.toDouble() / totalNegatives diff --git a/calculator/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/internal/RankMetricsCalculatorImpl.kt b/calculator/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/internal/RankMetricsCalculatorImpl.kt index 76edc64..2cfdb4a 100644 --- a/calculator/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/internal/RankMetricsCalculatorImpl.kt +++ b/calculator/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/internal/RankMetricsCalculatorImpl.kt @@ -34,9 +34,14 @@ internal class RankMetricsCalculatorImpl : RankMetricsCalculator { return listOf(macroAverage, weightedAverage) } - private fun calculateMacroAverage(singleRankMetricsResults: List): AggregatedRankMetricsResult { - return calculateWeightedAverage(singleRankMetricsResults, singleRankMetricsResults.map { 1 }, AggregationType.MACRO_AVERAGE) - } + private fun calculateMacroAverage(singleRankMetricsResults: List): AggregatedRankMetricsResult = + calculateWeightedAverage( + singleRankMetricsResults, + singleRankMetricsResults.map { + 1 + }, + AggregationType.MACRO_AVERAGE + ) private fun calculateWeightedAverage( singleRankMetricsResults: List, @@ -60,6 +65,7 @@ internal class RankMetricsCalculatorImpl : RankMetricsCalculator { map /= sumOfWeights lag /= sumOfWeights + auc /= sumOfWeights return if (singleRankMetricsResults.all { it.auc == null }) { AggregatedRankMetricsResult(type, map, lag, null, singleRankMetricsResults, weights) diff --git a/calculator/src/test/kotlin/edu/kit/kastel/mcse/ardoco/metrics/calculation/RankMetricsTest.kt b/calculator/src/test/kotlin/edu/kit/kastel/mcse/ardoco/metrics/calculation/RankMetricsTest.kt new file mode 100644 index 0000000..71d4b01 --- /dev/null +++ b/calculator/src/test/kotlin/edu/kit/kastel/mcse/ardoco/metrics/calculation/RankMetricsTest.kt @@ -0,0 +1,108 @@ +package edu.kit.kastel.mcse.ardoco.metrics.calculation + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class RankMetricsTest { + @Test + fun calculateAucShouldReturnOneForPerfectRanking() { + val auc = + calculateAUC( + rankedResults = listOf(listOf("tp1", "tp2", "fp1", "fp2")), + rankedRelevances = listOf(listOf(0.9, 0.8, 0.7, 0.6)), + groundTruth = setOf("tp1", "tp2"), + biggerIsMoreSimilar = true + ) + + assertEquals(1.0, auc, 1e-9) + } + + @Test + fun calculateAucShouldReturnZeroForWorstRanking() { + val auc = + calculateAUC( + rankedResults = listOf(listOf("tp1", "tp2", "fp1", "fp2")), + rankedRelevances = listOf(listOf(0.6, 0.7, 0.8, 0.9)), + groundTruth = setOf("tp1", "tp2"), + biggerIsMoreSimilar = true + ) + + assertEquals(0.0, auc, 1e-9) + } + + @Test + fun calculateAucShouldTreatTiedScoresAsHalfCorrect() { + val auc = + calculateAUC( + rankedResults = listOf(listOf("tp", "fp")), + rankedRelevances = listOf(listOf(0.5, 0.5)), + groundTruth = setOf("tp"), + biggerIsMoreSimilar = true + ) + + assertEquals(0.5, auc, 1e-9) + } + + @Test + fun calculateAucShouldSupportAscendingScores() { + val auc = + calculateAUC( + rankedResults = listOf(listOf("tp1", "tp2", "fp1", "fp2")), + rankedRelevances = listOf(listOf(0.1, 0.2, 0.3, 0.4)), + groundTruth = setOf("tp1", "tp2"), + biggerIsMoreSimilar = false + ) + + assertEquals(1.0, auc, 1e-9) + } + + // Tests based on internet examples (Google Developers ML Crash Course and related references): + // "If every positive example is ranked higher than every negative example, AUC = 1.0" + @Test + fun calculateAucShouldReturnOneForThreePositivesAboveThreeNegatives() { + // Positives: 0.9, 0.8, 0.7 — Negatives: 0.3, 0.2, 0.1 (exact internet example) + val auc = + calculateAUC( + rankedResults = listOf(listOf("tp1", "tp2", "tp3", "fp1", "fp2", "fp3")), + rankedRelevances = listOf(listOf(0.9, 0.8, 0.7, 0.3, 0.2, 0.1)), + groundTruth = setOf("tp1", "tp2", "tp3"), + biggerIsMoreSimilar = true + ) + + assertEquals(1.0, auc, 1e-9) + } + + // "If every positive example is ranked lower than every negative example, AUC = 0.0" + @Test + fun calculateAucShouldReturnZeroForThreePositivesBelowThreeNegatives() { + // Positives: 0.1, 0.2, 0.3 — Negatives: 0.7, 0.8, 0.9 (exact internet example) + val auc = + calculateAUC( + rankedResults = listOf(listOf("tp1", "tp2", "tp3", "fp1", "fp2", "fp3")), + rankedRelevances = listOf(listOf(0.1, 0.2, 0.3, 0.7, 0.8, 0.9)), + groundTruth = setOf("tp1", "tp2", "tp3"), + biggerIsMoreSimilar = true + ) + + assertEquals(0.0, auc, 1e-9) + } + + // "Half of the positive-negative pairs are correctly ranked, half are not" → AUC = 0.5 + // Non-tie partial example: + // Scores: tp1=0.8, tp2=0.2, fp1=0.9, fp2=0.1 + // Pairs: (tp1=0.8 vs fp1=0.9) → fp wins (0), (tp1=0.8 vs fp2=0.1) → tp wins (1), + // (tp2=0.2 vs fp1=0.9) → fp wins (0), (tp2=0.2 vs fp2=0.1) → tp wins (1) + // AUC = (0+1+0+1)/4 = 0.5 + @Test + fun calculateAucShouldReturnHalfForMixedRankingWithHalfConcordantPairs() { + val auc = + calculateAUC( + rankedResults = listOf(listOf("tp1", "tp2", "fp1", "fp2")), + rankedRelevances = listOf(listOf(0.8, 0.2, 0.9, 0.1)), + groundTruth = setOf("tp1", "tp2"), + biggerIsMoreSimilar = true + ) + + assertEquals(0.5, auc, 1e-9) + } +} diff --git a/calculator/src/test/kotlin/edu/kit/kastel/mcse/ardoco/metrics/internal/RankMetricsCalculatorImplTest.kt b/calculator/src/test/kotlin/edu/kit/kastel/mcse/ardoco/metrics/internal/RankMetricsCalculatorImplTest.kt new file mode 100644 index 0000000..c6a65ae --- /dev/null +++ b/calculator/src/test/kotlin/edu/kit/kastel/mcse/ardoco/metrics/internal/RankMetricsCalculatorImplTest.kt @@ -0,0 +1,27 @@ +package edu.kit.kastel.mcse.ardoco.metrics.internal + +import edu.kit.kastel.mcse.ardoco.metrics.RankMetricsCalculator +import edu.kit.kastel.mcse.ardoco.metrics.result.AggregationType +import edu.kit.kastel.mcse.ardoco.metrics.result.SingleRankMetricsResult +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Test + +class RankMetricsCalculatorImplTest { + @Test + fun calculateAveragesShouldAverageAucWithWeights() { + val results = + listOf( + SingleRankMetricsResult(map = 0.2, lag = 0.1, auc = 0.2, groundTruthSize = 1), + SingleRankMetricsResult(map = 0.8, lag = 0.3, auc = 0.6, groundTruthSize = 3) + ) + + val weightedAverage = + RankMetricsCalculator.Instance + .calculateAverages(results, listOf(1, 3)) + .first { it.type == AggregationType.WEIGHTED_AVERAGE } + + assertNotNull(weightedAverage.auc) + assertEquals(0.5, weightedAverage.auc!!, 1e-9) + } +}