string.go

package main

import (
	"strings"

	"github.com/hbollon/go-edlib"
)

const PERFECT_FACTOR = 0.85

func WhitespaceLength(line []rune) int {
	i := 0
	for i < len(line) {
		charAt := line[i]
		if charAt == ' ' || charAt == '\t' || charAt == '\n' || charAt == '\r' {
			i++
			continue
		}
		break
	}
	return i
}

func ContentSimilarity(targetLine string, candidateLine string) float32 {
	targetRunes := []rune(targetLine)
	candidateRunes := []rune(candidateLine)
	targetWhitespace := WhitespaceLength(targetRunes)
	candidateWhitespace := WhitespaceLength(candidateRunes)

	similarity, err := edlib.StringsSimilarity(targetLine[targetWhitespace:], candidateLine[candidateWhitespace:], edlib.Levenshtein)

	if err != nil {
		return 0.0
	} else {
		return similarity
	}
}

func LineContentSimilarity(targetLine string, candidateLine string) float32 {
	overallContentSimilarity := ContentSimilarity(targetLine, candidateLine)
	if overallContentSimilarity >= MINIMUM_RAW_SIMILARITY {
		return overallContentSimilarity
	}
	// Even if we didn't score high on raw similarity, maybe we can break apart the thing into chunks and compare those?
	splitTargetLine := strings.Split(targetLine, "=")
	splitCandidateLine := strings.Split(candidateLine, "=")
	if len(splitTargetLine) <= 1 || len(splitCandidateLine) <= 1 {
		return 0.0
	}
	firstTermContentSimilarity := ContentSimilarity(splitTargetLine[0], splitCandidateLine[0])
	if firstTermContentSimilarity >= MINIMUM_SIMILARITY_FIRST_TERM_ONLY {
		return (1.0*PERFECT_FACTOR + firstTermContentSimilarity) / (PERFECT_FACTOR + 1)
	}
	var bestSimilarityFound float32 = 0.0
	for i := 1; i < minint(len(splitCandidateLine), len(splitTargetLine)); i++ {
		termSimilarity := ContentSimilarity(splitTargetLine[i], splitCandidateLine[i])
		if termSimilarity > bestSimilarityFound {
			bestSimilarityFound = termSimilarity
		}
	}
	if bestSimilarityFound >= MINIMUM_SIMILARITY_SUBSEQUENT_TERMS {
		result := (1.0*PERFECT_FACTOR + overallContentSimilarity) / (PERFECT_FACTOR + 1)
		return result
	}
	return 0.0
}