fix(v1/algo): invalid q-gram similarity calculation (#22)

Author: stefan-berkner-tilotech

qgram.go

@@ -52,5 +52,12 @@ func QgramSimilarity(str1, str2 string, splitLength int) float32 {
 	splittedStr1 := Shingle(str1, splitLength)
 	splittedStr2 := Shingle(str2, splitLength)
 	res := float32(QgramDistanceCustomNgram(splittedStr1, splittedStr2))
-	return 1 - (res / float32(len(splittedStr1)+len(splittedStr2)))
+	totalShingles := 0
+	for _, i := range splittedStr1 {
+		totalShingles += i
+	}
+	for _, i := range splittedStr2 {
+		totalShingles += i
+	}
+	return 1 - (res / float32(totalShingles))
 }

string-analysis_test.go

@@ -175,16 +175,17 @@ func TestStringsSimilarity(t *testing.T) {
 		{"Qgram : Second arg empty", args{"abcde", "", Qgram}, 0.0, false},
 		{"Qgram : Same args", args{"abcde", "abcde", Qgram}, 1.0, false},
 		{"Qgram : No characters match", args{"abcd", "effgghh", Qgram}, 0.0, false},
+		{"Qgram : Repeated shingles", args{"aaaaaa", "aaa", Qgram}, 0.57142854, false},
 		{"Qgram : CRATE/TRACE", args{"CRATE", "TRACE", Qgram}, 0.25, false},
 		{"Qgram : MARTHA/MARHTA", args{"MARTHA", "MARHTA", Qgram}, 0.39999998, false},
 		{"Qgram : DIXON/DICKSONX", args{"DIXON", "DICKSONX", Qgram}, 0.36363637, false},
 		{"Qgram Sentence 1", args{"Radiohead", "Radiohead", Qgram}, 1.0, false},
 		{"Qgram Sentence 2", args{"ABCD", "ABCE", Qgram}, 0.6666666, false},
-		{"Qgram Sentence 3", args{"Radiohead", "Carly Rae Jepsen", Qgram}, 0.04545456, false},
-		{"Qgram Sentence 4", args{"I love horror movies", "Lights out is a horror movie", Qgram}, 0.47619045, false},
-		{"Qgram Sentence 5", args{"love horror movies", "Lights out horror movie", Qgram}, 0.5833334, false},
+		{"Qgram Sentence 3", args{"Radiohead", "Carly Rae Jepsen", Qgram}, 0.0869565, false},
+		{"Qgram Sentence 4", args{"I love horror movies", "Lights out is a horror movie", Qgram}, 0.5217391, false},
+		{"Qgram Sentence 5", args{"love horror movies", "Lights out horror movie", Qgram}, 0.6153846, false},
 		{"Qgram Sentence 6", args{"私の名前はジョンです", "私の名前はジョン・ドゥです", Qgram}, 0.7619048, false},
-		{"Qgram Sentence 7", args{"🙂😄🙂😄 😄🙂😄", "🙂😄🙂😄 😄🙂😄 🙂😄🙂", Qgram}, 0.5555556, false},
+		{"Qgram Sentence 7", args{"🙂😄🙂😄 😄🙂😄", "🙂😄🙂😄 😄🙂😄 🙂😄🙂", Qgram}, 0.7777778, false},
 
 		// TODO: Must refactor compare method to handle NaN values
 		// {"Qgram Sentence 8", args{"", "", Qgram}, float32(math.NaN()), false},