Merge pull request #75 from upfluence/thomasgallice/lib-173-influencers-email-wrongly-identified-through-igs-bio-peachy

stringutil/decoder: Until remove not ascii use space to separe work

Author: tgallice

stringutil/decoder.go

@@ -1,6 +1,7 @@
 package stringutil
 
 import (
+	"regexp"
 	"strings"
 	"unicode"
 	"unicode/utf8"
@@ -75,6 +76,8 @@ func IsASCII(s string) bool {
 	return true
 }
 
+var spaceRegex = regexp.MustCompile(`\s+`)
+
 func DecodeToASCII(s string, opts ...ASCIIDecodeOption) string {
 	if IsASCII(s) {
 		return s
@@ -90,7 +93,13 @@ func DecodeToASCII(s string, opts ...ASCIIDecodeOption) string {
 		t = transform.Chain(
 			os.decomposer,
 			runes.Remove(runes.In(unicode.Mn)),
-			runes.Remove(setFunc(isAboveASCII)),
+			runes.Map(func(r rune) rune {
+				if isAboveASCII(r) {
+					return rune(' ')
+				}
+
+				return r
+			}),
 			os.composer,
 		)
 
@@ -102,5 +111,5 @@ func DecodeToASCII(s string, opts ...ASCIIDecodeOption) string {
 		return ""
 	}
 
-	return result
+	return strings.Trim(spaceRegex.ReplaceAllString(result, " "), " ")
 }

stringutil/decoder_test.go

@@ -41,11 +41,20 @@ func TestDecodeToASCII(t *testing.T) {
 		},
 		{
 			in:  "Collaboration: 𝕸𝖎𝖆𝖒𝖎 🌞 x KiwiKurve",
-			out: "Collaboration:   x KiwiKurve",
+			out: "Collaboration: x KiwiKurve",
 		},
 		{
 			in:   "Collaboration: 𝕸𝖎𝖆𝖒𝖎 🌞 x KiwiKurve",
-			out:  "Collaboration: Miami  x KiwiKurve",
+			out:  "Collaboration: Miami x KiwiKurve",
+			opts: nfkd,
+		},
+		{
+			in:  "back soon ✌🏽📍ashleyrchand@gmail.com",
+			out: "back soon ashleyrchand@gmail.com",
+		},
+		{
+			in:   "Golden Girl 🌴\n🌿Discounts/links⬇️\nPR/Collab📧spfpleaseka𝐓ie@gmail.com",
+			out:  "Golden Girl Discounts/links PR/Collab spfpleasekaTie@gmail.com",
 			opts: nfkd,
 		},
 		{