diff --git a/stringutil/decoder.go b/stringutil/decoder.go index 3c56a39..eb875c4 100644 --- a/stringutil/decoder.go +++ b/stringutil/decoder.go @@ -1,6 +1,7 @@ package stringutil import ( + "regexp" "strings" "unicode" "unicode/utf8" @@ -75,6 +76,8 @@ func IsASCII(s string) bool { return true } +var spaceRegex = regexp.MustCompile(`\s+`) + func DecodeToASCII(s string, opts ...ASCIIDecodeOption) string { if IsASCII(s) { return s @@ -90,7 +93,13 @@ func DecodeToASCII(s string, opts ...ASCIIDecodeOption) string { t = transform.Chain( os.decomposer, runes.Remove(runes.In(unicode.Mn)), - runes.Remove(setFunc(isAboveASCII)), + runes.Map(func(r rune) rune { + if isAboveASCII(r) { + return rune(' ') + } + + return r + }), os.composer, ) @@ -102,5 +111,5 @@ func DecodeToASCII(s string, opts ...ASCIIDecodeOption) string { return "" } - return result + return strings.Trim(spaceRegex.ReplaceAllString(result, " "), " ") } diff --git a/stringutil/decoder_test.go b/stringutil/decoder_test.go index d27623d..93c1a65 100644 --- a/stringutil/decoder_test.go +++ b/stringutil/decoder_test.go @@ -41,11 +41,20 @@ func TestDecodeToASCII(t *testing.T) { }, { in: "Collaboration: 𝕸𝖎𝖆𝖒𝖎 🌞 x KiwiKurve", - out: "Collaboration: x KiwiKurve", + out: "Collaboration: x KiwiKurve", }, { in: "Collaboration: 𝕸𝖎𝖆𝖒𝖎 🌞 x KiwiKurve", - out: "Collaboration: Miami x KiwiKurve", + out: "Collaboration: Miami x KiwiKurve", + opts: nfkd, + }, + { + in: "back soon ✌🏽📍ashleyrchand@gmail.com", + out: "back soon ashleyrchand@gmail.com", + }, + { + in: "Golden Girl 🌴\n🌿Discounts/links⬇️\nPR/Collab📧spfpleaseka𝐓ie@gmail.com", + out: "Golden Girl Discounts/links PR/Collab spfpleasekaTie@gmail.com", opts: nfkd, }, {