diff --git a/common/geodata/strmatcher/benchmark_matchers_test.go b/common/geodata/strmatcher/benchmark_matchers_test.go index 9e00c816c..2cc82677d 100644 --- a/common/geodata/strmatcher/benchmark_matchers_test.go +++ b/common/geodata/strmatcher/benchmark_matchers_test.go @@ -1,6 +1,7 @@ package strmatcher_test import ( + "regexp" "strconv" "testing" @@ -72,6 +73,64 @@ func BenchmarkSubstrMatcher(b *testing.B) { }) } +func BenchmarkRegexMatcher(b *testing.B) { + patterns := []string{ // taken from geosite + `(^|\.)91porn\.(best|com|cool|fun|group|party|plus|site|tw|work)$`, + `(^|\.)91porn[0-9]{3}\.me$`, + `(^|\.)apiproxy-device-prod-nlb-.+\.amazonaws\.com$`, + `(^|\.)dualstack\.apiproxy-.+\.amazonaws\.com$`, + `(^|\.)aqdk[0-9]{3}\.com$`, + `(^|\.)bilibili3(0[1-9]|1[0-2])\.xyz$`, + `(^|\.)byyum([3589]|2[235689]|3[34]|4[1-9]|5[1-79]|6[0134679])?\.com$`, + `(^|\.)fiftymvapi\..+$`, + `(^|\.)gossipfuli[0-9]{3,4}\.xyz$`, + `(^|\.)kpkuang\.(bond|fun|info|one|us)$`, + `(^|\.)rule34\.(asia|us|world|xxx|xyz)$`, + `(^|\.)[a-z][1-9][0-9][a-z]\.com$`, + `.+\.awsdns-[0-9][0-9]\.(co\.uk|com|net|org)$`, + `.+\.dkr\.ecr\.[^\.]+\.amazonaws\.com$`, + `^(.+\.)*zh\.okaapps\.com$`, + `^cdn\d-epicgames-\d+\.file\.myqcloud\.com$`, + `^chatgpt-async-webps-prod-\S+-\d+\.webpubsub\.azure\.com$`, + `^r+[0-9]+(---|\.)sn-(2x3|ni5|j5o)\w{5}\.googlevideo\.com$`, + `^speed\.(coe|open)\.ad\.[a-z]{2,6}\.prod\.hosts\.ooklaserver\.net$`, + `javdb\d+\.com$`, + } + domains := []string{ + "www.google.com", "rr3---sn-4g5edndy.googlevideo.com", "r1---sn-2x3abcde.googlevideo.com", "i.ytimg.com", + "graph.facebook.com", "api.twitter.com", "www.baidu.com", "github.com", "objects.githubusercontent.com", + "login.microsoftonline.com", "e1234.dscb.akamaiedge.net", "d1a2b3c4d5e6f7.cloudfront.net", + "s3.us-east-1.amazonaws.com", "123456789012.dkr.ecr.us-east-1.amazonaws.com", "www.wikipedia.org", + "discord.com", "telegram.org", "store.steampowered.com", "www.91porn.com", "ns-1234.awsdns-12.org", + } + bench := func(b *testing.B, ctor func(pattern string) func(string) bool) { + var matchers []func(string) bool + for _, p := range patterns { + matchers = append(matchers, ctor(p)) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + for _, d := range domains { + for _, match := range matchers { + _ = match(d) + } + } + } + } + b.Run("regexp", func(b *testing.B) { + bench(b, func(pattern string) func(string) bool { + return regexp.MustCompile(pattern).MatchString + }) + }) + b.Run("prefilter", func(b *testing.B) { + bench(b, func(pattern string) func(string) bool { + m, err := Regex.New(pattern) + common.Must(err) + return m.Match + }) + }) +} + // Utility functions for benchmark func benchmarkMatcherType(b *testing.B, t Type, ctor func() MatcherGroup) { diff --git a/common/geodata/strmatcher/matchers.go b/common/geodata/strmatcher/matchers.go index a9df9d6f2..c2f46df64 100644 --- a/common/geodata/strmatcher/matchers.go +++ b/common/geodata/strmatcher/matchers.go @@ -3,6 +3,7 @@ package strmatcher import ( "errors" "regexp" + "regexp/syntax" "slices" "strings" "unicode/utf8" @@ -73,7 +74,43 @@ func (m SubstrMatcher) Match(s string) bool { // RegexMatcher is an implementation of Matcher. type RegexMatcher struct { - pattern *regexp.Regexp + pattern *regexp.Regexp + literals []string // every match contains all of them, longest first +} + +func newRegexMatcher(pattern string) (Matcher, error) { + regex, err := regexp.Compile(pattern) + if err != nil { + return nil, err + } + m := &RegexMatcher{pattern: regex} + if re, err := syntax.Parse(pattern, syntax.Perl); err == nil { // same flags as regexp.Compile + m.literals = requiredLiterals(re, nil) + slices.SortStableFunc(m.literals, func(a, b string) int { return len(b) - len(a) }) + } + return m, nil +} + +// requiredLiterals appends to dst the case-sensitive strings that every match of re contains. +func requiredLiterals(re *syntax.Regexp, dst []string) []string { + switch re.Op { + case syntax.OpLiteral: + // regexp matches U+FFFD against invalid UTF-8 bytes, strings.Contains does not + if re.Flags&syntax.FoldCase == 0 && !slices.Contains(re.Rune, utf8.RuneError) { + dst = append(dst, string(re.Rune)) + } + case syntax.OpCapture, syntax.OpPlus: + dst = requiredLiterals(re.Sub[0], dst) + case syntax.OpRepeat: + if re.Min > 0 { + dst = requiredLiterals(re.Sub[0], dst) + } + case syntax.OpConcat: + for _, sub := range re.Sub { + dst = requiredLiterals(sub, dst) + } + } + return dst } func (*RegexMatcher) Type() Type { @@ -89,6 +126,11 @@ func (m *RegexMatcher) String() string { } func (m *RegexMatcher) Match(s string) bool { + for _, l := range m.literals { + if !strings.Contains(s, l) { + return false + } + } return m.pattern.MatchString(s) } @@ -102,11 +144,7 @@ func (t Type) New(pattern string) (Matcher, error) { case Domain: return DomainMatcher(pattern), nil case Regex: // 1. regex matching is case-sensitive - regex, err := regexp.Compile(pattern) - if err != nil { - return nil, err - } - return &RegexMatcher{pattern: regex}, nil + return newRegexMatcher(pattern) default: return nil, errors.New("unknown matcher type") } @@ -135,11 +173,7 @@ func (t Type) NewDomainPattern(pattern string) (Matcher, error) { } return DomainMatcher(pattern), nil case Regex: // Regex's charset not in LDH subset - regex, err := regexp.Compile(pattern) - if err != nil { - return nil, err - } - return &RegexMatcher{pattern: regex}, nil + return newRegexMatcher(pattern) default: return nil, errors.New("unknown matcher type") } diff --git a/common/geodata/strmatcher/matchers_regex_test.go b/common/geodata/strmatcher/matchers_regex_test.go new file mode 100644 index 000000000..ea043f1f3 --- /dev/null +++ b/common/geodata/strmatcher/matchers_regex_test.go @@ -0,0 +1,60 @@ +package strmatcher + +import ( + "regexp" + "slices" + "testing" +) + +var regexLiteralCases = []struct { + pattern string + literals []string +}{ + {`(^|\.)91porn\.(best|com)$`, []string{"91porn."}}, + {`.+\.awsdns-cn-[0-9][0-9]\.(biz|com|net|top)$`, []string{".awsdns-cn-", "."}}, + {`^r+[0-9]+(---|\.)sn-(2x3|ni5|j5o)\w{5}\.googlevideo\.com$`, []string{".googlevideo.com", "sn-", "r"}}, + {`(?i)abc`, nil}, + {`ab(?i:CD)ef`, []string{"ab", "ef"}}, + {`(abc)?x`, []string{"x"}}, + {`(abc)*x`, []string{"x"}}, + {`x{0,3}yy`, []string{"yy"}}, + {`(ab)+c{2}`, []string{"ab", "c"}}, + {`abc|abd`, []string{"ab"}}, + {`\Qa.b\E`, []string{"a.b"}}, + {`a\x{FFFD}b`, nil}, + {`^[^.]+$`, nil}, +} + +func TestRegexRequiredLiterals(t *testing.T) { + for _, test := range regexLiteralCases { + m, err := newRegexMatcher(test.pattern) + if err != nil { + t.Fatal(err) + } + if got := m.(*RegexMatcher).literals; !slices.Equal(got, test.literals) { + t.Errorf("%s: got %q, want %q", test.pattern, got, test.literals) + } + } +} + +func FuzzRegexMatcher(f *testing.F) { + inputs := []string{ + "", "x", "yy", "abd", "ccc", "ABC", "abCDef", "abcdef", "abababcc", "a.b", "a\xffb", "a\uFFFDb", + "www.91porn.com", "ns1.awsdns-cn-01.top", "r1---sn-2x3abcde.googlevideo.com", + } + for _, test := range regexLiteralCases { + for _, s := range inputs { + f.Add(test.pattern, s) + } + } + f.Fuzz(func(t *testing.T, pattern, s string) { + re, err := regexp.Compile(pattern) + if err != nil { + return + } + m, _ := newRegexMatcher(pattern) + if got, want := m.Match(s), re.MatchString(s); got != want { + t.Errorf("pattern %q, input %q: got %v, want %v", pattern, s, got, want) + } + }) +}