You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
738 lines
24 KiB
738 lines
24 KiB
package grok |
|
|
|
import ( |
|
"bufio" |
|
"fmt" |
|
"strings" |
|
"testing" |
|
) |
|
|
|
func TestNew(t *testing.T) { |
|
g, _ := New() |
|
if len(g.patterns) == 0 { |
|
t.Fatal("the Grok object should have some patterns pre loaded") |
|
} |
|
|
|
g, _ = NewWithConfig(&Config{NamedCapturesOnly: true}) |
|
if len(g.patterns) == 0 { |
|
t.Fatal("the Grok object should have some patterns pre loaded") |
|
} |
|
} |
|
|
|
func TestParseWithDefaultCaptureMode(t *testing.T) { |
|
g, _ := NewWithConfig(&Config{NamedCapturesOnly: true}) |
|
if captures, err := g.Parse("%{COMMONAPACHELOG}", `127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`); err != nil { |
|
t.Fatalf("error can not capture : %s", err.Error()) |
|
} else { |
|
if captures["timestamp"] != "23/Apr/2014:22:58:32 +0200" { |
|
t.Fatalf("%s should be '%s' have '%s'", "timestamp", "23/Apr/2014:22:58:32 +0200", captures["timestamp"]) |
|
} |
|
if captures["TIME"] != "" { |
|
t.Fatalf("%s should be '%s' have '%s'", "TIME", "", captures["TIME"]) |
|
} |
|
} |
|
|
|
g, _ = New() |
|
if captures, err := g.Parse("%{COMMONAPACHELOG}", `127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`); err != nil { |
|
t.Fatalf("error can not capture : %s", err.Error()) |
|
} else { |
|
if captures["timestamp"] != "23/Apr/2014:22:58:32 +0200" { |
|
t.Fatalf("%s should be '%s' have '%s'", "timestamp", "23/Apr/2014:22:58:32 +0200", captures["timestamp"]) |
|
} |
|
if captures["TIME"] != "22:58:32" { |
|
t.Fatalf("%s should be '%s' have '%s'", "TIME", "22:58:32", captures["TIME"]) |
|
} |
|
} |
|
} |
|
|
|
func TestMultiParseWithDefaultCaptureMode(t *testing.T) { |
|
g, _ := NewWithConfig(&Config{NamedCapturesOnly: true}) |
|
res, _ := g.ParseToMultiMap("%{COMMONAPACHELOG} %{COMMONAPACHELOG}", `127.0.0.1 - - [23/Apr/2014:23:58:32 +0200] "GET /index.php HTTP/1.1" 404 207 127.0.0.1 - - [24/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`) |
|
if len(res["TIME"]) != 0 { |
|
t.Fatalf("DAY should be an array of 0 elements, but is '%s'", res["TIME"]) |
|
} |
|
|
|
g, _ = New() |
|
res, _ = g.ParseToMultiMap("%{COMMONAPACHELOG} %{COMMONAPACHELOG}", `127.0.0.1 - - [23/Apr/2014:23:58:32 +0200] "GET /index.php HTTP/1.1" 404 207 127.0.0.1 - - [24/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`) |
|
if len(res["TIME"]) != 2 { |
|
t.Fatalf("TIME should be an array of 2 elements, but is '%s'", res["TIME"]) |
|
} |
|
if len(res["timestamp"]) != 2 { |
|
t.Fatalf("timestamp should be an array of 2 elements, but is '%s'", res["timestamp"]) |
|
} |
|
} |
|
|
|
func TestNewWithNoDefaultPatterns(t *testing.T) { |
|
g, _ := NewWithConfig(&Config{SkipDefaultPatterns: true}) |
|
if len(g.patterns) != 0 { |
|
t.Fatal("Using SkipDefaultPatterns the Grok object should not have any patterns pre loaded") |
|
} |
|
} |
|
|
|
func TestAddPatternErr(t *testing.T) { |
|
name := "Error" |
|
pattern := "%{ERR}" |
|
|
|
g, _ := New() |
|
err := g.addPattern(name, pattern) |
|
if err == nil { |
|
t.Fatalf("AddPattern should returns an error when path is invalid") |
|
} |
|
} |
|
|
|
func TestAddPatternsFromPathErr(t *testing.T) { |
|
g, _ := New() |
|
err := g.AddPatternsFromPath("./Lorem ipsum Minim qui in.") |
|
if err == nil { |
|
t.Fatalf("AddPatternsFromPath should returns an error when path is invalid") |
|
} |
|
} |
|
|
|
func TestConfigPatternsDir(t *testing.T) { |
|
g, err := NewWithConfig(&Config{PatternsDir: []string{"./patterns"}}) |
|
if err != nil { |
|
t.Error(err) |
|
} |
|
|
|
if captures, err := g.Parse("%{SYSLOGLINE}", `Sep 12 23:19:02 docker syslog-ng[25389]: syslog-ng starting up; version='3.5.3'`); err != nil { |
|
t.Fatalf("error : %s", err.Error()) |
|
} else { |
|
// pp.Print(captures) |
|
if captures["program"] != "syslog-ng" { |
|
t.Fatalf("%s should be '%s' have '%s'", "program", "syslog-ng", captures["program"]) |
|
} |
|
} |
|
|
|
} |
|
|
|
func TestAddPatternsFromPathFileOpenErr(t *testing.T) { |
|
t.Skipped() |
|
} |
|
|
|
func TestAddPatternsFromPathFile(t *testing.T) { |
|
g, _ := New() |
|
err := g.AddPatternsFromPath("./patterns/grok-patterns") |
|
if err != nil { |
|
t.Fatalf("err %#v", err) |
|
} |
|
} |
|
|
|
func TestAddPattern(t *testing.T) { |
|
name := "DAYO" |
|
pattern := "(?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)" |
|
|
|
g, _ := New() |
|
cPatterns := len(g.patterns) |
|
g.AddPattern(name, pattern) |
|
g.AddPattern(name+"2", pattern) |
|
if len(g.patterns) != cPatterns+2 { |
|
t.Fatalf("%d Default patterns should be available, have %d", cPatterns+2, len(g.patterns)) |
|
} |
|
|
|
g, _ = NewWithConfig(&Config{NamedCapturesOnly: true}) |
|
cPatterns = len(g.patterns) |
|
g.AddPattern(name, pattern) |
|
g.AddPattern(name+"2", pattern) |
|
if len(g.patterns) != cPatterns+2 { |
|
t.Fatalf("%d NamedCapture patterns should be available, have %d", cPatterns+2, len(g.patterns)) |
|
} |
|
} |
|
|
|
func TestMatch(t *testing.T) { |
|
g, _ := New() |
|
g.AddPatternsFromPath("./patterns") |
|
|
|
if r, err := g.Match("%{MONTH}", "June"); !r { |
|
t.Fatalf("June should match %s: err=%s", "%{MONTH}", err.Error()) |
|
} |
|
|
|
} |
|
func TestDoesNotMatch(t *testing.T) { |
|
g, _ := New() |
|
g.AddPatternsFromPath("./patterns") |
|
if r, _ := g.Match("%{MONTH}", "13"); r { |
|
t.Fatalf("13 should not match %s", "%{MONTH}") |
|
} |
|
} |
|
|
|
func TestErrorMatch(t *testing.T) { |
|
g, _ := New() |
|
if _, err := g.Match("(", "13"); err == nil { |
|
t.Fatal("Error expected") |
|
} |
|
|
|
} |
|
|
|
func TestShortName(t *testing.T) { |
|
g, _ := New() |
|
g.AddPattern("A", "a") |
|
|
|
m, err := g.Match("%{A}", "a") |
|
if err != nil { |
|
t.Fatal("a should match %%{A}: err=%s", err.Error()) |
|
} |
|
if !m { |
|
t.Fatal("%%{A} didn't match 'a'") |
|
} |
|
} |
|
|
|
func TestDayCompile(t *testing.T) { |
|
g, _ := New() |
|
g.AddPattern("DAY", "(?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)") |
|
pattern := "%{DAY}" |
|
_, err := g.compile(pattern) |
|
if err != nil { |
|
t.Fatal("Error:", err) |
|
} |
|
} |
|
|
|
func TestErrorCompile(t *testing.T) { |
|
g, _ := New() |
|
_, err := g.compile("(") |
|
if err == nil { |
|
t.Fatal("Error:", err) |
|
} |
|
} |
|
|
|
func TestNamedCaptures(t *testing.T) { |
|
g, _ := New() |
|
g.AddPatternsFromPath("./patterns") |
|
|
|
check := func(key, value, pattern, text string) { |
|
captures, _ := g.Parse(pattern, text) |
|
if captures[key] != value { |
|
t.Fatalf("%s should be '%s' have '%s'", key, value, captures[key]) |
|
} |
|
} |
|
|
|
check("jour", "Tue", |
|
"%{DAY:jour}", |
|
"Tue May 15 11:21:42 [conn1047685] moveChunk deleted: 7157", |
|
) |
|
|
|
check("day-of.week", "Tue", |
|
"%{DAY:day-of.week}", |
|
"Tue May 15 11:21:42 [conn1047685] moveChunk deleted: 7157", |
|
) |
|
} |
|
|
|
func TestErrorCaptureUnknowPattern(t *testing.T) { |
|
g, _ := New() |
|
pattern := "%{UNKNOWPATTERN}" |
|
_, err := g.Parse(pattern, "") |
|
if err == nil { |
|
t.Fatal("Expected error not set") |
|
} |
|
} |
|
|
|
func TestParse(t *testing.T) { |
|
g, _ := New() |
|
g.AddPatternsFromPath("./patterns") |
|
res, _ := g.Parse("%{DAY}", "Tue qds") |
|
if res["DAY"] != "Tue" { |
|
t.Fatalf("DAY should be 'Tue' have '%s'", res["DAY"]) |
|
} |
|
} |
|
|
|
func TestErrorParseToMultiMap(t *testing.T) { |
|
g, _ := New() |
|
pattern := "%{UNKNOWPATTERN}" |
|
_, err := g.ParseToMultiMap(pattern, "") |
|
if err == nil { |
|
t.Fatal("Expected error not set") |
|
} |
|
} |
|
|
|
func TestParseToMultiMap(t *testing.T) { |
|
g, _ := New() |
|
g.AddPatternsFromPath("./patterns") |
|
res, _ := g.ParseToMultiMap("%{COMMONAPACHELOG} %{COMMONAPACHELOG}", `127.0.0.1 - - [23/Apr/2014:23:58:32 +0200] "GET /index.php HTTP/1.1" 404 207 127.0.0.1 - - [24/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`) |
|
if len(res["TIME"]) != 2 { |
|
t.Fatalf("DAY should be an array of 3 elements, but is '%s'", res["TIME"]) |
|
} |
|
if res["TIME"][0] != "23:58:32" { |
|
t.Fatalf("TIME[0] should be '23:58:32' have '%s'", res["TIME"][0]) |
|
} |
|
if res["TIME"][1] != "22:58:32" { |
|
t.Fatalf("TIME[1] should be '22:58:32' have '%s'", res["TIME"][1]) |
|
} |
|
} |
|
|
|
func TestParseToMultiMapOnlyNamedCaptures(t *testing.T) { |
|
g, _ := NewWithConfig(&Config{NamedCapturesOnly: true}) |
|
g.AddPatternsFromPath("./patterns") |
|
res, _ := g.ParseToMultiMap("%{COMMONAPACHELOG} %{COMMONAPACHELOG}", `127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207 127.0.0.1 - - [24/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`) |
|
if len(res["timestamp"]) != 2 { |
|
t.Fatalf("timestamp should be an array of 2 elements, but is '%s'", res["timestamp"]) |
|
} |
|
if res["timestamp"][0] != "23/Apr/2014:22:58:32 +0200" { |
|
t.Fatalf("timestamp[0] should be '23/Apr/2014:22:58:32 +0200' have '%s'", res["DAY"][0]) |
|
} |
|
if res["timestamp"][1] != "24/Apr/2014:22:58:32 +0200" { |
|
t.Fatalf("timestamp[1] should be '24/Apr/2014:22:58:32 +0200' have '%s'", res["DAY"][1]) |
|
} |
|
} |
|
|
|
func TestCaptureAll(t *testing.T) { |
|
g, _ := New() |
|
g.AddPatternsFromPath("./patterns") |
|
|
|
check := func(key, value, pattern, text string) { |
|
|
|
if captures, err := g.Parse(pattern, text); err != nil { |
|
t.Fatalf("error can not capture : %s", err.Error()) |
|
} else { |
|
if captures[key] != value { |
|
t.Fatalf("%s should be '%s' have '%s'", key, value, captures[key]) |
|
} |
|
} |
|
} |
|
|
|
check("timestamp", "23/Apr/2014:22:58:32 +0200", |
|
"%{COMMONAPACHELOG}", |
|
`127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`, |
|
) |
|
check("TIME", "22:58:32", |
|
"%{COMMONAPACHELOG}", |
|
`127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`, |
|
) |
|
check("SECOND", `17,1599`, "%{TIMESTAMP_ISO8601}", `s d9fq4999s ../ sdf 2013-11-06 04:50:17,1599sd`) |
|
check("HOSTNAME", `google.com`, "%{HOSTPORT}", `google.com:8080`) |
|
//HOSTPORT |
|
check("POSINT", `8080`, "%{HOSTPORT}", `google.com:8080`) |
|
} |
|
|
|
func TestNamedCapture(t *testing.T) { |
|
g, _ := NewWithConfig(&Config{NamedCapturesOnly: true}) |
|
g.AddPatternsFromPath("./patterns") |
|
|
|
check := func(key, value, pattern, text string) { |
|
if captures, err := g.Parse(pattern, text); err != nil { |
|
t.Fatalf("error can not capture : %s", err.Error()) |
|
} else { |
|
if captures[key] != value { |
|
t.Fatalf("%s should be '%s' have '%s'", key, value, captures[key]) |
|
} |
|
} |
|
} |
|
|
|
check("timestamp", "23/Apr/2014:22:58:32 +0200", |
|
"%{COMMONAPACHELOG}", |
|
`127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`, |
|
) |
|
check("TIME", "", |
|
"%{COMMONAPACHELOG}", |
|
`127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`, |
|
) |
|
check("SECOND", ``, "%{TIMESTAMP_ISO8601}", `s d9fq4999s ../ sdf 2013-11-06 04:50:17,1599sd`) |
|
check("HOSTNAME", ``, "%{HOSTPORT}", `google.com:8080`) |
|
//HOSTPORT |
|
check("POSINT", ``, "%{HOSTPORT}", `google.com:8080`) |
|
} |
|
|
|
func TestRemoveEmptyValues(t *testing.T) { |
|
g, _ := NewWithConfig(&Config{NamedCapturesOnly: true, RemoveEmptyValues: true}) |
|
|
|
capturesExists := func(key, pattern, text string) { |
|
if captures, err := g.Parse(pattern, text); err != nil { |
|
t.Fatalf("error can not capture : %s", err.Error()) |
|
} else { |
|
if _, ok := captures[key]; ok { |
|
t.Fatalf("%s should be absent", key) |
|
} |
|
} |
|
} |
|
|
|
capturesExists("rawrequest", "%{COMMONAPACHELOG}", |
|
`127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`, |
|
) |
|
|
|
} |
|
|
|
func TestCapturesAndNamedCapture(t *testing.T) { |
|
|
|
check := func(key, value, pattern, text string) { |
|
g, _ := New() |
|
if captures, err := g.Parse(pattern, text); err != nil { |
|
t.Fatalf("error can not capture : %s", err.Error()) |
|
} else { |
|
if captures[key] != value { |
|
t.Fatalf("%s should be '%s' have '%s'", key, value, captures[key]) |
|
} |
|
} |
|
} |
|
|
|
checkNamed := func(key, value, pattern, text string) { |
|
g, _ := NewWithConfig(&Config{NamedCapturesOnly: true}) |
|
if captures, err := g.Parse(pattern, text); err != nil { |
|
t.Fatalf("error can not capture : %s", err.Error()) |
|
} else { |
|
if captures[key] != value { |
|
t.Fatalf("%s should be '%s' have '%s'", key, value, captures[key]) |
|
} |
|
} |
|
|
|
} |
|
|
|
check("DAY", "Tue", |
|
"%{DAY}", |
|
"Tue May 15 11:21:42 [conn1047685] moveChunk deleted: 7157", |
|
) |
|
checkNamed("jour", "Tue", |
|
"%{DAY:jour}", |
|
"Tue May 15 11:21:42 [conn1047685] moveChunk deleted: 7157", |
|
) |
|
check("clientip", "127.0.0.1", |
|
"%{COMMONAPACHELOG}", |
|
`127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`, |
|
) |
|
check("verb", "GET", |
|
"%{COMMONAPACHELOG}", |
|
`127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`, |
|
) |
|
check("timestamp", "23/Apr/2014:22:58:32 +0200", |
|
"%{COMMONAPACHELOG}", |
|
`127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`, |
|
) |
|
check("bytes", "207", |
|
"%{COMMONAPACHELOG}", |
|
`127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`, |
|
) |
|
|
|
//PATH |
|
check("WINPATH", `c:\winfows\sdf.txt`, "%{WINPATH}", `s dfqs c:\winfows\sdf.txt`) |
|
check("WINPATH", `\\sdf\winfows\sdf.txt`, "%{WINPATH}", `s dfqs \\sdf\winfows\sdf.txt`) |
|
check("UNIXPATH", `/usr/lib/`, "%{UNIXPATH}", `s dfqs /usr/lib/ sqfd`) |
|
check("UNIXPATH", `/usr/lib`, "%{UNIXPATH}", `s dfqs /usr/lib sqfd`) |
|
check("UNIXPATH", `/usr/`, "%{UNIXPATH}", `s dfqs /usr/ sqfd`) |
|
check("UNIXPATH", `/usr`, "%{UNIXPATH}", `s dfqs /usr sqfd`) |
|
check("UNIXPATH", `/`, "%{UNIXPATH}", `s dfqs / sqfd`) |
|
|
|
//YEAR |
|
check("YEAR", `4999`, "%{YEAR}", `s d9fq4999s ../ sdf`) |
|
check("YEAR", `79`, "%{YEAR}", `s d79fq4999s ../ sdf`) |
|
check("TIMESTAMP_ISO8601", `2013-11-06 04:50:17,1599`, "%{TIMESTAMP_ISO8601}", `s d9fq4999s ../ sdf 2013-11-06 04:50:17,1599sd`) |
|
|
|
//MAC |
|
check("MAC", `01:02:03:04:ab:cf`, "%{MAC}", `s d9fq4999s ../ sdf 2013- 01:02:03:04:ab:cf 11-06 04:50:17,1599sd`) |
|
check("MAC", `01-02-03-04-ab-cd`, "%{MAC}", `s d9fq4999s ../ sdf 2013- 01-02-03-04-ab-cd 11-06 04:50:17,1599sd`) |
|
|
|
//QUOTEDSTRING |
|
check("QUOTEDSTRING", `"lkj"`, "%{QUOTEDSTRING}", `qsdklfjqsd fk"lkj"mkj`) |
|
check("QUOTEDSTRING", `'lkj'`, "%{QUOTEDSTRING}", `qsdklfjqsd fk'lkj'mkj`) |
|
check("QUOTEDSTRING", `"fk'lkj'm"`, "%{QUOTEDSTRING}", `qsdklfjqsd "fk'lkj'm"kj`) |
|
check("QUOTEDSTRING", `'fk"lkj"m'`, "%{QUOTEDSTRING}", `qsdklfjqsd 'fk"lkj"m'kj`) |
|
|
|
//BASE10NUM |
|
check("BASE10NUM", `1`, "%{BASE10NUM}", `1`) // this is a nice one |
|
check("BASE10NUM", `8080`, "%{BASE10NUM}", `qsfd8080qsfd`) |
|
|
|
} |
|
|
|
// Should be run with -race |
|
func TestConcurentParse(t *testing.T) { |
|
g, _ := New() |
|
g.AddPatternsFromPath("./patterns") |
|
|
|
check := func(key, value, pattern, text string) { |
|
|
|
if captures, err := g.Parse(pattern, text); err != nil { |
|
t.Fatalf("error can not capture : %s", err.Error()) |
|
} else { |
|
if captures[key] != value { |
|
t.Fatalf("%s should be '%s' have '%s'", key, value, captures[key]) |
|
} |
|
} |
|
} |
|
|
|
go check("QUOTEDSTRING", `"lkj"`, "%{QUOTEDSTRING}", `qsdklfjqsd fk"lkj"mkj`) |
|
go check("QUOTEDSTRING", `'lkj'`, "%{QUOTEDSTRING}", `qsdklfjqsd fk'lkj'mkj`) |
|
go check("QUOTEDSTRING", `'lkj'`, "%{QUOTEDSTRING}", `qsdklfjqsd fk'lkj'mkj`) |
|
go check("QUOTEDSTRING", `"fk'lkj'm"`, "%{QUOTEDSTRING}", `qsdklfjqsd "fk'lkj'm"kj`) |
|
go check("QUOTEDSTRING", `'fk"lkj"m'`, "%{QUOTEDSTRING}", `qsdklfjqsd 'fk"lkj"m'kj`) |
|
} |
|
|
|
func TestPatterns(t *testing.T) { |
|
g, _ := NewWithConfig(&Config{SkipDefaultPatterns: true}) |
|
if len(g.patterns) != 0 { |
|
t.Fatalf("Patterns should return 0, have '%d'", len(g.patterns)) |
|
} |
|
name := "DAY0" |
|
pattern := "(?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)" |
|
|
|
g.AddPattern(name, pattern) |
|
g.AddPattern(name+"1", pattern) |
|
if len(g.patterns) != 2 { |
|
t.Fatalf("Patterns should return 2, have '%d'", len(g.patterns)) |
|
} |
|
} |
|
|
|
func TestParseTypedWithDefaultCaptureMode(t *testing.T) { |
|
g, _ := NewWithConfig(&Config{NamedCapturesOnly: true}) |
|
if captures, err := g.ParseTyped("%{IPV4:ip:string} %{NUMBER:status:int} %{NUMBER:duration:float}", `127.0.0.1 200 0.8`); err != nil { |
|
t.Fatalf("error can not capture : %s", err.Error()) |
|
} else { |
|
if captures["ip"] != "127.0.0.1" { |
|
t.Fatalf("%s should be '%s' have '%s'", "ip", "127.0.0.1", captures["ip"]) |
|
} else { |
|
if captures["status"] != 200 { |
|
t.Fatalf("%s should be '%d' have '%d'", "status", 200, captures["status"]) |
|
} else { |
|
if captures["duration"] != 0.8 { |
|
t.Fatalf("%s should be '%f' have '%f'", "duration", 0.8, captures["duration"]) |
|
} |
|
} |
|
} |
|
} |
|
} |
|
|
|
func TestParseTypedWithNoTypeInfo(t *testing.T) { |
|
g, _ := NewWithConfig(&Config{NamedCapturesOnly: true}) |
|
if captures, err := g.ParseTyped("%{COMMONAPACHELOG}", `127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`); err != nil { |
|
t.Fatalf("error can not capture : %s", err.Error()) |
|
} else { |
|
if captures["timestamp"] != "23/Apr/2014:22:58:32 +0200" { |
|
t.Fatalf("%s should be '%s' have '%s'", "timestamp", "23/Apr/2014:22:58:32 +0200", captures["timestamp"]) |
|
} |
|
if captures["TIME"] != nil { |
|
t.Fatalf("%s should be nil have '%s'", "TIME", captures["TIME"]) |
|
} |
|
} |
|
|
|
g, _ = New() |
|
if captures, err := g.ParseTyped("%{COMMONAPACHELOG}", `127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`); err != nil { |
|
t.Fatalf("error can not capture : %s", err.Error()) |
|
} else { |
|
if captures["timestamp"] != "23/Apr/2014:22:58:32 +0200" { |
|
t.Fatalf("%s should be '%s' have '%s'", "timestamp", "23/Apr/2014:22:58:32 +0200", captures["timestamp"]) |
|
} |
|
if captures["TIME"] != "22:58:32" { |
|
t.Fatalf("%s should be '%s' have '%s'", "TIME", "22:58:32", captures["TIME"]) |
|
} |
|
} |
|
} |
|
|
|
func TestParseTypedWithIntegerTypeCoercion(t *testing.T) { |
|
g, _ := NewWithConfig(&Config{NamedCapturesOnly: true}) |
|
if captures, err := g.ParseTyped("%{WORD:coerced:int}", `5.75`); err != nil { |
|
t.Fatalf("error can not capture : %s", err.Error()) |
|
} else { |
|
if captures["coerced"] != 5 { |
|
t.Fatalf("%s should be '%s' have '%s'", "coerced", "5", captures["coerced"]) |
|
} |
|
} |
|
} |
|
|
|
func TestParseTypedWithUnknownType(t *testing.T) { |
|
g, _ := NewWithConfig(&Config{NamedCapturesOnly: true}) |
|
if _, err := g.ParseTyped("%{WORD:word:unknown}", `hello`); err == nil { |
|
t.Fatalf("parsing an unknown type must result in a conversion error") |
|
} |
|
} |
|
|
|
func TestParseTypedErrorCaptureUnknowPattern(t *testing.T) { |
|
g, _ := New() |
|
pattern := "%{UNKNOWPATTERN}" |
|
_, err := g.ParseTyped(pattern, "") |
|
if err == nil { |
|
t.Fatal("Expected error not set") |
|
} |
|
} |
|
|
|
func TestParseTypedWithTypedParents(t *testing.T) { |
|
g, _ := NewWithConfig(&Config{NamedCapturesOnly: true}) |
|
g.AddPattern("TESTCOMMON", `%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes:int}|-)`) |
|
if captures, err := g.ParseTyped("%{TESTCOMMON}", `127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`); err != nil { |
|
t.Fatalf("error can not capture : %s", err.Error()) |
|
} else { |
|
if captures["bytes"] != 207 { |
|
t.Fatalf("%s should be '%s' have '%s'", "bytes", "207", captures["bytes"]) |
|
} |
|
} |
|
} |
|
|
|
func TestParseTypedWithSemanticHomonyms(t *testing.T) { |
|
g, _ := NewWithConfig(&Config{NamedCapturesOnly: true, SkipDefaultPatterns: true}) |
|
|
|
g.AddPattern("BASE10NUM", `([+-]?(?:[0-9]+(?:\.[0-9]+)?)|\.[0-9]+)`) |
|
g.AddPattern("NUMBER", `(?:%{BASE10NUM})`) |
|
g.AddPattern("MYNUM", `%{NUMBER:bytes:int}`) |
|
g.AddPattern("MYSTR", `%{NUMBER:bytes:string}`) |
|
|
|
if captures, err := g.ParseTyped("%{MYNUM}", `207`); err != nil { |
|
t.Fatalf("error can not scapture : %s", err.Error()) |
|
} else { |
|
if captures["bytes"] != 207 { |
|
t.Fatalf("%s should be %#v have %#v", "bytes", 207, captures["bytes"]) |
|
} |
|
} |
|
if captures, err := g.ParseTyped("%{MYSTR}", `207`); err != nil { |
|
t.Fatalf("error can not capture : %s", err.Error()) |
|
} else { |
|
if captures["bytes"] != "207" { |
|
t.Fatalf("%s should be %#v have %#v", "bytes", "207", captures["bytes"]) |
|
} |
|
} |
|
} |
|
|
|
var resultNew *Grok |
|
|
|
func BenchmarkNew(b *testing.B) { |
|
b.ReportAllocs() |
|
b.ResetTimer() |
|
var g *Grok |
|
// run the check function b.N times |
|
for n := 0; n < b.N; n++ { |
|
g, _ = NewWithConfig(&Config{NamedCapturesOnly: true}) |
|
} |
|
resultNew = g |
|
} |
|
|
|
func BenchmarkCaptures(b *testing.B) { |
|
g, _ := NewWithConfig(&Config{NamedCapturesOnly: true}) |
|
b.ReportAllocs() |
|
b.ResetTimer() |
|
// run the check function b.N times |
|
for n := 0; n < b.N; n++ { |
|
g.Parse(`%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)`, `127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`) |
|
} |
|
} |
|
|
|
func BenchmarkCapturesTypedFake(b *testing.B) { |
|
g, _ := NewWithConfig(&Config{NamedCapturesOnly: true}) |
|
b.ReportAllocs() |
|
b.ResetTimer() |
|
// run the check function b.N times |
|
for n := 0; n < b.N; n++ { |
|
g.Parse(`%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)`, `127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`) |
|
} |
|
} |
|
|
|
func BenchmarkCapturesTypedReal(b *testing.B) { |
|
g, _ := NewWithConfig(&Config{NamedCapturesOnly: true}) |
|
b.ReportAllocs() |
|
b.ResetTimer() |
|
// run the check function b.N times |
|
for n := 0; n < b.N; n++ { |
|
g.ParseTyped(`%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion:int})?|%{DATA:rawrequest})" %{NUMBER:response:int} (?:%{NUMBER:bytes:int}|-)`, `127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`) |
|
} |
|
} |
|
|
|
func BenchmarkParallelCaptures(b *testing.B) { |
|
g, _ := NewWithConfig(&Config{NamedCapturesOnly: true}) |
|
b.ReportAllocs() |
|
b.ResetTimer() |
|
|
|
b.RunParallel(func(b *testing.PB) { |
|
for b.Next() { |
|
g.Parse(`%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)`, `127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`) |
|
} |
|
}) |
|
} |
|
|
|
func TestGrok_AddPatternsFromMap_not_exist(t *testing.T) { |
|
defer func() { |
|
if r := recover(); r != nil { |
|
t.Fatalf("AddPatternsFromMap panics: %v", r) |
|
} |
|
}() |
|
g, _ := NewWithConfig(&Config{SkipDefaultPatterns: true}) |
|
err := g.AddPatternsFromMap(map[string]string{ |
|
"SOME": "%{NOT_EXIST}", |
|
}) |
|
if err == nil { |
|
t.Errorf("AddPatternsFromMap should returns an error") |
|
} |
|
} |
|
|
|
func TestGrok_AddPatternsFromMap_simple(t *testing.T) { |
|
defer func() { |
|
if r := recover(); r != nil { |
|
t.Fatalf("AddPatternsFromMap panics: %v", r) |
|
} |
|
}() |
|
g, _ := NewWithConfig(&Config{SkipDefaultPatterns: true}) |
|
err := g.AddPatternsFromMap(map[string]string{ |
|
"NO3": `\d{3}`, |
|
}) |
|
if err != nil { |
|
t.Errorf("AddPatternsFromMap returns an error: %v", err) |
|
} |
|
mss, err := g.Parse("%{NO3:match}", "333") |
|
if err != nil { |
|
t.Error("parsing error:", err) |
|
t.FailNow() |
|
} |
|
if mss["match"] != "333" { |
|
t.Errorf("bad match: expected 333, got %s", mss["match"]) |
|
} |
|
} |
|
|
|
func TestGrok_AddPatternsFromMap_complex(t *testing.T) { |
|
defer func() { |
|
if r := recover(); r != nil { |
|
t.Fatalf("AddPatternsFromMap panics: %v", r) |
|
} |
|
}() |
|
g, _ := NewWithConfig(&Config{ |
|
SkipDefaultPatterns: true, |
|
NamedCapturesOnly: true, |
|
}) |
|
err := g.AddPatternsFromMap(map[string]string{ |
|
"NO3": `\d{3}`, |
|
"NO6": "%{NO3}%{NO3}", |
|
}) |
|
if err != nil { |
|
t.Errorf("AddPatternsFromMap returns an error: %v", err) |
|
} |
|
mss, err := g.Parse("%{NO6:number}", "333666") |
|
if err != nil { |
|
t.Error("parsing error:", err) |
|
t.FailNow() |
|
} |
|
if mss["number"] != "333666" { |
|
t.Errorf("bad match: expected 333666, got %s", mss["match"]) |
|
} |
|
} |
|
|
|
func TestParseStream(t *testing.T) { |
|
g, _ := New() |
|
pTest := func(m map[string]string) error { |
|
ts, ok := m["timestamp"] |
|
if !ok { |
|
t.Error("timestamp not found") |
|
} |
|
if len(ts) == 0 { |
|
t.Error("empty timestamp") |
|
} |
|
return nil |
|
} |
|
const testLog = `127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207 |
|
127.0.0.1 - - [23/Apr/2014:22:59:32 +0200] "GET /index.php HTTP/1.1" 404 207 |
|
127.0.0.1 - - [23/Apr/2014:23:00:32 +0200] "GET /index.php HTTP/1.1" 404 207 |
|
` |
|
|
|
r := bufio.NewReader(strings.NewReader(testLog)) |
|
if err := g.ParseStream(r, "%{COMMONAPACHELOG}", pTest); err != nil { |
|
t.Fatal(err) |
|
} |
|
} |
|
|
|
func TestParseStreamError(t *testing.T) { |
|
g, _ := New() |
|
pTest := func(m map[string]string) error { |
|
if _, ok := m["timestamp"]; !ok { |
|
return fmt.Errorf("timestamp not found") |
|
} |
|
return nil |
|
} |
|
const testLog = `127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207 |
|
127.0.0.1 - - [xxxxxxxxxxxxxxxxxxxx +0200] "GET /index.php HTTP/1.1" 404 207 |
|
127.0.0.1 - - [23/Apr/2014:23:00:32 +0200] "GET /index.php HTTP/1.1" 404 207 |
|
` |
|
|
|
r := bufio.NewReader(strings.NewReader(testLog)) |
|
if err := g.ParseStream(r, "%{COMMONAPACHELOG}", pTest); err == nil { |
|
t.Fatal("Error expected") |
|
} |
|
}
|
|
|