prose/document_test.go at master · Shnitzelil/prose · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
package prose

import (
	"path/filepath"
	"strings"
	"testing"
)

func BenchmarkDoc(b *testing.B) {
	content := readDataFile(filepath.Join(testdata, "sherlock.txt"))
	text := string(content)
	for n := 0; n < b.N; n++ {
		_, err := NewDocument(text)
		if err != nil {
			panic(err)
		}
	}
}

func BenchmarkCustomTokenizer(b *testing.B) {
	content := readDataFile(filepath.Join(testdata, "sherlock.txt"))
	tok := NewIterTokenizer(
		UsingSanitizer(strings.NewReplacer()), // Disable sanitizer
		UsingPrefixes([]string{"(", `"`, "[", "'"}),
		UsingSuffixes([]string{",", ")", `"`, "]", "!", ";", ".", "?", ":", "'"}),
	)
	text := string(content)
	for n := 0; n < b.N; n++ {
		_, err := NewDocument(text, UsingTokenizer(tok))
		if err != nil {
			panic(err)
		}
	}
}