Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 33 additions & 38 deletions internal/source/weebcentral/scraper/chapters.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,22 @@ type ChapterInfo struct {
URL string
}

// Extract the first chapter number from its dedicated label.
// Accepts "Chapter 271.5", "Episode 139.5", "#100",
// "Part 12", or simply "12.5".
var chapterLabelPattern = regexp.MustCompile(
`(?i)^(?:chapter\s*|#\s*)(\d+(?:\.\d+)?)(?:$|[\s:-])`,
`^[^0-9]*([0-9]+(?:\.[0-9]+)?)(?:$|[\s:;,()\-–—])`,
)

func parseChapterLabel(
text string,
) (common.ChapterNumber, string, bool) {
var zero common.ChapterNumber

label := strings.TrimSpace(text)
label := strings.Join(
strings.Fields(text),
" ",
)

match := chapterLabelPattern.FindStringSubmatch(label)
if len(match) != 2 {
Expand Down Expand Up @@ -136,50 +142,39 @@ func GetScanInfo(
)
}

chapterList := doc.Find("#chapter-list")

// The full-chapter-list endpoint may return an HTML fragment
// without the outer #chapter-list container.
if chapterList.Length() == 0 {
log.Debug(
"Chapter list container not found; parsing response as an HTML fragment\n",
)

chapterList = doc.Selection
}

var chapters []ChapterInfo

doc.Find(`a[href^="/chapters/"]`).Each(
func(i int, selection *goquery.Selection) {
chapterList.Find(`a[href^="/chapters/"]`).Each(
func(_ int, selection *goquery.Selection) {
href, exists := selection.Attr("href")
if !exists {
return
}

var (
chapterNumber common.ChapterNumber
label string
found bool
)

// The first direct spans inside span.grow may contain:
//
// "Chapter 100"
// "Chapter 100 - Some title"
// "# 100"
// "#100"
//
// Other nested spans such as "Last Read" are ignored.
selection.Find("span.grow > span").EachWithBreak(
func(i int, span *goquery.Selection) bool {
text := strings.TrimSpace(span.Text())

number, parsedLabel, ok := parseChapterLabel(text)
if !ok {
return true
}

chapterNumber = number
label = parsedLabel
found = true

return false
},
)
// Read only the chapter's dedicated label.
// Ignore nested status elements such as "Last Read".
rawLabel := selection.
Find("span.grow > span").
First().
Text()

if !found {
number, label, ok := parseChapterLabel(rawLabel)
if !ok {
log.Debug(
"Skipping chapter with unrecognized label: %s\n",
strings.TrimSpace(selection.Text()),
"Skipping chapter with unrecognized label: %q\n",
strings.TrimSpace(rawLabel),
)
return
}
Expand All @@ -200,7 +195,7 @@ func GetScanInfo(
chapters = append(
chapters,
ChapterInfo{
Number: chapterNumber,
Number: number,
Label: label,
URL: chapterURL,
},
Expand Down
50 changes: 50 additions & 0 deletions internal/source/weebcentral/scraper/chapters_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package weebcentralscraper

import "testing"

func TestParseChapterLabel(t *testing.T) {
tests := []struct {
label string
want string
valid bool
}{
{"Chapter 271.5", "271.5", true},
{"Episode 139.5", "139.5", true},
{"Episode 9", "9", true},
{"#100", "100", true},
{"# 12.5", "12.5", true},
{"Part 12 - Bonus", "12", true},
{"Chapter 0", "0", true},
{"12.5", "12.5", true},
{"Chapter 12: Bonus 2", "12", true},
{"Last Read", "", false},
{"Chapter ???", "", false},
{"Special", "", false},
}

for _, tt := range tests {
t.Run(tt.label, func(t *testing.T) {
number, _, ok := parseChapterLabel(tt.label)

if ok != tt.valid {
t.Fatalf(
"valid = %v, want %v",
ok,
tt.valid,
)
}

if !tt.valid {
return
}

if got := number.String(); got != tt.want {
t.Errorf(
"number = %q, want %q",
got,
tt.want,
)
}
})
}
}
Loading