mirrors/hugo
1
0
Fork 0
mirror of https://github.com/gohugoio/hugo.git synced 2025-04-28 14:40:43 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

parser/pageparser: Don't store the byte slices

Browse source 
On its own this change doesn't do any magic, but this is part of a bigger picture about making Hugo leaner in the
memory usage department.
This commit is contained in:
Bjørn Erik Pedersen 2022-07-07 16:11:47 +02:00
parent 72b0ccdb01
commit 223bf28004
13 changed files with 385 additions and 198 deletions
Download patch file Download diff file Expand all files Collapse all files

118
parser/pageparser/pageparser_intro_test.go
View file

@ -15,19 +15,25 @@ package pageparser
import (
"fmt"
"reflect"
"strings"
"testing"
qt "github.com/frankban/quicktest"
)
type lexerTest struct {
name string
input string
items []Item
items []typeText
}
func nti(tp ItemType, val string) Item {
return Item{tp, 0, []byte(val), false}
type typeText struct {
typ ItemType
text string
}
func nti(tp ItemType, val string) typeText {
return typeText{typ: tp, text: val}
}
var (
@ -52,48 +58,79 @@ var crLfReplacer = strings.NewReplacer("\r", "#", "\n", "$")
// TODO(bep) a way to toggle ORG mode vs the rest.
var frontMatterTests = []lexerTest{
{"empty", "", []Item{tstEOF}},
{"Byte order mark", "\ufeff\nSome text.\n", []Item{nti(TypeIgnore, "\ufeff"), tstSomeText, tstEOF}},
{"HTML Document", ` <html> `, []Item{nti(tError, "plain HTML documents not supported")}},
{"HTML Document with shortcode", `<html>{{< sc1 >}}</html>`, []Item{nti(tError, "plain HTML documents not supported")}},
{"No front matter", "\nSome text.\n", []Item{tstSomeText, tstEOF}},
{"YAML front matter", "---\nfoo: \"bar\"\n---\n\nSome text.\n", []Item{tstFrontMatterYAML, tstSomeText, tstEOF}},
{"YAML empty front matter", "---\n---\n\nSome text.\n", []Item{nti(TypeFrontMatterYAML, ""), tstSomeText, tstEOF}},
{"YAML commented out front matter", "<!--\n---\nfoo: \"bar\"\n---\n-->\nSome text.\n", []Item{nti(TypeIgnore, "<!--\n"), tstFrontMatterYAML, nti(TypeIgnore, "-->"), tstSomeText, tstEOF}},
{"YAML commented out front matter, no end", "<!--\n---\nfoo: \"bar\"\n---\nSome text.\n", []Item{nti(TypeIgnore, "<!--\n"), tstFrontMatterYAML, nti(tError, "starting HTML comment with no end")}},
{"empty", "", []typeText{tstEOF}},
{"Byte order mark", "\ufeff\nSome text.\n", []typeText{nti(TypeIgnore, "\ufeff"), tstSomeText, tstEOF}},
{"HTML Document", ` <html> `, []typeText{nti(tError, "plain HTML documents not supported")}},
{"HTML Document with shortcode", `<html>{{< sc1 >}}</html>`, []typeText{nti(tError, "plain HTML documents not supported")}},
{"No front matter", "\nSome text.\n", []typeText{tstSomeText, tstEOF}},
{"YAML front matter", "---\nfoo: \"bar\"\n---\n\nSome text.\n", []typeText{tstFrontMatterYAML, tstSomeText, tstEOF}},
{"YAML empty front matter", "---\n---\n\nSome text.\n", []typeText{nti(TypeFrontMatterYAML, ""), tstSomeText, tstEOF}},
{"YAML commented out front matter", "<!--\n---\nfoo: \"bar\"\n---\n-->\nSome text.\n", []typeText{nti(TypeIgnore, "<!--\n"), tstFrontMatterYAML, nti(TypeIgnore, "-->"), tstSomeText, tstEOF}},
{"YAML commented out front matter, no end", "<!--\n---\nfoo: \"bar\"\n---\nSome text.\n", []typeText{nti(TypeIgnore, "<!--\n"), tstFrontMatterYAML, nti(tError, "starting HTML comment with no end")}},
// Note that we keep all bytes as they are, but we need to handle CRLF
{"YAML front matter CRLF", "---\r\nfoo: \"bar\"\r\n---\n\nSome text.\n", []Item{tstFrontMatterYAMLCRLF, tstSomeText, tstEOF}},
{"TOML front matter", "+++\nfoo = \"bar\"\n+++\n\nSome text.\n", []Item{tstFrontMatterTOML, tstSomeText, tstEOF}},
{"JSON front matter", tstJSON + "\r\n\nSome text.\n", []Item{tstFrontMatterJSON, tstSomeText, tstEOF}},
{"ORG front matter", tstORG + "\nSome text.\n", []Item{tstFrontMatterORG, tstSomeText, tstEOF}},
{"Summary divider ORG", tstORG + "\nSome text.\n# more\nSome text.\n", []Item{tstFrontMatterORG, tstSomeText, nti(TypeLeadSummaryDivider, "# more\n"), nti(tText, "Some text.\n"), tstEOF}},
{"Summary divider", "+++\nfoo = \"bar\"\n+++\n\nSome text.\n<!--more-->\nSome text.\n", []Item{tstFrontMatterTOML, tstSomeText, tstSummaryDivider, nti(tText, "Some text.\n"), tstEOF}},
{"Summary divider same line", "+++\nfoo = \"bar\"\n+++\n\nSome text.<!--more-->Some text.\n", []Item{tstFrontMatterTOML, nti(tText, "\nSome text."), nti(TypeLeadSummaryDivider, "<!--more-->"), nti(tText, "Some text.\n"), tstEOF}},
{"YAML front matter CRLF", "---\r\nfoo: \"bar\"\r\n---\n\nSome text.\n", []typeText{tstFrontMatterYAMLCRLF, tstSomeText, tstEOF}},
{"TOML front matter", "+++\nfoo = \"bar\"\n+++\n\nSome text.\n", []typeText{tstFrontMatterTOML, tstSomeText, tstEOF}},
{"JSON front matter", tstJSON + "\r\n\nSome text.\n", []typeText{tstFrontMatterJSON, tstSomeText, tstEOF}},
{"ORG front matter", tstORG + "\nSome text.\n", []typeText{tstFrontMatterORG, tstSomeText, tstEOF}},
{"Summary divider ORG", tstORG + "\nSome text.\n# more\nSome text.\n", []typeText{tstFrontMatterORG, tstSomeText, nti(TypeLeadSummaryDivider, "# more\n"), nti(tText, "Some text.\n"), tstEOF}},
{"Summary divider", "+++\nfoo = \"bar\"\n+++\n\nSome text.\n<!--more-->\nSome text.\n", []typeText{tstFrontMatterTOML, tstSomeText, tstSummaryDivider, nti(tText, "Some text.\n"), tstEOF}},
{"Summary divider same line", "+++\nfoo = \"bar\"\n+++\n\nSome text.<!--more-->Some text.\n", []typeText{tstFrontMatterTOML, nti(tText, "\nSome text."), nti(TypeLeadSummaryDivider, "<!--more-->"), nti(tText, "Some text.\n"), tstEOF}},
// https://github.com/gohugoio/hugo/issues/5402
{"Summary and shortcode, no space", "+++\nfoo = \"bar\"\n+++\n\nSome text.\n<!--more-->{{< sc1 >}}\nSome text.\n", []Item{tstFrontMatterTOML, tstSomeText, nti(TypeLeadSummaryDivider, "<!--more-->"), tstLeftNoMD, tstSC1, tstRightNoMD, tstSomeText, tstEOF}},
{"Summary and shortcode, no space", "+++\nfoo = \"bar\"\n+++\n\nSome text.\n<!--more-->{{< sc1 >}}\nSome text.\n", []typeText{tstFrontMatterTOML, tstSomeText, nti(TypeLeadSummaryDivider, "<!--more-->"), tstLeftNoMD, tstSC1, tstRightNoMD, tstSomeText, tstEOF}},
// https://github.com/gohugoio/hugo/issues/5464
{"Summary and shortcode only", "+++\nfoo = \"bar\"\n+++\n{{< sc1 >}}\n<!--more-->\n{{< sc2 >}}", []Item{tstFrontMatterTOML, tstLeftNoMD, tstSC1, tstRightNoMD, tstNewline, tstSummaryDivider, tstLeftNoMD, tstSC2, tstRightNoMD, tstEOF}},
{"Summary and shortcode only", "+++\nfoo = \"bar\"\n+++\n{{< sc1 >}}\n<!--more-->\n{{< sc2 >}}", []typeText{tstFrontMatterTOML, tstLeftNoMD, tstSC1, tstRightNoMD, tstNewline, tstSummaryDivider, tstLeftNoMD, tstSC2, tstRightNoMD, tstEOF}},
}
func TestFrontMatter(t *testing.T) {
t.Parallel()
c := qt.New(t)
for i, test := range frontMatterTests {
items := collect([]byte(test.input), false, lexIntroSection)
if !equal(items, test.items) {
got := crLfReplacer.Replace(fmt.Sprint(items))
expected := crLfReplacer.Replace(fmt.Sprint(test.items))
t.Errorf("[%d] %s: got\n\t%v\nexpected\n\t%v", i, test.name, got, expected)
if !equal(test.input, items, test.items) {
got := itemsToString(items, []byte(test.input))
expected := testItemsToString(test.items)
c.Assert(got, qt.Equals, expected, qt.Commentf("Test %d: %s", i, test.name))
}
}
}
func itemsToString(items []Item, source []byte) string {
var sb strings.Builder
for i, item := range items {
var s string
if item.Err != nil {
s = item.Err.Error()
} else {
s = string(item.Val(source))
}
sb.WriteString(fmt.Sprintf("%s: %s\n", item.Type, s))
if i < len(items)-1 {
sb.WriteString("\n")
}
}
return crLfReplacer.Replace(sb.String())
}
func testItemsToString(items []typeText) string {
var sb strings.Builder
for i, item := range items {
sb.WriteString(fmt.Sprintf("%s: %s\n", item.typ, item.text))
if i < len(items)-1 {
sb.WriteString("\n")
}
}
return crLfReplacer.Replace(sb.String())
}
func collectWithConfig(input []byte, skipFrontMatter bool, stateStart stateFunc, cfg Config) (items []Item) {
l := newPageLexer(input, stateStart, cfg)
l.run()
t := l.newIterator()
iter := NewIterator(l.items)
for {
item := t.Next()
item := iter.Next()
items = append(items, item)
if item.Type == tEOF || item.Type == tError {
break
@ -108,19 +145,34 @@ func collect(input []byte, skipFrontMatter bool, stateStart stateFunc) (items []
return collectWithConfig(input, skipFrontMatter, stateStart, cfg)
}
func collectStringMain(input string) []Item {
return collect([]byte(input), true, lexMainSection)
}
// no positional checking, for now ...
func equal(i1, i2 []Item) bool {
if len(i1) != len(i2) {
func equal(source string, got []Item, expect []typeText) bool {
if len(got) != len(expect) {
return false
}
for k := range i1 {
if i1[k].Type != i2[k].Type {
sourceb := []byte(source)
for k := range got {
g := got[k]
e := expect[k]
if g.Type != e.typ {
return false
}
if !reflect.DeepEqual(i1[k].Val, i2[k].Val) {
var s string
if g.Err != nil {
s = g.Err.Error()
} else {
s = string(g.Val(sourceb))
}
if s != e.text {
return false
}
}
return true
}