mirrors/hugo
1
0
Fork 0
mirror of https://github.com/gohugoio/hugo.git synced 2025-04-28 14:40:43 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

hugolib: Continue the file context/line number errors work

Browse source 
See #5324
This commit is contained in:
Bjørn Erik Pedersen 2018-10-21 12:20:21 +02:00
parent 7930d2132a
commit d1661b823a
No known key found for this signature in database
GPG key ID: 330E6E2BD4859D8F
23 changed files with 444 additions and 236 deletions
Download patch file Download diff file Expand all files Collapse all files

2
commands/server_errors.go
View file

@ -72,7 +72,7 @@ var buildErrorTemplate = `<!doctype html>
<main> <main>
{{ highlight .Error "apl" "noclasses=true,style=monokai" }} {{ highlight .Error "apl" "noclasses=true,style=monokai" }}
{{ with .File }} {{ with .File }}
{{ $params := printf "noclasses=true,style=monokai,linenos=table,hl_lines=%d,linenostart=%d" (add .Pos 1) .LineNumber }} {{ $params := printf "noclasses=true,style=monokai,linenos=table,hl_lines=%d,linenostart=%d" (add .Pos 1) (sub .LineNumber .Pos) }}
{{ $lexer := .ChromaLexer | default "go-html-template" }} {{ $lexer := .ChromaLexer | default "go-html-template" }}
{{ highlight (delimit .Lines "\n") $lexer $params }} {{ highlight (delimit .Lines "\n") $lexer $params }}
{{ end }} {{ end }}

49
common/herrors/error_locator.go
View file

@ -16,12 +16,17 @@ package herrors
import ( import (
"bufio" "bufio"
"fmt"
"io" "io"
"strings" "strings"
"github.com/gohugoio/hugo/helpers"
"github.com/spf13/afero" "github.com/spf13/afero"
) )
var fileErrorFormat = "\"%s:%d:%d\": %s"
// LineMatcher is used to match a line with an error. // LineMatcher is used to match a line with an error.
type LineMatcher func(le FileError, lineNumber int, line string) bool type LineMatcher func(le FileError, lineNumber int, line string) bool
@ -34,6 +39,8 @@ var SimpleLineMatcher = func(le FileError, lineNumber int, line string) bool {
// ErrorContext contains contextual information about an error. This will // ErrorContext contains contextual information about an error. This will
// typically be the lines surrounding some problem in a file. // typically be the lines surrounding some problem in a file.
type ErrorContext struct { type ErrorContext struct {
// The source filename.
Filename string
// If a match will contain the matched line and up to 2 lines before and after. // If a match will contain the matched line and up to 2 lines before and after.
// Will be empty if no match. // Will be empty if no match.
@ -45,6 +52,9 @@ type ErrorContext struct {
// The linenumber in the source file from where the Lines start. Starting at 1. // The linenumber in the source file from where the Lines start. Starting at 1.
LineNumber int LineNumber int
// The column number in the source file. Starting at 1.
ColumnNumber int
// The lexer to use for syntax highlighting. // The lexer to use for syntax highlighting.
// https://gohugo.io/content-management/syntax-highlighting/#list-of-chroma-highlighting-languages // https://gohugo.io/content-management/syntax-highlighting/#list-of-chroma-highlighting-languages
ChromaLexer string ChromaLexer string
@ -60,7 +70,7 @@ type ErrorWithFileContext struct {
} }
func (e *ErrorWithFileContext) Error() string { func (e *ErrorWithFileContext) Error() string {
return e.cause.Error() return fmt.Sprintf(fileErrorFormat, e.Filename, e.LineNumber, e.ColumnNumber, e.cause.Error())
} }
func (e *ErrorWithFileContext) Cause() error { func (e *ErrorWithFileContext) Cause() error {
@ -69,39 +79,40 @@ func (e *ErrorWithFileContext) Cause() error {
// WithFileContextForFile will try to add a file context with lines matching the given matcher. // WithFileContextForFile will try to add a file context with lines matching the given matcher.
// If no match could be found, the original error is returned with false as the second return value. // If no match could be found, the original error is returned with false as the second return value.
func WithFileContextForFile(e error, filename string, fs afero.Fs, chromaLexer string, matcher LineMatcher) (error, bool) { func WithFileContextForFile(e error, realFilename, filename string, fs afero.Fs, matcher LineMatcher) (error, bool) {
f, err := fs.Open(filename) f, err := fs.Open(filename)
if err != nil { if err != nil {
return e, false return e, false
} }
defer f.Close() defer f.Close()
return WithFileContext(e, f, chromaLexer, matcher) return WithFileContext(e, realFilename, f, matcher)
} }
// WithFileContextForFile will try to add a file context with lines matching the given matcher. // WithFileContextForFile will try to add a file context with lines matching the given matcher.
// If no match could be found, the original error is returned with false as the second return value. // If no match could be found, the original error is returned with false as the second return value.
func WithFileContext(e error, r io.Reader, chromaLexer string, matcher LineMatcher) (error, bool) { func WithFileContext(e error, realFilename string, r io.Reader, matcher LineMatcher) (error, bool) {
if e == nil { if e == nil {
panic("error missing") panic("error missing")
} }
le := UnwrapFileError(e) le := UnwrapFileError(e)
if le == nil { if le == nil {
var ok bool var ok bool
if le, ok = ToFileError("bash", e).(FileError); !ok { if le, ok = ToFileError("", e).(FileError); !ok {
return e, false return e, false
} }
} }
errCtx := locateError(r, le, matcher) errCtx := locateError(r, le, matcher)
errCtx.Filename = realFilename
if errCtx.LineNumber == -1 { if errCtx.LineNumber == -1 {
return e, false return e, false
} }
if chromaLexer != "" { if le.Type() != "" {
errCtx.ChromaLexer = chromaLexer
} else {
errCtx.ChromaLexer = chromaLexerFromType(le.Type()) errCtx.ChromaLexer = chromaLexerFromType(le.Type())
} else {
errCtx.ChromaLexer = chromaLexerFromFilename(realFilename)
} }
return &ErrorWithFileContext{cause: e, ErrorContext: errCtx}, true return &ErrorWithFileContext{cause: e, ErrorContext: errCtx}, true
@ -124,9 +135,22 @@ func UnwrapErrorWithFileContext(err error) *ErrorWithFileContext {
} }
func chromaLexerFromType(fileType string) string { func chromaLexerFromType(fileType string) string {
switch fileType {
case "html", "htm":
return "go-html-template"
}
return fileType return fileType
} }
func chromaLexerFromFilename(filename string) string {
if strings.Contains(filename, "layouts") {
return "go-html-template"
}
ext := helpers.ExtNoDelimiter(filename)
return chromaLexerFromType(ext)
}
func locateErrorInString(le FileError, src string, matcher LineMatcher) ErrorContext { func locateErrorInString(le FileError, src string, matcher LineMatcher) ErrorContext {
return locateError(strings.NewReader(src), nil, matcher) return locateError(strings.NewReader(src), nil, matcher)
} }
@ -135,6 +159,11 @@ func locateError(r io.Reader, le FileError, matches LineMatcher) ErrorContext {
var errCtx ErrorContext var errCtx ErrorContext
s := bufio.NewScanner(r) s := bufio.NewScanner(r)
errCtx.ColumnNumber = 1
if le != nil {
errCtx.ColumnNumber = le.ColumnNumber()
}
lineNo := 0 lineNo := 0
var buff [6]string var buff [6]string
@ -152,7 +181,7 @@ func locateError(r io.Reader, le FileError, matches LineMatcher) ErrorContext {
if errCtx.Pos == -1 && matches(le, lineNo, txt) { if errCtx.Pos == -1 && matches(le, lineNo, txt) {
errCtx.Pos = i errCtx.Pos = i
errCtx.LineNumber = lineNo - i errCtx.LineNumber = lineNo
} }
if errCtx.Pos == -1 && i == 2 { if errCtx.Pos == -1 && i == 2 {
@ -171,7 +200,7 @@ func locateError(r io.Reader, le FileError, matches LineMatcher) ErrorContext {
if matches(le, lineNo, "") { if matches(le, lineNo, "") {
buff[i] = "" buff[i] = ""
errCtx.Pos = i errCtx.Pos = i
errCtx.LineNumber = lineNo - 1 errCtx.LineNumber = lineNo
i++ i++
} }

6
common/herrors/error_locator_test.go
View file

@ -41,7 +41,7 @@ LINE 8
location := locateErrorInString(nil, lines, lineMatcher) location := locateErrorInString(nil, lines, lineMatcher)
assert.Equal([]string{"LINE 3", "LINE 4", "This is THEONE", "LINE 6", "LINE 7"}, location.Lines) assert.Equal([]string{"LINE 3", "LINE 4", "This is THEONE", "LINE 6", "LINE 7"}, location.Lines)
assert.Equal(3, location.LineNumber) assert.Equal(5, location.LineNumber)
assert.Equal(2, location.Pos) assert.Equal(2, location.Pos)
assert.Equal([]string{"This is THEONE"}, locateErrorInString(nil, `This is THEONE`, lineMatcher).Lines) assert.Equal([]string{"This is THEONE"}, locateErrorInString(nil, `This is THEONE`, lineMatcher).Lines)
@ -92,7 +92,7 @@ I
J`, lineMatcher) J`, lineMatcher)
assert.Equal([]string{"D", "E", "F", "G", "H"}, location.Lines) assert.Equal([]string{"D", "E", "F", "G", "H"}, location.Lines)
assert.Equal(4, location.LineNumber) assert.Equal(6, location.LineNumber)
assert.Equal(2, location.Pos) assert.Equal(2, location.Pos)
// Test match EOF // Test match EOF
@ -106,7 +106,7 @@ C
`, lineMatcher) `, lineMatcher)
assert.Equal([]string{"B", "C", ""}, location.Lines) assert.Equal([]string{"B", "C", ""}, location.Lines)
assert.Equal(3, location.LineNumber) assert.Equal(4, location.LineNumber)
assert.Equal(2, location.Pos) assert.Equal(2, location.Pos)
} }

41
common/herrors/file_error.go
View file

@ -13,10 +13,6 @@
package herrors package herrors
import (
"fmt"
)
var _ causer = (*fileError)(nil) var _ causer = (*fileError)(nil)
// FileError represents an error when handling a file: Parsing a config file, // FileError represents an error when handling a file: Parsing a config file,
@ -27,6 +23,8 @@ type FileError interface {
// LineNumber gets the error location, starting at line 1. // LineNumber gets the error location, starting at line 1.
LineNumber() int LineNumber() int
ColumnNumber() int
// A string identifying the type of file, e.g. JSON, TOML, markdown etc. // A string identifying the type of file, e.g. JSON, TOML, markdown etc.
Type() string Type() string
} }
@ -34,9 +32,9 @@ type FileError interface {
var _ FileError = (*fileError)(nil) var _ FileError = (*fileError)(nil)
type fileError struct { type fileError struct {
lineNumber int lineNumber int
fileType string columnNumber int
msg string fileType string
cause error cause error
} }
@ -45,32 +43,28 @@ func (e *fileError) LineNumber() int {
return e.lineNumber return e.lineNumber
} }
func (e *fileError) ColumnNumber() int {
return e.columnNumber
}
func (e *fileError) Type() string { func (e *fileError) Type() string {
return e.fileType return e.fileType
} }
func (e *fileError) Error() string { func (e *fileError) Error() string {
return e.msg if e.cause == nil {
return ""
}
return e.cause.Error()
} }
func (f *fileError) Cause() error { func (f *fileError) Cause() error {
return f.cause return f.cause
} }
func (e *fileError) Format(s fmt.State, verb rune) {
switch verb {
case 'v':
fallthrough
case 's':
fmt.Fprintf(s, "%s:%d: %s:%s", e.fileType, e.lineNumber, e.msg, e.cause)
case 'q':
fmt.Fprintf(s, "%q:%d: %q:%q", e.fileType, e.lineNumber, e.msg, e.cause)
}
}
// NewFileError creates a new FileError. // NewFileError creates a new FileError.
func NewFileError(fileType string, lineNumber int, msg string, err error) FileError { func NewFileError(fileType string, lineNumber, columnNumber int, err error) FileError {
return &fileError{cause: err, fileType: fileType, lineNumber: lineNumber, msg: msg} return &fileError{cause: err, fileType: fileType, lineNumber: lineNumber, columnNumber: columnNumber}
} }
// UnwrapFileError tries to unwrap a FileError from err. // UnwrapFileError tries to unwrap a FileError from err.
@ -101,9 +95,10 @@ func ToFileError(fileType string, err error) error {
// If will fall back to returning the original error if a line number cannot be extracted. // If will fall back to returning the original error if a line number cannot be extracted.
func ToFileErrorWithOffset(fileType string, err error, offset int) error { func ToFileErrorWithOffset(fileType string, err error, offset int) error {
for _, handle := range lineNumberExtractors { for _, handle := range lineNumberExtractors {
lno, msg := handle(err, offset)
lno, col := handle(err)
if lno > 0 { if lno > 0 {
return NewFileError(fileType, lno, msg, err) return NewFileError(fileType, lno+offset, col, err)
} }
} }
// Fall back to the original. // Fall back to the original.

19
common/herrors/file_error_test.go
View file

@ -28,16 +28,16 @@ func TestToLineNumberError(t *testing.T) {
assert := require.New(t) assert := require.New(t)
for i, test := range []struct { for i, test := range []struct {
in error in error
offset int offset int
lineNumber int lineNumber int
columnNumber int
}{ }{
{errors.New("no line number for you"), 0, -1}, {errors.New("no line number for you"), 0, -1, 1},
{errors.New(`template: _default/single.html:2:15: executing "_default/single.html" at <.Titles>: can't evaluate field`), 0, 2}, {errors.New(`template: _default/single.html:4:15: executing "_default/single.html" at <.Titles>: can't evaluate field Titles in type *hugolib.PageOutput`), 0, 4, 15},
{errors.New("parse failed: template: _default/bundle-resource-meta.html:11: unexpected in operand"), 0, 11}, {errors.New("parse failed: template: _default/bundle-resource-meta.html:11: unexpected in operand"), 0, 11, 1},
{errors.New(`failed:: template: _default/bundle-resource-meta.html:2:7: executing "main" at <.Titles>`), 0, 2}, {errors.New(`failed:: template: _default/bundle-resource-meta.html:2:7: executing "main" at <.Titles>`), 0, 2, 7},
{errors.New("error in front matter: Near line 32 (last key parsed 'title')"), 0, 32}, {errors.New("error in front matter: Near line 32 (last key parsed 'title')"), 0, 32, 1},
{errors.New("error in front matter: Near line 32 (last key parsed 'title')"), 2, 34},
} { } {
got := ToFileErrorWithOffset("template", test.in, test.offset) got := ToFileErrorWithOffset("template", test.in, test.offset)
@ -48,6 +48,7 @@ func TestToLineNumberError(t *testing.T) {
if test.lineNumber > 0 { if test.lineNumber > 0 {
assert.True(ok) assert.True(ok)
assert.Equal(test.lineNumber, le.LineNumber(), errMsg) assert.Equal(test.lineNumber, le.LineNumber(), errMsg)
assert.Equal(test.columnNumber, le.ColumnNumber(), errMsg)
assert.Contains(got.Error(), strconv.Itoa(le.LineNumber())) assert.Contains(got.Error(), strconv.Itoa(le.LineNumber()))
} else { } else {
assert.False(ok) assert.False(ok)

27
common/herrors/line_number_extractors.go
View file

@ -14,14 +14,13 @@
package herrors package herrors
import ( import (
"fmt"
"regexp" "regexp"
"strconv" "strconv"
) )
var lineNumberExtractors = []lineNumberExtractor{ var lineNumberExtractors = []lineNumberExtractor{
// Template/shortcode parse errors // Template/shortcode parse errors
newLineNumberErrHandlerFromRegexp("(.*?:)(\\d+)(:.*)"), newLineNumberErrHandlerFromRegexp("(.*?:)(\\d+)(:)(\\d+)?(.*)"),
// TOML parse errors // TOML parse errors
newLineNumberErrHandlerFromRegexp("(.*Near line )(\\d+)(\\s.*)"), newLineNumberErrHandlerFromRegexp("(.*Near line )(\\d+)(\\s.*)"),
@ -30,7 +29,7 @@ var lineNumberExtractors = []lineNumberExtractor{
newLineNumberErrHandlerFromRegexp("(line )(\\d+)(:)"), newLineNumberErrHandlerFromRegexp("(line )(\\d+)(:)"),
} }
type lineNumberExtractor func(e error, offset int) (int, string) type lineNumberExtractor func(e error) (int, int)
func newLineNumberErrHandlerFromRegexp(expression string) lineNumberExtractor { func newLineNumberErrHandlerFromRegexp(expression string) lineNumberExtractor {
re := regexp.MustCompile(expression) re := regexp.MustCompile(expression)
@ -38,22 +37,26 @@ func newLineNumberErrHandlerFromRegexp(expression string) lineNumberExtractor {
} }
func extractLineNo(re *regexp.Regexp) lineNumberExtractor { func extractLineNo(re *regexp.Regexp) lineNumberExtractor {
return func(e error, offset int) (int, string) { return func(e error) (int, int) {
if e == nil { if e == nil {
panic("no error") panic("no error")
} }
col := 1
s := e.Error() s := e.Error()
m := re.FindStringSubmatch(s) m := re.FindStringSubmatch(s)
if len(m) == 4 { if len(m) >= 4 {
i, _ := strconv.Atoi(m[2]) lno, _ := strconv.Atoi(m[2])
msg := e.Error() if len(m) > 4 {
if offset != 0 { col, _ = strconv.Atoi(m[4])
i = i + offset
msg = re.ReplaceAllString(s, fmt.Sprintf("${1}%d${3}", i))
} }
return i, msg
if col <= 0 {
col = 1
}
return lno, col
} }
return -1, "" return -1, col
} }
} }

30
deps/deps.go vendored
View file

@ -5,7 +5,6 @@ import (
"time" "time"
"github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/hugofs"
@ -16,6 +15,7 @@ import (
"github.com/gohugoio/hugo/resource" "github.com/gohugoio/hugo/resource"
"github.com/gohugoio/hugo/source" "github.com/gohugoio/hugo/source"
"github.com/gohugoio/hugo/tpl" "github.com/gohugoio/hugo/tpl"
jww "github.com/spf13/jwalterweatherman"
) )
// Deps holds dependencies used by many. // Deps holds dependencies used by many.
@ -73,6 +73,33 @@ type Deps struct {
// BuildStartListeners will be notified before a build starts. // BuildStartListeners will be notified before a build starts.
BuildStartListeners *Listeners BuildStartListeners *Listeners
*globalErrHandler
}
type globalErrHandler struct {
// Channel for some "hard to get to" build errors
buildErrors chan error
}
// SendErr sends the error on a channel to be handled later.
// This can be used in situations where returning and aborting the current
// operation isn't practical.
func (e *globalErrHandler) SendError(err error) {
if e.buildErrors != nil {
select {
case e.buildErrors <- err:
default:
}
return
}
jww.ERROR.Println(err)
}
func (e *globalErrHandler) StartErrorCollector() chan error {
e.buildErrors = make(chan error, 10)
return e.buildErrors
} }
// Listeners represents an event listener. // Listeners represents an event listener.
@ -194,6 +221,7 @@ func New(cfg DepsCfg) (*Deps, error) {
Language: cfg.Language, Language: cfg.Language,
BuildStartListeners: &Listeners{}, BuildStartListeners: &Listeners{},
Timeout: time.Duration(timeoutms) * time.Millisecond, Timeout: time.Duration(timeoutms) * time.Millisecond,
globalErrHandler: &globalErrHandler{},
} }
if cfg.Cfg.GetBool("templateMetrics") { if cfg.Cfg.GetBool("templateMetrics") {

36
hugolib/hugo_sites.go
View file

@ -21,6 +21,7 @@ import (
"strings" "strings"
"sync" "sync"
"github.com/gohugoio/hugo/common/herrors"
"github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/deps"
"github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/helpers"
@ -53,6 +54,40 @@ type HugoSites struct {
gitInfo *gitInfo gitInfo *gitInfo
} }
func (h *HugoSites) pickOneAndLogTheRest(errors []error) error {
if len(errors) == 0 {
return nil
}
var i int
for j, err := range errors {
// If this is in server mode, we want to return an error to the client
// with a file context, if possible.
if herrors.UnwrapErrorWithFileContext(err) != nil {
i = j
break
}
}
// Log the rest, but add a threshold to avoid flooding the log.
const errLogThreshold = 5
for j, err := range errors {
if j == i || err == nil {
continue
}
if j >= errLogThreshold {
break
}
h.Log.ERROR.Println(err)
}
return errors[i]
}
func (h *HugoSites) IsMultihost() bool { func (h *HugoSites) IsMultihost() bool {
return h != nil && h.multihost return h != nil && h.multihost
} }
@ -636,6 +671,7 @@ func handleShortcodes(p *PageWithoutContent, rawContentCopy []byte) ([]byte, err
err := p.shortcodeState.executeShortcodesForDelta(p) err := p.shortcodeState.executeShortcodesForDelta(p)
if err != nil { if err != nil {
return rawContentCopy, err return rawContentCopy, err
} }

75
hugolib/hugo_sites_build.go
View file

@ -26,13 +26,29 @@ import (
// Build builds all sites. If filesystem events are provided, // Build builds all sites. If filesystem events are provided,
// this is considered to be a potential partial rebuild. // this is considered to be a potential partial rebuild.
func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error {
errCollector := h.StartErrorCollector()
errs := make(chan error)
go func(from, to chan error) {
var errors []error
i := 0
for e := range from {
i++
if i > 50 {
break
}
errors = append(errors, e)
}
to <- h.pickOneAndLogTheRest(errors)
close(to)
}(errCollector, errs)
if h.Metrics != nil { if h.Metrics != nil {
h.Metrics.Reset() h.Metrics.Reset()
} }
//t0 := time.Now()
// Need a pointer as this may be modified. // Need a pointer as this may be modified.
conf := &config conf := &config
@ -41,33 +57,46 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error {
conf.whatChanged = &whatChanged{source: true, other: true} conf.whatChanged = &whatChanged{source: true, other: true}
} }
var prepareErr error
if !config.PartialReRender { if !config.PartialReRender {
for _, s := range h.Sites { prepare := func() error {
s.Deps.BuildStartListeners.Notify() for _, s := range h.Sites {
} s.Deps.BuildStartListeners.Notify()
}
if len(events) > 0 { if len(events) > 0 {
// Rebuild // Rebuild
if err := h.initRebuild(conf); err != nil { if err := h.initRebuild(conf); err != nil {
return err
}
} else {
if err := h.init(conf); err != nil {
return err
}
}
if err := h.process(conf, events...); err != nil {
return err return err
} }
} else {
if err := h.init(conf); err != nil { if err := h.assemble(conf); err != nil {
return err return err
} }
return nil
} }
if err := h.process(conf, events...); err != nil { prepareErr = prepare()
return err if prepareErr != nil {
h.SendError(prepareErr)
} }
if err := h.assemble(conf); err != nil {
return err
}
} }
if err := h.render(conf); err != nil { if prepareErr == nil {
return err if err := h.render(conf); err != nil {
h.SendError(err)
}
} }
if h.Metrics != nil { if h.Metrics != nil {
@ -79,6 +108,18 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error {
h.Log.FEEDBACK.Println() h.Log.FEEDBACK.Println()
} }
select {
// Make sure the channel always gets something.
case errCollector <- nil:
default:
}
close(errCollector)
err := <-errs
if err != nil {
return err
}
errorCount := h.Log.ErrorCounter.Count() errorCount := h.Log.ErrorCounter.Count()
if errorCount > 0 { if errorCount > 0 {
return fmt.Errorf("logged %d error(s)", errorCount) return fmt.Errorf("logged %d error(s)", errorCount)

87
hugolib/hugo_sites_build_errors_test.go
View file

@ -2,6 +2,7 @@ package hugolib
import ( import (
"fmt" "fmt"
"path/filepath"
"strings" "strings"
"testing" "testing"
@ -17,13 +18,20 @@ type testSiteBuildErrorAsserter struct {
func (t testSiteBuildErrorAsserter) getFileError(err error) *herrors.ErrorWithFileContext { func (t testSiteBuildErrorAsserter) getFileError(err error) *herrors.ErrorWithFileContext {
t.assert.NotNil(err, t.name) t.assert.NotNil(err, t.name)
ferr := herrors.UnwrapErrorWithFileContext(err) ferr := herrors.UnwrapErrorWithFileContext(err)
t.assert.NotNil(ferr, fmt.Sprintf("[%s] got %T: %+v", t.name, err, err)) t.assert.NotNil(ferr, fmt.Sprintf("[%s] got %T: %+v\n%s", t.name, err, err, trace()))
return ferr return ferr
} }
func (t testSiteBuildErrorAsserter) assertLineNumber(lineNumber int, err error) { func (t testSiteBuildErrorAsserter) assertLineNumber(lineNumber int, err error) {
fe := t.getFileError(err) fe := t.getFileError(err)
t.assert.Equal(lineNumber, fe.LineNumber, fmt.Sprintf("[%s] got => %s", t.name, fe)) t.assert.Equal(lineNumber, fe.LineNumber, fmt.Sprintf("[%s] got => %s\n%s", t.name, fe, trace()))
}
func (t testSiteBuildErrorAsserter) assertErrorMessage(e1, e2 string) {
// The error message will contain filenames with OS slashes. Normalize before compare.
e1, e2 = filepath.ToSlash(e1), filepath.ToSlash(e2)
t.assert.Equal(e1, e2, trace())
} }
func TestSiteBuildErrors(t *testing.T) { func TestSiteBuildErrors(t *testing.T) {
@ -32,6 +40,7 @@ func TestSiteBuildErrors(t *testing.T) {
const ( const (
yamlcontent = "yamlcontent" yamlcontent = "yamlcontent"
tomlcontent = "tomlcontent"
shortcode = "shortcode" shortcode = "shortcode"
base = "base" base = "base"
single = "single" single = "single"
@ -55,7 +64,7 @@ func TestSiteBuildErrors(t *testing.T) {
return strings.Replace(content, ".Title }}", ".Title }", 1) return strings.Replace(content, ".Title }}", ".Title }", 1)
}, },
assertCreateError: func(a testSiteBuildErrorAsserter, err error) { assertCreateError: func(a testSiteBuildErrorAsserter, err error) {
a.assertLineNumber(2, err) a.assertLineNumber(4, err)
}, },
}, },
{ {
@ -65,7 +74,7 @@ func TestSiteBuildErrors(t *testing.T) {
return strings.Replace(content, ".Title", ".Titles", 1) return strings.Replace(content, ".Title", ".Titles", 1)
}, },
assertBuildError: func(a testSiteBuildErrorAsserter, err error) { assertBuildError: func(a testSiteBuildErrorAsserter, err error) {
a.assertLineNumber(2, err) a.assertLineNumber(4, err)
}, },
}, },
{ {
@ -75,7 +84,12 @@ func TestSiteBuildErrors(t *testing.T) {
return strings.Replace(content, ".Title }}", ".Title }", 1) return strings.Replace(content, ".Title }}", ".Title }", 1)
}, },
assertCreateError: func(a testSiteBuildErrorAsserter, err error) { assertCreateError: func(a testSiteBuildErrorAsserter, err error) {
a.assertLineNumber(3, err) fe := a.getFileError(err)
assert.Equal(5, fe.LineNumber)
assert.Equal(1, fe.ColumnNumber)
assert.Equal("go-html-template", fe.ChromaLexer)
a.assertErrorMessage("\"layouts/_default/single.html:5:1\": parse failed: template: _default/single.html:5: unexpected \"}\" in operand", fe.Error())
}, },
}, },
{ {
@ -85,7 +99,12 @@ func TestSiteBuildErrors(t *testing.T) {
return strings.Replace(content, ".Title", ".Titles", 1) return strings.Replace(content, ".Title", ".Titles", 1)
}, },
assertBuildError: func(a testSiteBuildErrorAsserter, err error) { assertBuildError: func(a testSiteBuildErrorAsserter, err error) {
a.assertLineNumber(3, err) fe := a.getFileError(err)
assert.Equal(5, fe.LineNumber)
assert.Equal(14, fe.ColumnNumber)
assert.Equal("md", fe.ChromaLexer)
a.assertErrorMessage("asdfadf", fe.Error())
}, },
}, },
{ {
@ -95,7 +114,7 @@ func TestSiteBuildErrors(t *testing.T) {
return strings.Replace(content, ".Title }}", ".Title }", 1) return strings.Replace(content, ".Title }}", ".Title }", 1)
}, },
assertCreateError: func(a testSiteBuildErrorAsserter, err error) { assertCreateError: func(a testSiteBuildErrorAsserter, err error) {
a.assertLineNumber(2, err) a.assertLineNumber(4, err)
}, },
}, },
{ {
@ -105,10 +124,47 @@ func TestSiteBuildErrors(t *testing.T) {
return strings.Replace(content, ".Title", ".Titles", 1) return strings.Replace(content, ".Title", ".Titles", 1)
}, },
assertBuildError: func(a testSiteBuildErrorAsserter, err error) { assertBuildError: func(a testSiteBuildErrorAsserter, err error) {
a.assertLineNumber(25, err) a.assertLineNumber(4, err)
}, },
}, },
{
name: "Shortode does not exist",
fileType: yamlcontent,
fileFixer: func(content string) string {
return strings.Replace(content, "{{< sc >}}", "{{< nono >}}", 1)
},
assertBuildError: func(a testSiteBuildErrorAsserter, err error) {
fe := a.getFileError(err)
assert.Equal(7, fe.LineNumber)
assert.Equal(14, fe.ColumnNumber)
assert.Equal("md", fe.ChromaLexer)
a.assertErrorMessage("\"content/myyaml.md:7:14\": failed to extract shortcode: template for shortcode \"nono\" not found", fe.Error())
},
},
{
name: "Invalid YAML front matter",
fileType: yamlcontent,
fileFixer: func(content string) string {
// TODO(bep) 2errors YAML line numbers seems to be off by one for > 1 line.
return strings.Replace(content, "title:", "title", 1)
},
assertBuildError: func(a testSiteBuildErrorAsserter, err error) {
a.assertLineNumber(2, err)
},
},
{
name: "Invalid TOML front matter",
fileType: tomlcontent,
fileFixer: func(content string) string {
return strings.Replace(content, "description = ", "description &", 1)
},
assertBuildError: func(a testSiteBuildErrorAsserter, err error) {
fe := a.getFileError(err)
assert.Equal(6, fe.LineNumber)
assert.Equal("toml", fe.ErrorContext.ChromaLexer)
},
},
{ {
name: "Panic in template Execute", name: "Panic in template Execute",
fileType: single, fileType: single,
@ -166,12 +222,25 @@ title: "The YAML"
Some content. Some content.
{{< sc >}} {{< sc >}}
Some more text. Some more text.
The end. The end.
`))
b.WithContent("mytoml.md", f(tomlcontent, `+++
title = "The TOML"
p1 = "v"
p2 = "v"
p3 = "v"
description = "Descriptioon"
+++
Some content.
`)) `))
createErr := b.CreateSitesE() createErr := b.CreateSitesE()

53
hugolib/page.go
View file

@ -20,11 +20,10 @@ import (
"fmt" "fmt"
"reflect" "reflect"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/media"
_errors "github.com/pkg/errors" _errors "github.com/pkg/errors"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/langs"
"github.com/gohugoio/hugo/related" "github.com/gohugoio/hugo/related"
@ -304,7 +303,7 @@ func (p *Page) initContent() {
if len(p.summary) == 0 { if len(p.summary) == 0 {
if err = p.setAutoSummary(); err != nil { if err = p.setAutoSummary(); err != nil {
err = _errors.Wrapf(err, "Failed to set user auto summary for page %q:", p.pathOrTitle()) err = p.errorf(err, "failed to set auto summary")
} }
} }
c <- err c <- err
@ -315,11 +314,11 @@ func (p *Page) initContent() {
p.s.Log.WARN.Printf("WARNING: Timed out creating content for page %q (.Content will be empty). This is most likely a circular shortcode content loop that should be fixed. If this is just a shortcode calling a slow remote service, try to set \"timeout=20000\" (or higher, value is in milliseconds) in config.toml.\n", p.pathOrTitle()) p.s.Log.WARN.Printf("WARNING: Timed out creating content for page %q (.Content will be empty). This is most likely a circular shortcode content loop that should be fixed. If this is just a shortcode calling a slow remote service, try to set \"timeout=20000\" (or higher, value is in milliseconds) in config.toml.\n", p.pathOrTitle())
case err := <-c: case err := <-c:
if err != nil { if err != nil {
// TODO(bep) 2errors needs to be transported to the caller. p.s.SendError(err)
p.s.Log.ERROR.Println(err)
} }
} }
}) })
} }
// This is sent to the shortcodes for this page. Not doing that will create an infinite regress. So, // This is sent to the shortcodes for this page. Not doing that will create an infinite regress. So,
@ -560,11 +559,6 @@ func (ps Pages) findPagePos(page *Page) int {
return -1 return -1
} }
func (p *Page) createWorkContentCopy() {
p.workContent = make([]byte, len(p.rawContent))
copy(p.workContent, p.rawContent)
}
func (p *Page) Plain() string { func (p *Page) Plain() string {
p.initContent() p.initContent()
p.initPlain(true) p.initPlain(true)
@ -697,12 +691,6 @@ func (p *Page) UniqueID() string {
return p.File.UniqueID() return p.File.UniqueID()
} }
// for logging
// TODO(bep) 2errors remove
func (p *Page) lineNumRawContentStart() int {
return bytes.Count(p.frontmatter, []byte("\n")) + 1
}
// Returns the page as summary and main. // Returns the page as summary and main.
func (p *Page) setUserDefinedSummary(rawContentCopy []byte) (*summaryContent, error) { func (p *Page) setUserDefinedSummary(rawContentCopy []byte) (*summaryContent, error) {
@ -936,31 +924,18 @@ func (s *Site) NewPage(name string) (*Page, error) {
return p, nil return p, nil
} }
func (p *Page) errorf(err error, format string, a ...interface{}) error {
args := append([]interface{}{p.Lang(), p.pathOrTitle()}, a...)
format = "[%s] Page %q: " + format
if err == nil {
return fmt.Errorf(format, args...)
}
return _errors.Wrapf(err, format, args...)
}
func (p *Page) ReadFrom(buf io.Reader) (int64, error) { func (p *Page) ReadFrom(buf io.Reader) (int64, error) {
// Parse for metadata & body // Parse for metadata & body
if err := p.parse(buf); err != nil { if err := p.parse(buf); err != nil {
return 0, p.errorf(err, "parse failed") return 0, p.errWithFileContext(err)
} }
// Work on a copy of the raw content from now on.
// TODO(bep) 2errors
//p.createWorkContentCopy()
if err := p.mapContent(); err != nil { if err := p.mapContent(); err != nil {
return 0, err return 0, p.errWithFileContext(err)
} }
return int64(len(p.rawContent)), nil return int64(len(p.source.parsed.Input())), nil
} }
func (p *Page) WordCount() int { func (p *Page) WordCount() int {
@ -1169,7 +1144,7 @@ func (p *Page) initMainOutputFormat() error {
pageOutput, err := newPageOutput(p, false, false, outFormat) pageOutput, err := newPageOutput(p, false, false, outFormat)
if err != nil { if err != nil {
return _errors.Wrapf(err, "Failed to create output page for type %q for page %q:", outFormat.Name, p.pathOrTitle()) return p.errorf(err, "failed to create output page for type %q", outFormat.Name)
} }
p.mainPageOutput = pageOutput p.mainPageOutput = pageOutput
@ -1485,7 +1460,7 @@ func (p *Page) updateMetaData(frontmatter map[string]interface{}) error {
if isCJKLanguage != nil { if isCJKLanguage != nil {
p.isCJKLanguage = *isCJKLanguage p.isCJKLanguage = *isCJKLanguage
} else if p.s.Cfg.GetBool("hasCJKLanguage") { } else if p.s.Cfg.GetBool("hasCJKLanguage") {
if cjk.Match(p.rawContent) { if cjk.Match(p.source.parsed.Input()) {
p.isCJKLanguage = true p.isCJKLanguage = true
} else { } else {
p.isCJKLanguage = false p.isCJKLanguage = false
@ -1711,7 +1686,8 @@ func (p *Page) shouldRenderTo(f output.Format) bool {
} }
func (p *Page) RawContent() string { func (p *Page) RawContent() string {
return string(p.rawContent) // TODO(bep) 2errors
return string(p.source.parsed.Input())
} }
func (p *Page) FullFilePath() string { func (p *Page) FullFilePath() string {
@ -2145,12 +2121,7 @@ func (p *Page) setValuesForKind(s *Site) {
// Used in error logs. // Used in error logs.
func (p *Page) pathOrTitle() string { func (p *Page) pathOrTitle() string {
if p.Filename() != "" { if p.Filename() != "" {
// Make a path relative to the working dir if possible. return p.Filename()
filename := strings.TrimPrefix(p.Filename(), p.s.WorkingDir)
if filename != p.Filename() {
filename = strings.TrimPrefix(filename, helpers.FilePathSeparator)
}
return filename
} }
return p.title return p.title
} }

35
hugolib/page_content.go
View file

@ -14,11 +14,14 @@
package hugolib package hugolib
import ( import (
"fmt" "bytes"
"io" "io"
errors "github.com/pkg/errors"
bp "github.com/gohugoio/hugo/bufferpool" bp "github.com/gohugoio/hugo/bufferpool"
"github.com/gohugoio/hugo/common/herrors"
"github.com/gohugoio/hugo/parser/metadecoders" "github.com/gohugoio/hugo/parser/metadecoders"
"github.com/gohugoio/hugo/parser/pageparser" "github.com/gohugoio/hugo/parser/pageparser"
) )
@ -31,11 +34,6 @@ var (
type pageContent struct { type pageContent struct {
renderable bool renderable bool
frontmatter []byte
// rawContent is the raw content read from the content file.
rawContent []byte
// workContent is a copy of rawContent that may be mutated during site build. // workContent is a copy of rawContent that may be mutated during site build.
workContent []byte workContent []byte
@ -66,6 +64,10 @@ func (p *Page) mapContent() error {
iter := p.source.parsed.Iterator() iter := p.source.parsed.Iterator()
fail := func(err error, i pageparser.Item) error {
return parseError(err, iter.Input(), i.Pos)
}
// the parser is guaranteed to return items in proper order or fail, so … // the parser is guaranteed to return items in proper order or fail, so …
// … it's safe to keep some "global" state // … it's safe to keep some "global" state
var currShortcode shortcode var currShortcode shortcode
@ -87,7 +89,7 @@ Loop:
f := metadecoders.FormatFromFrontMatterType(it.Type) f := metadecoders.FormatFromFrontMatterType(it.Type)
m, err := metadecoders.UnmarshalToMap(it.Val, f) m, err := metadecoders.UnmarshalToMap(it.Val, f)
if err != nil { if err != nil {
return err return herrors.ToFileErrorWithOffset(string(f), err, iter.LineNumber()-1)
} }
if err := p.updateMetaData(m); err != nil { if err := p.updateMetaData(m); err != nil {
return err return err
@ -125,7 +127,7 @@ Loop:
} }
if err != nil { if err != nil {
return err return fail(errors.Wrap(err, "failed to extract shortcode"), it)
} }
if currShortcode.params == nil { if currShortcode.params == nil {
@ -139,10 +141,10 @@ Loop:
case it.IsEOF(): case it.IsEOF():
break Loop break Loop
case it.IsError(): case it.IsError():
err := fmt.Errorf("%s:shortcode:%d: %s", err := fail(errors.WithStack(errors.New(it.ValStr())), it)
p.pathOrTitle(), iter.LineNumber(), it)
currShortcode.err = err currShortcode.err = err
return err return err
default: default:
result.Write(it.Val) result.Write(it.Val)
} }
@ -180,3 +182,16 @@ func (p *Page) parse(reader io.Reader) error {
return nil return nil
} }
func parseError(err error, input []byte, pos int) error {
if herrors.UnwrapFileError(err) != nil {
// Use the most specific location.
return err
}
lf := []byte("\n")
input = input[:pos]
lineNumber := bytes.Count(input, lf) + 1
endOfLastLine := bytes.LastIndex(input, lf)
return herrors.NewFileError("md", lineNumber, pos-endOfLastLine, err)
}

47
hugolib/page_errors.go Normal file
View file

@ -0,0 +1,47 @@
// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package hugolib
import (
"fmt"
"github.com/gohugoio/hugo/common/herrors"
errors "github.com/pkg/errors"
)
func (p *Page) errorf(err error, format string, a ...interface{}) error {
if herrors.UnwrapErrorWithFileContext(err) != nil {
// More isn't always better.
return err
}
args := append([]interface{}{p.Lang(), p.pathOrTitle()}, a...)
format = "[%s] page %q: " + format
if err == nil {
errors.Errorf(format, args...)
return fmt.Errorf(format, args...)
}
return errors.Wrapf(err, format, args...)
}
func (p *Page) errWithFileContext(err error) error {
err, _ = herrors.WithFileContextForFile(
err,
p.Filename(),
p.Filename(),
p.s.SourceSpec.Fs.Source,
herrors.SimpleLineMatcher)
return err
}

30
hugolib/shortcode.go
View file

@ -18,7 +18,9 @@ import (
"errors" "errors"
"fmt" "fmt"
"html/template" "html/template"
"reflect" "reflect"
"regexp" "regexp"
"sort" "sort"
@ -139,6 +141,7 @@ type shortcode struct {
ordinal int ordinal int
err error err error
doMarkup bool doMarkup bool
pos int // the position in bytes in the source file
} }
func (sc shortcode) String() string { func (sc shortcode) String() string {
@ -458,7 +461,13 @@ func (s *shortcodeHandler) executeShortcodesForDelta(p *PageWithoutContent) erro
render := s.contentShortcodesDelta.getShortcodeRenderer(k) render := s.contentShortcodesDelta.getShortcodeRenderer(k)
renderedShortcode, err := render() renderedShortcode, err := render()
if err != nil { if err != nil {
return _errors.Wrapf(err, "Failed to execute shortcode in page %q:", p.Path()) sc := s.shortcodes.getShortcode(k.(scKey).ShortcodePlaceholder)
if sc != nil {
err = p.errWithFileContext(parseError(_errors.Wrapf(err, "failed to render shortcode %q", sc.name), p.source.parsed.Input(), sc.pos))
}
p.s.SendError(err)
continue
} }
s.renderedShortcodes[k.(scKey).ShortcodePlaceholder] = renderedShortcode s.renderedShortcodes[k.(scKey).ShortcodePlaceholder] = renderedShortcode
@ -495,15 +504,8 @@ func (s *shortcodeHandler) extractShortcode(ordinal int, pt *pageparser.Iterator
var cnt = 0 var cnt = 0
var nestedOrdinal = 0 var nestedOrdinal = 0
// TODO(bep) 2errors revisit after https://github.com/gohugoio/hugo/issues/5324 fail := func(err error, i pageparser.Item) error {
msgf := func(i pageparser.Item, format string, args ...interface{}) string { return parseError(err, pt.Input(), i.Pos)
format = format + ":%d:"
// TODO(bep) 2errors
c1 := 32 // strings.Count(pt.lexer.input[:i.pos], "\n") + 1
c2 := bytes.Count(p.frontmatter, []byte{'\n'})
args = append(args, c1+c2)
return fmt.Sprintf(format, args...)
} }
Loop: Loop:
@ -511,6 +513,7 @@ Loop:
currItem := pt.Next() currItem := pt.Next()
switch { switch {
case currItem.IsLeftShortcodeDelim(): case currItem.IsLeftShortcodeDelim():
sc.pos = currItem.Pos
next := pt.Peek() next := pt.Peek()
if next.IsShortcodeClose() { if next.IsShortcodeClose() {
continue continue
@ -550,7 +553,8 @@ Loop:
// return that error, more specific // return that error, more specific
continue continue
} }
return sc, errors.New(msgf(next, "shortcode %q has no .Inner, yet a closing tag was provided", next.Val))
return sc, fail(_errors.Errorf("shortcode %q has no .Inner, yet a closing tag was provided", next.Val), next)
} }
if next.IsRightShortcodeDelim() { if next.IsRightShortcodeDelim() {
// self-closing // self-closing
@ -568,13 +572,13 @@ Loop:
// if more than one. It is "all inner or no inner". // if more than one. It is "all inner or no inner".
tmpl := getShortcodeTemplateForTemplateKey(scKey{}, sc.name, p.s.Tmpl) tmpl := getShortcodeTemplateForTemplateKey(scKey{}, sc.name, p.s.Tmpl)
if tmpl == nil { if tmpl == nil {
return sc, errors.New(msgf(currItem, "unable to locate template for shortcode %q", sc.name)) return sc, fail(_errors.Errorf("template for shortcode %q not found", sc.name), currItem)
} }
var err error var err error
isInner, err = isInnerShortcode(tmpl.(tpl.TemplateExecutor)) isInner, err = isInnerShortcode(tmpl.(tpl.TemplateExecutor))
if err != nil { if err != nil {
return sc, _errors.Wrap(err, msgf(currItem, "failed to handle template for shortcode %q", sc.name)) return sc, fail(_errors.Wrapf(err, "failed to handle template for shortcode %q", sc.name), currItem)
} }
case currItem.IsShortcodeParam(): case currItem.IsShortcodeParam():

39
hugolib/site.go
View file

@ -30,7 +30,6 @@ import (
_errors "github.com/pkg/errors" _errors "github.com/pkg/errors"
"github.com/gohugoio/hugo/common/herrors"
"github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/publisher" "github.com/gohugoio/hugo/publisher"
"github.com/gohugoio/hugo/resource" "github.com/gohugoio/hugo/resource"
@ -1552,7 +1551,7 @@ func (s *Site) preparePages() error {
} }
} }
return s.pickOneAndLogTheRest(errors) return s.owner.pickOneAndLogTheRest(errors)
} }
func (s *Site) errorCollator(results <-chan error, errs chan<- error) { func (s *Site) errorCollator(results <-chan error, errs chan<- error) {
@ -1561,45 +1560,11 @@ func (s *Site) errorCollator(results <-chan error, errs chan<- error) {
errors = append(errors, e) errors = append(errors, e)
} }
errs <- s.pickOneAndLogTheRest(errors) errs <- s.owner.pickOneAndLogTheRest(errors)
close(errs) close(errs)
} }
func (s *Site) pickOneAndLogTheRest(errors []error) error {
if len(errors) == 0 {
return nil
}
var i int
for j, err := range errors {
// If this is in server mode, we want to return an error to the client
// with a file context, if possible.
if herrors.UnwrapErrorWithFileContext(err) != nil {
i = j
break
}
}
// Log the rest, but add a threshold to avoid flooding the log.
const errLogThreshold = 5
for j, err := range errors {
if j == i {
continue
}
if j >= errLogThreshold {
break
}
s.Log.ERROR.Println(err)
}
return errors[i]
}
func (s *Site) appendThemeTemplates(in []string) []string { func (s *Site) appendThemeTemplates(in []string) []string {
if !s.PathSpec.ThemeSet() { if !s.PathSpec.ThemeSet() {
return in return in

6
hugolib/testhelpers_test.go
View file

@ -465,12 +465,16 @@ func (s *sitesBuilder) Fatalf(format string, args ...interface{}) {
} }
func Fatalf(t testing.TB, format string, args ...interface{}) { func Fatalf(t testing.TB, format string, args ...interface{}) {
trace := strings.Join(assert.CallerInfo(), "\n\r\t\t\t") trace := trace()
format = format + "\n%s" format = format + "\n%s"
args = append(args, trace) args = append(args, trace)
t.Fatalf(format, args...) t.Fatalf(format, args...)
} }
func trace() string {
return strings.Join(assert.CallerInfo(), "\n\r\t\t\t")
}
func (s *sitesBuilder) AssertFileContent(filename string, matches ...string) { func (s *sitesBuilder) AssertFileContent(filename string, matches ...string) {
content := readDestination(s.T, s.Fs, filename) content := readDestination(s.T, s.Fs, filename)
for _, match := range matches { for _, match := range matches {

17
parser/pageparser/pagelexer.go
View file

@ -408,15 +408,22 @@ func (l *pageLexer) lexFrontMatterSection(tp ItemType, delimr rune, name string,
} }
} }
// Let front matter start at line 1
wasEndOfLine := l.consumeCRLF()
// We don't care about the delimiters. // We don't care about the delimiters.
l.ignore() l.ignore()
var r rune
for { for {
r := l.next() if !wasEndOfLine {
if r == eof { r = l.next()
return l.errorf("EOF looking for end %s front matter delimiter", name) if r == eof {
return l.errorf("EOF looking for end %s front matter delimiter", name)
}
} }
if isEndOfLine(r) {
if wasEndOfLine || isEndOfLine(r) {
if l.hasPrefix(delim) { if l.hasPrefix(delim) {
l.emit(tp) l.emit(tp)
l.pos += 3 l.pos += 3
@ -425,6 +432,8 @@ func (l *pageLexer) lexFrontMatterSection(tp ItemType, delimr rune, name string,
break break
} }
} }
wasEndOfLine = false
} }
return lexMainSection return lexMainSection

5
parser/pageparser/pageparser.go
View file

@ -66,6 +66,11 @@ func (t *Iterator) Next() Item {
return t.current() return t.current()
} }
// Input returns the input source.
func (t *Iterator) Input() []byte {
return t.l.Input()
}
var errIndexOutOfBounds = Item{tError, 0, []byte("no more tokens")} var errIndexOutOfBounds = Item{tError, 0, []byte("no more tokens")}
func (t *Iterator) current() Item { func (t *Iterator) current() Item {

8
parser/pageparser/pageparser_intro_test.go
View file

@ -32,9 +32,9 @@ func nti(tp ItemType, val string) Item {
var ( var (
tstJSON = `{ "a": { "b": "\"Hugo\"}" } }` tstJSON = `{ "a": { "b": "\"Hugo\"}" } }`
tstFrontMatterTOML = nti(TypeFrontMatterTOML, "\nfoo = \"bar\"\n") tstFrontMatterTOML = nti(TypeFrontMatterTOML, "foo = \"bar\"\n")
tstFrontMatterYAML = nti(TypeFrontMatterYAML, "\nfoo: \"bar\"\n") tstFrontMatterYAML = nti(TypeFrontMatterYAML, "foo: \"bar\"\n")
tstFrontMatterYAMLCRLF = nti(TypeFrontMatterYAML, "\r\nfoo: \"bar\"\r\n") tstFrontMatterYAMLCRLF = nti(TypeFrontMatterYAML, "foo: \"bar\"\r\n")
tstFrontMatterJSON = nti(TypeFrontMatterJSON, tstJSON+"\r\n") tstFrontMatterJSON = nti(TypeFrontMatterJSON, tstJSON+"\r\n")
tstSomeText = nti(tText, "\nSome text.\n") tstSomeText = nti(tText, "\nSome text.\n")
tstSummaryDivider = nti(TypeLeadSummaryDivider, "<!--more-->") tstSummaryDivider = nti(TypeLeadSummaryDivider, "<!--more-->")
@ -58,7 +58,7 @@ var frontMatterTests = []lexerTest{
{"HTML Document 2", `<html><h1>Hugo Rocks</h1></html>`, []Item{nti(TypeHTMLDocument, "<html><h1>Hugo Rocks</h1></html>"), tstEOF}}, {"HTML Document 2", `<html><h1>Hugo Rocks</h1></html>`, []Item{nti(TypeHTMLDocument, "<html><h1>Hugo Rocks</h1></html>"), tstEOF}},
{"No front matter", "\nSome text.\n", []Item{tstSomeText, tstEOF}}, {"No front matter", "\nSome text.\n", []Item{tstSomeText, tstEOF}},
{"YAML front matter", "---\nfoo: \"bar\"\n---\n\nSome text.\n", []Item{tstFrontMatterYAML, tstSomeText, tstEOF}}, {"YAML front matter", "---\nfoo: \"bar\"\n---\n\nSome text.\n", []Item{tstFrontMatterYAML, tstSomeText, tstEOF}},
{"YAML empty front matter", "---\n---\n\nSome text.\n", []Item{nti(TypeFrontMatterYAML, "\n"), tstSomeText, tstEOF}}, {"YAML empty front matter", "---\n---\n\nSome text.\n", []Item{nti(TypeFrontMatterYAML, ""), tstSomeText, tstEOF}},
{"YAML commented out front matter", "<!--\n---\nfoo: \"bar\"\n---\n-->\nSome text.\n", []Item{nti(TypeHTMLComment, "<!--\n---\nfoo: \"bar\"\n---\n-->"), tstSomeText, tstEOF}}, {"YAML commented out front matter", "<!--\n---\nfoo: \"bar\"\n---\n-->\nSome text.\n", []Item{nti(TypeHTMLComment, "<!--\n---\nfoo: \"bar\"\n---\n-->"), tstSomeText, tstEOF}},
// Note that we keep all bytes as they are, but we need to handle CRLF // Note that we keep all bytes as they are, but we need to handle CRLF
{"YAML front matter CRLF", "---\r\nfoo: \"bar\"\r\n---\n\nSome text.\n", []Item{tstFrontMatterYAMLCRLF, tstSomeText, tstEOF}}, {"YAML front matter CRLF", "---\r\nfoo: \"bar\"\r\n---\n\nSome text.\n", []Item{tstFrontMatterYAMLCRLF, tstSomeText, tstEOF}},

24
tpl/data/data.go
View file

@ -59,7 +59,7 @@ func (ns *Namespace) GetCSV(sep string, urlParts ...string) (d [][]string, err e
var req *http.Request var req *http.Request
req, err = http.NewRequest("GET", url, nil) req, err = http.NewRequest("GET", url, nil)
if err != nil { if err != nil {
return nil, _errors.Wrapf(err, "Failed to create request for getCSV for resource %s:", url) return nil, _errors.Wrapf(err, "failed to create request for getCSV for resource %s", url)
} }
req.Header.Add("Accept", "text/csv") req.Header.Add("Accept", "text/csv")
@ -68,28 +68,22 @@ func (ns *Namespace) GetCSV(sep string, urlParts ...string) (d [][]string, err e
var c []byte var c []byte
c, err = ns.getResource(req) c, err = ns.getResource(req)
if err != nil { if err != nil {
ns.deps.Log.ERROR.Printf("Failed to read CSV resource %q: %s", url, err) return nil, _errors.Wrapf(err, "failed to read CSV resource %q", url)
return nil, nil
} }
if !bytes.Contains(c, []byte(sep)) { if !bytes.Contains(c, []byte(sep)) {
ns.deps.Log.ERROR.Printf("Cannot find separator %s in CSV for %s", sep, url) return nil, _errors.Errorf("cannot find separator %s in CSV for %s", sep, url)
return nil, nil
} }
if d, err = parseCSV(c, sep); err != nil { if d, err = parseCSV(c, sep); err != nil {
ns.deps.Log.WARN.Printf("Failed to parse CSV file %s: %s", url, err) err = _errors.Wrapf(err, "failed to parse CSV file %s", url)
clearCacheSleep(i, url) clearCacheSleep(i, url)
continue continue
} }
break break
} }
if err != nil {
ns.deps.Log.ERROR.Printf("Failed to read CSV resource %q: %s", url, err)
return nil, nil
}
return return
} }
@ -103,7 +97,7 @@ func (ns *Namespace) GetJSON(urlParts ...string) (v interface{}, err error) {
var req *http.Request var req *http.Request
req, err = http.NewRequest("GET", url, nil) req, err = http.NewRequest("GET", url, nil)
if err != nil { if err != nil {
return nil, _errors.Wrapf(err, "Failed to create request for getJSON resource %s:", url) return nil, _errors.Wrapf(err, "Failed to create request for getJSON resource %s", url)
} }
req.Header.Add("Accept", "application/json") req.Header.Add("Accept", "application/json")
@ -111,10 +105,8 @@ func (ns *Namespace) GetJSON(urlParts ...string) (v interface{}, err error) {
var c []byte var c []byte
c, err = ns.getResource(req) c, err = ns.getResource(req)
if err != nil { if err != nil {
ns.deps.Log.ERROR.Printf("Failed to get JSON resource %s: %s", url, err) return nil, _errors.Wrapf(err, "failed to get getJSON resource %q", url)
return nil, nil
} }
err = json.Unmarshal(c, &v) err = json.Unmarshal(c, &v)
if err != nil { if err != nil {
ns.deps.Log.WARN.Printf("Cannot read JSON from resource %s: %s", url, err) ns.deps.Log.WARN.Printf("Cannot read JSON from resource %s: %s", url, err)
@ -127,7 +119,7 @@ func (ns *Namespace) GetJSON(urlParts ...string) (v interface{}, err error) {
} }
if err != nil { if err != nil {
ns.deps.Log.ERROR.Printf("Failed to get JSON resource %s: %s", url, err) return nil, _errors.Wrapf(err, "failed to get getJSON resource %q", url)
return nil, nil return nil, nil
} }
return return

21
tpl/data/data_test.go
View file

@ -21,8 +21,6 @@ import (
"strings" "strings"
"testing" "testing"
jww "github.com/spf13/jwalterweatherman"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@ -110,13 +108,13 @@ func TestGetCSV(t *testing.T) {
// Get on with it // Get on with it
got, err := ns.GetCSV(test.sep, test.url) got, err := ns.GetCSV(test.sep, test.url)
require.NoError(t, err, msg)
if _, ok := test.expect.(bool); ok { if _, ok := test.expect.(bool); ok {
require.Equal(t, 1, int(ns.deps.Log.ErrorCounter.Count())) require.Error(t, err, msg)
require.Nil(t, got) require.Nil(t, got)
continue continue
} }
require.NoError(t, err, msg)
require.Equal(t, 0, int(ns.deps.Log.ErrorCounter.Count())) require.Equal(t, 0, int(ns.deps.Log.ErrorCounter.Count()))
require.NotNil(t, got, msg) require.NotNil(t, got, msg)
@ -140,12 +138,12 @@ func TestGetJSON(t *testing.T) {
{ {
`http://malformed/`, `http://malformed/`,
`{gomeetup:["Sydney","San Francisco","Stockholm"]}`, `{gomeetup:["Sydney","San Francisco","Stockholm"]}`,
jww.LevelError, false,
}, },
{ {
`http://nofound/404`, `http://nofound/404`,
``, ``,
jww.LevelError, false,
}, },
// Locals // Locals
{ {
@ -156,7 +154,7 @@ func TestGetJSON(t *testing.T) {
{ {
"fail/no-file", "fail/no-file",
"", "",
jww.LevelError, false,
}, },
} { } {
@ -198,13 +196,6 @@ func TestGetJSON(t *testing.T) {
continue continue
} }
if errLevel, ok := test.expect.(jww.Threshold); ok && errLevel >= jww.LevelError {
logCount := ns.deps.Log.ErrorCounter.Count()
require.True(t, logCount >= 1, fmt.Sprintf("got log count %d", logCount))
continue
}
require.NoError(t, err, msg)
require.Equal(t, 0, int(ns.deps.Log.ErrorCounter.Count()), msg) require.Equal(t, 0, int(ns.deps.Log.ErrorCounter.Count()), msg)
require.NotNil(t, got, msg) require.NotNil(t, got, msg)

19
tpl/template.go
View file

@ -145,15 +145,20 @@ func (t *TemplateAdapter) extractIdentifiers(line string) []string {
} }
func (t *TemplateAdapter) addFileContext(name string, inerr error) error { func (t *TemplateAdapter) addFileContext(name string, inerr error) error {
if strings.HasPrefix(t.Name(), "_internal") {
return inerr
}
f, realFilename, err := t.fileAndFilename(t.Name()) f, realFilename, err := t.fileAndFilename(t.Name())
if err != nil { if err != nil {
return err return inerr
} }
defer f.Close() defer f.Close()
master, hasMaster := t.NameBaseTemplateName[name] master, hasMaster := t.NameBaseTemplateName[name]
ferr1 := errors.Wrapf(inerr, "execute of template %q failed", realFilename) ferr := errors.Wrap(inerr, "execute of template failed")
// Since this can be a composite of multiple template files (single.html + baseof.html etc.) // Since this can be a composite of multiple template files (single.html + baseof.html etc.)
// we potentially need to look in both -- and cannot rely on line number alone. // we potentially need to look in both -- and cannot rely on line number alone.
@ -174,9 +179,8 @@ func (t *TemplateAdapter) addFileContext(name string, inerr error) error {
} }
return false return false
} }
// TODO(bep) 2errors text vs HTML // TODO(bep) 2errors text vs HTML
fe, ok := herrors.WithFileContext(ferr1, f, "go-html-template", lineMatcher) fe, ok := herrors.WithFileContext(ferr, realFilename, f, lineMatcher)
if ok || !hasMaster { if ok || !hasMaster {
return fe return fe
} }
@ -188,12 +192,11 @@ func (t *TemplateAdapter) addFileContext(name string, inerr error) error {
} }
defer f.Close() defer f.Close()
ferr2 := errors.Wrapf(inerr, "execute of template %q failed", realFilename) fe, ok = herrors.WithFileContext(ferr, realFilename, f, lineMatcher)
fe, ok = herrors.WithFileContext(ferr2, f, "go-html-template", lineMatcher)
if !ok { if !ok {
// Return the most specific. // Return the most specific.
return ferr1 return ferr
} }
return fe return fe
@ -206,7 +209,7 @@ func (t *TemplateAdapter) fileAndFilename(name string) (afero.File, string, erro
fi, err := fs.Stat(filename) fi, err := fs.Stat(filename)
if err != nil { if err != nil {
return nil, "", errors.Wrapf(err, "failed to Stat %q", filename) return nil, "", err
} }
f, err := fs.Open(filename) f, err := fs.Open(filename)
if err != nil { if err != nil {

4
tpl/tplimpl/template_errors.go
View file

@ -33,13 +33,13 @@ type templateInfo struct {
} }
func (info templateInfo) errWithFileContext(what string, err error) error { func (info templateInfo) errWithFileContext(what string, err error) error {
err = errors.Wrapf(err, "file %q: %s:", info.realFilename, what) err = errors.Wrapf(err, what)
err, _ = herrors.WithFileContextForFile( err, _ = herrors.WithFileContextForFile(
err, err,
info.realFilename,
info.filename, info.filename,
info.fs, info.fs,
"go-html-template",
herrors.SimpleLineMatcher) herrors.SimpleLineMatcher)
return err return err