parser/metadecoders: Consolidate the metadata decoders

See #5324

commands/convert.go

@@ -14,8 +14,19 @@
 package commands
 
 import (
+	"bytes"
+	"fmt"
+	"strings"
 	"time"
 
+	"github.com/gohugoio/hugo/hugofs"
+
+	"github.com/gohugoio/hugo/helpers"
+
+	"github.com/gohugoio/hugo/parser"
+	"github.com/gohugoio/hugo/parser/metadecoders"
+	"github.com/gohugoio/hugo/parser/pageparser"
+
 	src "github.com/gohugoio/hugo/source"
 	"github.com/pkg/errors"
 
@@ -23,7 +34,6 @@ import (
 
 	"path/filepath"
 
-	"github.com/gohugoio/hugo/parser"
 	"github.com/spf13/cast"
 	"github.com/spf13/cobra"
 )
@@ -60,7 +70,7 @@ See convert's subcommands toJSON, toTOML and toYAML for more information.`,
 			Long: `toJSON converts all front matter in the content directory
 to use JSON for the front matter.`,
 			RunE: func(cmd *cobra.Command, args []string) error {
-				return cc.convertContents(rune([]byte(parser.JSONLead)[0]))
+				return cc.convertContents(metadecoders.JSON)
 			},
 		},
 		&cobra.Command{
@@ -69,7 +79,7 @@ to use JSON for the front matter.`,
 			Long: `toTOML converts all front matter in the content directory
 to use TOML for the front matter.`,
 			RunE: func(cmd *cobra.Command, args []string) error {
-				return cc.convertContents(rune([]byte(parser.TOMLLead)[0]))
+				return cc.convertContents(metadecoders.TOML)
 			},
 		},
 		&cobra.Command{
@@ -78,7 +88,7 @@ to use TOML for the front matter.`,
 			Long: `toYAML converts all front matter in the content directory
 to use YAML for the front matter.`,
 			RunE: func(cmd *cobra.Command, args []string) error {
-				return cc.convertContents(rune([]byte(parser.YAMLLead)[0]))
+				return cc.convertContents(metadecoders.YAML)
 			},
 		},
 	)
@@ -91,7 +101,7 @@ to use YAML for the front matter.`,
 	return cc
 }
 
-func (cc *convertCmd) convertContents(mark rune) error {
+func (cc *convertCmd) convertContents(format metadecoders.Format) error {
 	if cc.outputDir == "" && !cc.unsafe {
 		return newUserError("Unsafe operation not allowed, use --unsafe or set a different output path")
 	}
@@ -114,17 +124,17 @@ func (cc *convertCmd) convertContents(mark rune) error {
 
 	site.Log.FEEDBACK.Println("processing", len(site.AllPages), "content files")
 	for _, p := range site.AllPages {
-		if err := cc.convertAndSavePage(p, site, mark); err != nil {
+		if err := cc.convertAndSavePage(p, site, format); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func (cc *convertCmd) convertAndSavePage(p *hugolib.Page, site *hugolib.Site, mark rune) error {
+func (cc *convertCmd) convertAndSavePage(p *hugolib.Page, site *hugolib.Site, targetFormat metadecoders.Format) error {
 	// The resources are not in .Site.AllPages.
 	for _, r := range p.Resources.ByType("page") {
-		if err := cc.convertAndSavePage(r.(*hugolib.Page), site, mark); err != nil {
+		if err := cc.convertAndSavePage(r.(*hugolib.Page), site, targetFormat); err != nil {
 			return err
 		}
 	}
@@ -134,37 +144,56 @@ func (cc *convertCmd) convertAndSavePage(p *hugolib.Page, site *hugolib.Site, ma
 		return nil
 	}
 
+	errMsg := fmt.Errorf("Error processing file %q", p.Path())
+
 	site.Log.INFO.Println("Attempting to convert", p.LogicalName())
-	newPage, err := site.NewPage(p.LogicalName())
-	if err != nil {
-		return err
-	}
 
 	f, _ := p.File.(src.ReadableFile)
 	file, err := f.Open()
 	if err != nil {
-		site.Log.ERROR.Println("Error reading file:", p.Path())
+		site.Log.ERROR.Println(errMsg)
 		file.Close()
 		return nil
 	}
 
-	psr, err := parser.ReadFrom(file)
+	psr, err := pageparser.Parse(file)
 	if err != nil {
-		site.Log.ERROR.Println("Error processing file:", p.Path())
+		site.Log.ERROR.Println(errMsg)
 		file.Close()
 		return err
 	}
 
 	file.Close()
 
-	metadata, err := psr.Metadata()
+	var sourceFormat, sourceContent []byte
+	var fromFormat metadecoders.Format
+
+	iter := psr.Iterator()
+
+	walkFn := func(item pageparser.Item) bool {
+		if sourceFormat != nil {
+			// The rest is content.
+			sourceContent = psr.Input()[item.Pos:]
+			// Done
+			return false
+		} else if item.IsFrontMatter() {
+			fromFormat = metadecoders.FormatFromFrontMatterType(item.Type)
+			sourceFormat = item.Val
+		}
+		return true
+
+	}
+
+	iter.PeekWalk(walkFn)
+
+	metadata, err := metadecoders.UnmarshalToMap(sourceFormat, fromFormat)
 	if err != nil {
-		site.Log.ERROR.Println("Error processing file:", p.Path())
+		site.Log.ERROR.Println(errMsg)
 		return err
 	}
 
 	// better handling of dates in formats that don't have support for them
-	if mark == parser.FormatToLeadRune("json") || mark == parser.FormatToLeadRune("yaml") || mark == parser.FormatToLeadRune("toml") {
+	if fromFormat == metadecoders.JSON || fromFormat == metadecoders.YAML || fromFormat == metadecoders.TOML {
 		newMetadata := cast.ToStringMap(metadata)
 		for k, v := range newMetadata {
 			switch vv := v.(type) {
@@ -175,18 +204,26 @@ func (cc *convertCmd) convertAndSavePage(p *hugolib.Page, site *hugolib.Site, ma
 		metadata = newMetadata
 	}
 
-	newPage.SetSourceContent(psr.Content())
+	var newContent bytes.Buffer
-	if err = newPage.SetSourceMetaData(metadata, mark); err != nil {
+	err = parser.InterfaceToFrontMatter2(metadata, targetFormat, &newContent)
-		site.Log.ERROR.Printf("Failed to set source metadata for file %q: %s. For more info see For more info see https://github.com/gohugoio/hugo/issues/2458", newPage.FullFilePath(), err)
+	if err != nil {
-		return nil
+		site.Log.ERROR.Println(errMsg)
+		return err
 	}
 
+	newContent.Write(sourceContent)
+
 	newFilename := p.Filename()
+
 	if cc.outputDir != "" {
-		newFilename = filepath.Join(cc.outputDir, p.Dir(), newPage.LogicalName())
+		contentDir := strings.TrimSuffix(newFilename, p.Path())
+		contentDir = filepath.Base(contentDir)
+
+		newFilename = filepath.Join(cc.outputDir, contentDir, p.Path())
 	}
 
-	if err = newPage.SaveSourceAs(newFilename); err != nil {
+	fs := hugofs.Os
+	if err := helpers.WriteToDisk(newFilename, &newContent, fs); err != nil {
 		return errors.Wrapf(err, "Failed to save file %q:", newFilename)
 	}
 

commands/hugo.go

@@ -42,7 +42,7 @@ import (
 
 	"github.com/gohugoio/hugo/config"
 
-	"github.com/gohugoio/hugo/parser"
+	"github.com/gohugoio/hugo/parser/metadecoders"
 	flag "github.com/spf13/pflag"
 
 	"github.com/fsnotify/fsnotify"
@@ -1017,7 +1017,7 @@ func (c *commandeer) isThemeVsHugoVersionMismatch(fs afero.Fs) (dir string, mism
 
 		b, err := afero.ReadFile(fs, path)
 
-		tomlMeta, err := parser.HandleTOMLMetaData(b)
+		tomlMeta, err := metadecoders.UnmarshalToMap(b, metadecoders.TOML)
 
 		if err != nil {
 			continue

commands/import_jekyll.go

@@ -25,6 +25,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/gohugoio/hugo/parser/metadecoders"
+
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/gohugoio/hugo/hugolib"
@@ -253,7 +255,7 @@ func (i *importCmd) loadJekyllConfig(fs afero.Fs, jekyllRoot string) map[string]
 		return nil
 	}
 
-	c, err := parser.HandleYAMLMetaData(b)
+	c, err := metadecoders.UnmarshalToMap(b, metadecoders.YAML)
 
 	if err != nil {
 		return nil

go.mod

@@ -51,7 +51,7 @@ require (
 	github.com/spf13/nitro v0.0.0-20131003134307-24d7ef30a12d
 	github.com/spf13/pflag v1.0.2
 	github.com/spf13/viper v1.2.0
-	github.com/stretchr/testify v1.2.2
+	github.com/stretchr/testify v1.2.3-0.20181002232621-f2347ac6c9c9
 	github.com/tdewolff/minify v2.3.5+incompatible
 	github.com/tdewolff/parse v2.3.3+incompatible // indirect
 	github.com/tdewolff/test v0.0.0-20171106182207-265427085153 // indirect
@@ -63,6 +63,7 @@ require (
 	golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e // indirect
 	golang.org/x/text v0.3.0
 	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
+	gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0
 	gopkg.in/yaml.v2 v2.2.1
 )
 

go.sum

@@ -65,6 +65,7 @@ github.com/magefile/mage v1.4.0 h1:RI7B1CgnPAuu2O9lWszwya61RLmfL0KCdo+QyyI/Bhk=
 github.com/magefile/mage v1.4.0/go.mod h1:IUDi13rsHje59lecXokTfGX0QIzO45uVPlXnJYsXepA=
 github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/markbates/inflect v0.0.0-20171215194931-a12c3aec81a6 h1:LZhVjIISSbj8qLf2qDPP0D8z0uvOWAW5C85ly5mJW6c=
 github.com/markbates/inflect v0.0.0-20171215194931-a12c3aec81a6/go.mod h1:oTeZL2KHA7CUX6X+fovmK9OvIOFuqu0TwdQrZjLTh88=
 github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
@@ -118,6 +119,8 @@ github.com/spf13/viper v1.2.0 h1:M4Rzxlu+RgU4pyBRKhKaVN1VeYOm8h2jgyXnAseDgCc=
 github.com/spf13/viper v1.2.0/go.mod h1:P4AexN0a+C9tGAnUFNwDMYYZv3pjFuvmeiMyKRaNVlI=
 github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.2.3-0.20181002232621-f2347ac6c9c9 h1:kcVw9CGDqYBy0TTpIq2+BNR4W9poqiwEPBh/OYX5CaU=
+github.com/stretchr/testify v1.2.3-0.20181002232621-f2347ac6c9c9/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/tdewolff/minify v2.3.5+incompatible h1:oFxBKxTIY1F/1DEJhLeh/T507W56JqZtWVrawFcdadI=
 github.com/tdewolff/minify v2.3.5+incompatible/go.mod h1:9Ov578KJUmAWpS6NeZwRZyT56Uf6o3Mcz9CEsg8USYs=
 github.com/tdewolff/parse v2.3.3+incompatible h1:q6OSjvHtvBucLb34z24OH1xl5wGdw1mI9Vd38Qj9evs=
@@ -143,5 +146,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 h1:POO/ycCATvegFmVuPpQzZFJ+pGZeX22Ufu6fibxDVjU=
+gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg=
 gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

hugolib/site.go

@@ -44,6 +44,7 @@ import (
 	"github.com/gohugoio/hugo/config"
 
 	"github.com/gohugoio/hugo/media"
+	"github.com/gohugoio/hugo/parser/metadecoders"
 
 	"github.com/markbates/inflect"
 
@@ -53,7 +54,6 @@ import (
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugolib/pagemeta"
 	"github.com/gohugoio/hugo/output"
-	"github.com/gohugoio/hugo/parser"
 	"github.com/gohugoio/hugo/related"
 	"github.com/gohugoio/hugo/source"
 	"github.com/gohugoio/hugo/tpl"
@@ -949,16 +949,8 @@ func (s *Site) readData(f source.ReadableFile) (interface{}, error) {
 	defer file.Close()
 	content := helpers.ReaderToBytes(file)
 
-	switch f.Extension() {
+	format := metadecoders.FormatFromString(f.Extension())
-	case "yaml", "yml":
+	return metadecoders.Unmarshal(content, format)
-		return parser.HandleYAMLData(content)
-	case "json":
-		return parser.HandleJSONData(content)
-	case "toml":
-		return parser.HandleTOMLMetaData(content)
-	default:
-		return nil, fmt.Errorf("Data not supported for extension '%s'", f.Extension())
-	}
 }
 
 func (s *Site) readDataFromSourceFS() error {

parser/frontmatter.go

@@ -19,16 +19,12 @@ import (
 	"bytes"
 	"encoding/json"
 	"errors"
-	"fmt"
 	"io"
 	"strings"
 
-	"github.com/gohugoio/hugo/helpers"
+	"github.com/gohugoio/hugo/parser/metadecoders"
-
-	"github.com/spf13/cast"
 
 	"github.com/BurntSushi/toml"
-	"github.com/chaseadamsio/goorgeous"
 
 	"gopkg.in/yaml.v2"
 )
@@ -79,6 +75,82 @@ func InterfaceToConfig(in interface{}, mark rune, w io.Writer) error {
 	}
 }
 
+func InterfaceToConfig2(in interface{}, format metadecoders.Format, w io.Writer) error {
+	if in == nil {
+		return errors.New("input was nil")
+	}
+
+	switch format {
+	case metadecoders.YAML:
+		b, err := yaml.Marshal(in)
+		if err != nil {
+			return err
+		}
+
+		_, err = w.Write(b)
+		return err
+
+	case metadecoders.TOML:
+		return toml.NewEncoder(w).Encode(in)
+	case metadecoders.JSON:
+		b, err := json.MarshalIndent(in, "", "   ")
+		if err != nil {
+			return err
+		}
+
+		_, err = w.Write(b)
+		if err != nil {
+			return err
+		}
+
+		_, err = w.Write([]byte{'\n'})
+		return err
+
+	default:
+		return errors.New("Unsupported Format provided")
+	}
+}
+
+func InterfaceToFrontMatter2(in interface{}, format metadecoders.Format, w io.Writer) error {
+	if in == nil {
+		return errors.New("input was nil")
+	}
+
+	switch format {
+	case metadecoders.YAML:
+		_, err := w.Write([]byte(YAMLDelimUnix))
+		if err != nil {
+			return err
+		}
+
+		err = InterfaceToConfig2(in, format, w)
+		if err != nil {
+			return err
+		}
+
+		_, err = w.Write([]byte(YAMLDelimUnix))
+		return err
+
+	case metadecoders.TOML:
+		_, err := w.Write([]byte(TOMLDelimUnix))
+		if err != nil {
+			return err
+		}
+
+		err = InterfaceToConfig2(in, format, w)
+
+		if err != nil {
+			return err
+		}
+
+		_, err = w.Write([]byte("\n" + TOMLDelimUnix))
+		return err
+
+	default:
+		return InterfaceToConfig2(in, format, w)
+	}
+}
+
 // InterfaceToFrontMatter encodes a given input into a frontmatter
 // representation based upon the mark with the appropriate front matter delimiters
 // surrounding the output, which is written to w.
@@ -155,34 +227,6 @@ func FormatSanitize(kind string) string {
 	}
 }
 
-// DetectFrontMatter detects the type of frontmatter analysing its first character.
-func DetectFrontMatter(mark rune) (f *FrontmatterType) {
-	switch mark {
-	case '-':
-		return &FrontmatterType{HandleYAMLMetaData, []byte(YAMLDelim), []byte(YAMLDelim), false}
-	case '+':
-		return &FrontmatterType{HandleTOMLMetaData, []byte(TOMLDelim), []byte(TOMLDelim), false}
-	case '{':
-		return &FrontmatterType{HandleJSONMetaData, []byte{'{'}, []byte{'}'}, true}
-	case '#':
-		return &FrontmatterType{HandleOrgMetaData, []byte("#+"), []byte("\n"), false}
-	default:
-		return nil
-	}
-}
-
-// HandleTOMLMetaData unmarshals TOML-encoded datum and returns a Go interface
-// representing the encoded data structure.
-func HandleTOMLMetaData(datum []byte) (map[string]interface{}, error) {
-	m := map[string]interface{}{}
-	datum = removeTOMLIdentifier(datum)
-
-	_, err := toml.Decode(string(datum), &m)
-
-	return m, err
-
-}
-
 // removeTOMLIdentifier removes, if necessary, beginning and ending TOML
 // frontmatter delimiters from a byte slice.
 func removeTOMLIdentifier(datum []byte) []byte {
@@ -200,125 +244,3 @@ func removeTOMLIdentifier(datum []byte) []byte {
 	b = bytes.Trim(b, "\r\n")
 	return bytes.TrimSuffix(b, []byte(TOMLDelim))
 }
-
-// HandleYAMLMetaData unmarshals YAML-encoded datum and returns a Go interface
-// representing the encoded data structure.
-// TODO(bep) 2errors remove these handlers (and hopefully package)
-func HandleYAMLMetaData(datum []byte) (map[string]interface{}, error) {
-	m := map[string]interface{}{}
-	err := yaml.Unmarshal(datum, &m)
-
-	// To support boolean keys, the `yaml` package unmarshals maps to
-	// map[interface{}]interface{}. Here we recurse through the result
-	// and change all maps to map[string]interface{} like we would've
-	// gotten from `json`.
-	if err == nil {
-		for k, v := range m {
-			if vv, changed := stringifyMapKeys(v); changed {
-				m[k] = vv
-			}
-		}
-	}
-
-	return m, err
-}
-
-// HandleYAMLData unmarshals YAML-encoded datum and returns a Go interface
-// representing the encoded data structure.
-func HandleYAMLData(datum []byte) (interface{}, error) {
-	var m interface{}
-	err := yaml.Unmarshal(datum, &m)
-	if err != nil {
-		return nil, err
-	}
-
-	// To support boolean keys, the `yaml` package unmarshals maps to
-	// map[interface{}]interface{}. Here we recurse through the result
-	// and change all maps to map[string]interface{} like we would've
-	// gotten from `json`.
-	if mm, changed := stringifyMapKeys(m); changed {
-		return mm, nil
-	}
-
-	return m, nil
-}
-
-// stringifyMapKeys recurses into in and changes all instances of
-// map[interface{}]interface{} to map[string]interface{}. This is useful to
-// work around the impedence mismatch between JSON and YAML unmarshaling that's
-// described here: https://github.com/go-yaml/yaml/issues/139
-//
-// Inspired by https://github.com/stripe/stripe-mock, MIT licensed
-func stringifyMapKeys(in interface{}) (interface{}, bool) {
-	switch in := in.(type) {
-	case []interface{}:
-		for i, v := range in {
-			if vv, replaced := stringifyMapKeys(v); replaced {
-				in[i] = vv
-			}
-		}
-	case map[interface{}]interface{}:
-		res := make(map[string]interface{})
-		var (
-			ok  bool
-			err error
-		)
-		for k, v := range in {
-			var ks string
-
-			if ks, ok = k.(string); !ok {
-				ks, err = cast.ToStringE(k)
-				if err != nil {
-					ks = fmt.Sprintf("%v", k)
-				}
-				// TODO(bep) added in Hugo 0.37, remove some time in the future.
-				helpers.DistinctFeedbackLog.Printf("WARNING: YAML data/frontmatter with keys of type %T is since Hugo 0.37 converted to strings", k)
-			}
-			if vv, replaced := stringifyMapKeys(v); replaced {
-				res[ks] = vv
-			} else {
-				res[ks] = v
-			}
-		}
-		return res, true
-	}
-
-	return nil, false
-}
-
-// HandleJSONMetaData unmarshals JSON-encoded datum and returns a Go interface
-// representing the encoded data structure.
-func HandleJSONMetaData(datum []byte) (map[string]interface{}, error) {
-	m := make(map[string]interface{})
-
-	if datum == nil {
-		// Package json returns on error on nil input.
-		// Return an empty map to be consistent with our other supported
-		// formats.
-		return m, nil
-	}
-
-	err := json.Unmarshal(datum, &m)
-	return m, err
-}
-
-// HandleJSONData unmarshals JSON-encoded datum and returns a Go interface
-// representing the encoded data structure.
-func HandleJSONData(datum []byte) (interface{}, error) {
-	if datum == nil {
-		// Package json returns on error on nil input.
-		// Return an empty map to be consistent with our other supported
-		// formats.
-		return make(map[string]interface{}), nil
-	}
-
-	var f interface{}
-	err := json.Unmarshal(datum, &f)
-	return f, err
-}
-
-// HandleOrgMetaData unmarshals org-mode encoded datum and returns a Go
-// interface representing the encoded data structure.
-func HandleOrgMetaData(datum []byte) (map[string]interface{}, error) {
-	return goorgeous.OrgHeaders(datum)
-}

parser/frontmatter_test.go

@@ -132,116 +132,6 @@ func TestInterfaceToFrontMatter(t *testing.T) {
 	}
 }
 
-func TestHandleTOMLMetaData(t *testing.T) {
-	cases := []struct {
-		input []byte
-		want  interface{}
-		isErr bool
-	}{
-		{nil, map[string]interface{}{}, false},
-		{[]byte("title = \"test 1\""), map[string]interface{}{"title": "test 1"}, false},
-		{[]byte("a = [1, 2, 3]"), map[string]interface{}{"a": []interface{}{int64(1), int64(2), int64(3)}}, false},
-		{[]byte("b = [\n[1, 2],\n[3, 4]\n]"), map[string]interface{}{"b": []interface{}{[]interface{}{int64(1), int64(2)}, []interface{}{int64(3), int64(4)}}}, false},
-		// errors
-		{[]byte("z = [\n[1, 2]\n[3, 4]\n]"), nil, true},
-	}
-
-	for i, c := range cases {
-		res, err := HandleTOMLMetaData(c.input)
-		if err != nil {
-			if c.isErr {
-				continue
-			}
-			t.Fatalf("[%d] unexpected error value: %v", i, err)
-		}
-
-		if !reflect.DeepEqual(res, c.want) {
-			t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res)
-		}
-	}
-}
-
-func TestHandleYAMLMetaData(t *testing.T) {
-	cases := []struct {
-		input []byte
-		want  interface{}
-		isErr bool
-	}{
-		{nil, map[string]interface{}{}, false},
-		{[]byte("title: test 1"), map[string]interface{}{"title": "test 1"}, false},
-		{[]byte("a: Easy!\nb:\n  c: 2\n  d: [3, 4]"), map[string]interface{}{"a": "Easy!", "b": map[string]interface{}{"c": 2, "d": []interface{}{3, 4}}}, false},
-		{[]byte("a:\n  true: 1\n  false: 2"), map[string]interface{}{"a": map[string]interface{}{"true": 1, "false": 2}}, false},
-		// errors
-		{[]byte("z = not toml"), nil, true},
-	}
-
-	for i, c := range cases {
-		res, err := HandleYAMLMetaData(c.input)
-		if err != nil {
-			if c.isErr {
-				continue
-			}
-			t.Fatalf("[%d] unexpected error value: %v", i, err)
-		}
-
-		if !reflect.DeepEqual(res, c.want) {
-			t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res)
-		}
-	}
-}
-
-func TestHandleJSONMetaData(t *testing.T) {
-	cases := []struct {
-		input []byte
-		want  interface{}
-		isErr bool
-	}{
-		{nil, map[string]interface{}{}, false},
-		{[]byte("{\"title\": \"test 1\"}"), map[string]interface{}{"title": "test 1"}, false},
-		// errors
-		{[]byte("{noquotes}"), nil, true},
-	}
-
-	for i, c := range cases {
-		res, err := HandleJSONMetaData(c.input)
-		if err != nil {
-			if c.isErr {
-				continue
-			}
-			t.Fatalf("[%d] unexpected error value: %v", i, err)
-		}
-
-		if !reflect.DeepEqual(res, c.want) {
-			t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res)
-		}
-	}
-}
-
-func TestHandleOrgMetaData(t *testing.T) {
-	cases := []struct {
-		input []byte
-		want  interface{}
-		isErr bool
-	}{
-		{nil, map[string]interface{}{}, false},
-		{[]byte("#+title: test 1\n"), map[string]interface{}{"title": "test 1"}, false},
-	}
-
-	for i, c := range cases {
-		res, err := HandleOrgMetaData(c.input)
-		if err != nil {
-			if c.isErr {
-				continue
-			}
-			t.Fatalf("[%d] unexpected error value: %v", i, err)
-		}
-
-		if !reflect.DeepEqual(res, c.want) {
-			t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res)
-		}
-	}
-}
-
 func TestFormatToLeadRune(t *testing.T) {
 	for i, this := range []struct {
 		kind   string
@@ -264,41 +154,6 @@ func TestFormatToLeadRune(t *testing.T) {
 	}
 }
 
-func TestDetectFrontMatter(t *testing.T) {
-	cases := []struct {
-		mark rune
-		want *FrontmatterType
-	}{
-		// funcs are uncomparable, so we ignore FrontmatterType.Parse in these tests
-		{'-', &FrontmatterType{nil, []byte(YAMLDelim), []byte(YAMLDelim), false}},
-		{'+', &FrontmatterType{nil, []byte(TOMLDelim), []byte(TOMLDelim), false}},
-		{'{', &FrontmatterType{nil, []byte("{"), []byte("}"), true}},
-		{'#', &FrontmatterType{nil, []byte("#+"), []byte("\n"), false}},
-		{'$', nil},
-	}
-
-	for _, c := range cases {
-		res := DetectFrontMatter(c.mark)
-		if res == nil {
-			if c.want == nil {
-				continue
-			}
-
-			t.Fatalf("want %v, got %v", *c.want, res)
-		}
-
-		if !reflect.DeepEqual(res.markstart, c.want.markstart) {
-			t.Errorf("markstart mismatch: want %v, got %v", c.want.markstart, res.markstart)
-		}
-		if !reflect.DeepEqual(res.markend, c.want.markend) {
-			t.Errorf("markend mismatch: want %v, got %v", c.want.markend, res.markend)
-		}
-		if !reflect.DeepEqual(res.includeMark, c.want.includeMark) {
-			t.Errorf("includeMark mismatch: want %v, got %v", c.want.includeMark, res.includeMark)
-		}
-	}
-}
-
 func TestRemoveTOMLIdentifier(t *testing.T) {
 	cases := []struct {
 		input string
@@ -321,64 +176,6 @@ func TestRemoveTOMLIdentifier(t *testing.T) {
 	}
 }
 
-func TestStringifyYAMLMapKeys(t *testing.T) {
-	cases := []struct {
-		input    interface{}
-		want     interface{}
-		replaced bool
-	}{
-		{
-			map[interface{}]interface{}{"a": 1, "b": 2},
-			map[string]interface{}{"a": 1, "b": 2},
-			true,
-		},
-		{
-			map[interface{}]interface{}{"a": []interface{}{1, map[interface{}]interface{}{"b": 2}}},
-			map[string]interface{}{"a": []interface{}{1, map[string]interface{}{"b": 2}}},
-			true,
-		},
-		{
-			map[interface{}]interface{}{true: 1, "b": false},
-			map[string]interface{}{"true": 1, "b": false},
-			true,
-		},
-		{
-			map[interface{}]interface{}{1: "a", 2: "b"},
-			map[string]interface{}{"1": "a", "2": "b"},
-			true,
-		},
-		{
-			map[interface{}]interface{}{"a": map[interface{}]interface{}{"b": 1}},
-			map[string]interface{}{"a": map[string]interface{}{"b": 1}},
-			true,
-		},
-		{
-			map[string]interface{}{"a": map[string]interface{}{"b": 1}},
-			map[string]interface{}{"a": map[string]interface{}{"b": 1}},
-			false,
-		},
-		{
-			[]interface{}{map[interface{}]interface{}{1: "a", 2: "b"}},
-			[]interface{}{map[string]interface{}{"1": "a", "2": "b"}},
-			false,
-		},
-	}
-
-	for i, c := range cases {
-		res, replaced := stringifyMapKeys(c.input)
-
-		if c.replaced != replaced {
-			t.Fatalf("[%d] Replaced mismatch: %t", i, replaced)
-		}
-		if !c.replaced {
-			res = c.input
-		}
-		if !reflect.DeepEqual(res, c.want) {
-			t.Errorf("[%d] given %q\nwant: %q\n got: %q", i, c.input, c.want, res)
-		}
-	}
-}
-
 func BenchmarkFrontmatterTags(b *testing.B) {
 
 	for _, frontmatter := range []string{"JSON", "YAML", "YAML2", "TOML"} {
@@ -388,69 +185,6 @@ func BenchmarkFrontmatterTags(b *testing.B) {
 	}
 }
 
-func BenchmarkStringifyMapKeysStringsOnlyInterfaceMaps(b *testing.B) {
-	maps := make([]map[interface{}]interface{}, b.N)
-	for i := 0; i < b.N; i++ {
-		maps[i] = map[interface{}]interface{}{
-			"a": map[interface{}]interface{}{
-				"b": 32,
-				"c": 43,
-				"d": map[interface{}]interface{}{
-					"b": 32,
-					"c": 43,
-				},
-			},
-			"b": []interface{}{"a", "b"},
-			"c": "d",
-		}
-	}
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		stringifyMapKeys(maps[i])
-	}
-}
-
-func BenchmarkStringifyMapKeysStringsOnlyStringMaps(b *testing.B) {
-	m := map[string]interface{}{
-		"a": map[string]interface{}{
-			"b": 32,
-			"c": 43,
-			"d": map[string]interface{}{
-				"b": 32,
-				"c": 43,
-			},
-		},
-		"b": []interface{}{"a", "b"},
-		"c": "d",
-	}
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		stringifyMapKeys(m)
-	}
-}
-
-func BenchmarkStringifyMapKeysIntegers(b *testing.B) {
-	maps := make([]map[interface{}]interface{}, b.N)
-	for i := 0; i < b.N; i++ {
-		maps[i] = map[interface{}]interface{}{
-			1: map[interface{}]interface{}{
-				4: 32,
-				5: 43,
-				6: map[interface{}]interface{}{
-					7: 32,
-					8: 43,
-				},
-			},
-			2: []interface{}{"a", "b"},
-			3: "d",
-		}
-	}
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		stringifyMapKeys(maps[i])
-	}
-}
 func doBenchmarkFrontmatter(b *testing.B, fileformat string, numTags int) {
 	yamlTemplate := `---
 name: "Tags"

parser/metadecoders/decoder.go

@@ -15,81 +15,139 @@ package metadecoders
 
 import (
 	"encoding/json"
+	"fmt"
 
 	"github.com/BurntSushi/toml"
 	"github.com/chaseadamsio/goorgeous"
-	"github.com/gohugoio/hugo/parser/pageparser"
 	"github.com/pkg/errors"
+	"github.com/spf13/cast"
 	yaml "gopkg.in/yaml.v2"
 )
 
-type Format string
-
-const (
-	// These are the supported metdata  formats in Hugo. Most of these are also
-	// supported as /data formats.
-	ORG  Format = "org"
-	JSON Format = "json"
-	TOML Format = "toml"
-	YAML Format = "yaml"
-)
-
-// FormatFromFrontMatterType will return empty if not supported.
-func FormatFromFrontMatterType(typ pageparser.ItemType) Format {
-	switch typ {
-	case pageparser.TypeFrontMatterJSON:
-		return JSON
-	case pageparser.TypeFrontMatterORG:
-		return ORG
-	case pageparser.TypeFrontMatterTOML:
-		return TOML
-	case pageparser.TypeFrontMatterYAML:
-		return YAML
-	default:
-		return ""
-	}
-}
-
 // UnmarshalToMap will unmarshall data in format f into a new map. This is
 // what's needed for Hugo's front matter decoding.
 func UnmarshalToMap(data []byte, f Format) (map[string]interface{}, error) {
 	m := make(map[string]interface{})
-
 	if data == nil {
 		return m, nil
 	}
 
+	err := unmarshal(data, f, &m)
+
+	return m, err
+
+}
+
+// Unmarshal will unmarshall data in format f into an interface{}.
+// This is what's needed for Hugo's /data handling.
+func Unmarshal(data []byte, f Format) (interface{}, error) {
+	if data == nil {
+		return make(map[string]interface{}), nil
+	}
+	var v interface{}
+	err := unmarshal(data, f, &v)
+
+	return v, err
+}
+
+// unmarshal unmarshals data in format f into v.
+func unmarshal(data []byte, f Format, v interface{}) error {
+
 	var err error
 
 	switch f {
 	case ORG:
-		m, err = goorgeous.OrgHeaders(data)
+		vv, err := goorgeous.OrgHeaders(data)
+		if err != nil {
+			return err
+		}
+		switch v.(type) {
+		case *map[string]interface{}:
+			*v.(*map[string]interface{}) = vv
+		default:
+			*v.(*interface{}) = vv
+		}
 	case JSON:
-		err = json.Unmarshal(data, &m)
+		err = json.Unmarshal(data, v)
 	case TOML:
-		_, err = toml.Decode(string(data), &m)
+		err = toml.Unmarshal(data, v)
 	case YAML:
-		err = yaml.Unmarshal(data, &m)
+		err = yaml.Unmarshal(data, v)
 
-		// To support boolean keys, the `yaml` package unmarshals maps to
+		// To support boolean keys, the YAML package unmarshals maps to
 		// map[interface{}]interface{}. Here we recurse through the result
 		// and change all maps to map[string]interface{} like we would've
 		// gotten from `json`.
-		if err == nil {
+		var ptr interface{}
-			for k, v := range m {
+		switch v.(type) {
-				if vv, changed := stringifyMapKeys(v); changed {
+		case *map[string]interface{}:
-					m[k] = vv
+			ptr = *v.(*map[string]interface{})
-				}
+		case *interface{}:
+			ptr = *v.(*interface{})
+		default:
+			return errors.Errorf("unknown type %T in YAML unmarshal", v)
+		}
+
+		if mm, changed := stringifyMapKeys(ptr); changed {
+			switch v.(type) {
+			case *map[string]interface{}:
+				*v.(*map[string]interface{}) = mm.(map[string]interface{})
+			case *interface{}:
+				*v.(*interface{}) = mm
 			}
 		}
 	default:
-		return nil, errors.Errorf("unmarshal of format %q is not supported", f)
+		return errors.Errorf("unmarshal of format %q is not supported", f)
 	}
 
-	if err != nil {
+	return err
-		return nil, errors.Wrapf(err, "unmarshal failed for format %q", f)
-	}
-
-	return m, nil
 
 }
+
+// stringifyMapKeys recurses into in and changes all instances of
+// map[interface{}]interface{} to map[string]interface{}. This is useful to
+// work around the impedence mismatch between JSON and YAML unmarshaling that's
+// described here: https://github.com/go-yaml/yaml/issues/139
+//
+// Inspired by https://github.com/stripe/stripe-mock, MIT licensed
+func stringifyMapKeys(in interface{}) (interface{}, bool) {
+
+	switch in := in.(type) {
+	case []interface{}:
+		for i, v := range in {
+			if vv, replaced := stringifyMapKeys(v); replaced {
+				in[i] = vv
+			}
+		}
+	case map[string]interface{}:
+		for k, v := range in {
+			if vv, changed := stringifyMapKeys(v); changed {
+				in[k] = vv
+			}
+		}
+	case map[interface{}]interface{}:
+		res := make(map[string]interface{})
+		var (
+			ok  bool
+			err error
+		)
+		for k, v := range in {
+			var ks string
+
+			if ks, ok = k.(string); !ok {
+				ks, err = cast.ToStringE(k)
+				if err != nil {
+					ks = fmt.Sprintf("%v", k)
+				}
+			}
+			if vv, replaced := stringifyMapKeys(v); replaced {
+				res[ks] = vv
+			} else {
+				res[ks] = v
+			}
+		}
+		return res, true
+	}
+
+	return nil, false
+}

parser/metadecoders/decoder_test.go

@@ -0,0 +1,207 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metadecoders
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestUnmarshalToMap(t *testing.T) {
+	assert := require.New(t)
+
+	expect := map[string]interface{}{"a": "b"}
+
+	for i, test := range []struct {
+		data   string
+		format Format
+		expect interface{}
+	}{
+		{`a = "b"`, TOML, expect},
+		{`a: "b"`, YAML, expect},
+		// Make sure we get all string keys, even for YAML
+		{"a: Easy!\nb:\n  c: 2\n  d: [3, 4]", YAML, map[string]interface{}{"a": "Easy!", "b": map[string]interface{}{"c": 2, "d": []interface{}{3, 4}}}},
+		{"a:\n  true: 1\n  false: 2", YAML, map[string]interface{}{"a": map[string]interface{}{"true": 1, "false": 2}}},
+		{`{ "a": "b" }`, JSON, expect},
+		{`#+a: b`, ORG, expect},
+		// errors
+		{`a = b`, TOML, false},
+	} {
+		msg := fmt.Sprintf("%d: %s", i, test.format)
+		m, err := UnmarshalToMap([]byte(test.data), test.format)
+		if b, ok := test.expect.(bool); ok && !b {
+			assert.Error(err, msg)
+		} else {
+			assert.NoError(err, msg)
+			assert.Equal(test.expect, m, msg)
+		}
+	}
+}
+
+func TestUnmarshalToInterface(t *testing.T) {
+	assert := require.New(t)
+
+	expect := map[string]interface{}{"a": "b"}
+
+	for i, test := range []struct {
+		data   string
+		format Format
+		expect interface{}
+	}{
+		{`[ "Brecker", "Blake", "Redman" ]`, JSON, []interface{}{"Brecker", "Blake", "Redman"}},
+		{`{ "a": "b" }`, JSON, expect},
+		{`#+a: b`, ORG, expect},
+		{`a = "b"`, TOML, expect},
+		{`a: "b"`, YAML, expect},
+		{"a: Easy!\nb:\n  c: 2\n  d: [3, 4]", YAML, map[string]interface{}{"a": "Easy!", "b": map[string]interface{}{"c": 2, "d": []interface{}{3, 4}}}},
+		// errors
+		{`a = "`, TOML, false},
+	} {
+		msg := fmt.Sprintf("%d: %s", i, test.format)
+		m, err := Unmarshal([]byte(test.data), test.format)
+		if b, ok := test.expect.(bool); ok && !b {
+			assert.Error(err, msg)
+		} else {
+			assert.NoError(err, msg)
+			assert.Equal(test.expect, m, msg)
+		}
+
+	}
+
+}
+
+func TestStringifyYAMLMapKeys(t *testing.T) {
+	cases := []struct {
+		input    interface{}
+		want     interface{}
+		replaced bool
+	}{
+		{
+			map[interface{}]interface{}{"a": 1, "b": 2},
+			map[string]interface{}{"a": 1, "b": 2},
+			true,
+		},
+		{
+			map[interface{}]interface{}{"a": []interface{}{1, map[interface{}]interface{}{"b": 2}}},
+			map[string]interface{}{"a": []interface{}{1, map[string]interface{}{"b": 2}}},
+			true,
+		},
+		{
+			map[interface{}]interface{}{true: 1, "b": false},
+			map[string]interface{}{"true": 1, "b": false},
+			true,
+		},
+		{
+			map[interface{}]interface{}{1: "a", 2: "b"},
+			map[string]interface{}{"1": "a", "2": "b"},
+			true,
+		},
+		{
+			map[interface{}]interface{}{"a": map[interface{}]interface{}{"b": 1}},
+			map[string]interface{}{"a": map[string]interface{}{"b": 1}},
+			true,
+		},
+		{
+			map[string]interface{}{"a": map[string]interface{}{"b": 1}},
+			map[string]interface{}{"a": map[string]interface{}{"b": 1}},
+			false,
+		},
+		{
+			[]interface{}{map[interface{}]interface{}{1: "a", 2: "b"}},
+			[]interface{}{map[string]interface{}{"1": "a", "2": "b"}},
+			false,
+		},
+	}
+
+	for i, c := range cases {
+		res, replaced := stringifyMapKeys(c.input)
+
+		if c.replaced != replaced {
+			t.Fatalf("[%d] Replaced mismatch: %t", i, replaced)
+		}
+		if !c.replaced {
+			res = c.input
+		}
+		if !reflect.DeepEqual(res, c.want) {
+			t.Errorf("[%d] given %q\nwant: %q\n got: %q", i, c.input, c.want, res)
+		}
+	}
+}
+
+func BenchmarkStringifyMapKeysStringsOnlyInterfaceMaps(b *testing.B) {
+	maps := make([]map[interface{}]interface{}, b.N)
+	for i := 0; i < b.N; i++ {
+		maps[i] = map[interface{}]interface{}{
+			"a": map[interface{}]interface{}{
+				"b": 32,
+				"c": 43,
+				"d": map[interface{}]interface{}{
+					"b": 32,
+					"c": 43,
+				},
+			},
+			"b": []interface{}{"a", "b"},
+			"c": "d",
+		}
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		stringifyMapKeys(maps[i])
+	}
+}
+
+func BenchmarkStringifyMapKeysStringsOnlyStringMaps(b *testing.B) {
+	m := map[string]interface{}{
+		"a": map[string]interface{}{
+			"b": 32,
+			"c": 43,
+			"d": map[string]interface{}{
+				"b": 32,
+				"c": 43,
+			},
+		},
+		"b": []interface{}{"a", "b"},
+		"c": "d",
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		stringifyMapKeys(m)
+	}
+}
+
+func BenchmarkStringifyMapKeysIntegers(b *testing.B) {
+	maps := make([]map[interface{}]interface{}, b.N)
+	for i := 0; i < b.N; i++ {
+		maps[i] = map[interface{}]interface{}{
+			1: map[interface{}]interface{}{
+				4: 32,
+				5: 43,
+				6: map[interface{}]interface{}{
+					7: 32,
+					8: 43,
+				},
+			},
+			2: []interface{}{"a", "b"},
+			3: "d",
+		}
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		stringifyMapKeys(maps[i])
+	}
+}

parser/metadecoders/format.go

@@ -0,0 +1,66 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metadecoders
+
+import (
+	"strings"
+
+	"github.com/gohugoio/hugo/parser/pageparser"
+)
+
+type Format string
+
+const (
+	// These are the supported metdata  formats in Hugo. Most of these are also
+	// supported as /data formats.
+	ORG  Format = "org"
+	JSON Format = "json"
+	TOML Format = "toml"
+	YAML Format = "yaml"
+)
+
+// FormatFromString turns formatStr, typically a file extension without any ".",
+// into a Format. It returns an empty string for unknown formats.
+func FormatFromString(formatStr string) Format {
+	formatStr = strings.ToLower(formatStr)
+	switch formatStr {
+	case "yaml", "yml":
+		return YAML
+	case "json":
+		return JSON
+	case "toml":
+		return TOML
+	case "org":
+		return ORG
+	}
+
+	return ""
+
+}
+
+// FormatFromFrontMatterType will return empty if not supported.
+func FormatFromFrontMatterType(typ pageparser.ItemType) Format {
+	switch typ {
+	case pageparser.TypeFrontMatterJSON:
+		return JSON
+	case pageparser.TypeFrontMatterORG:
+		return ORG
+	case pageparser.TypeFrontMatterTOML:
+		return TOML
+	case pageparser.TypeFrontMatterYAML:
+		return YAML
+	default:
+		return ""
+	}
+}

parser/metadecoders/format_test.go

@@ -0,0 +1,57 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metadecoders
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/gohugoio/hugo/parser/pageparser"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestFormatFromString(t *testing.T) {
+	assert := require.New(t)
+	for i, test := range []struct {
+		s      string
+		expect Format
+	}{
+		{"json", JSON},
+		{"yaml", YAML},
+		{"yml", YAML},
+		{"toml", TOML},
+		{"tOMl", TOML},
+		{"org", ORG},
+		{"foo", ""},
+	} {
+		assert.Equal(test.expect, FormatFromString(test.s), fmt.Sprintf("t%d", i))
+	}
+}
+
+func TestFormatFromFrontMatterType(t *testing.T) {
+	assert := require.New(t)
+	for i, test := range []struct {
+		typ    pageparser.ItemType
+		expect Format
+	}{
+		{pageparser.TypeFrontMatterJSON, JSON},
+		{pageparser.TypeFrontMatterTOML, TOML},
+		{pageparser.TypeFrontMatterYAML, YAML},
+		{pageparser.TypeFrontMatterORG, ORG},
+		{pageparser.TypeIgnore, ""},
+	} {
+		assert.Equal(test.expect, FormatFromFrontMatterType(test.typ), fmt.Sprintf("t%d", i))
+	}
+}

parser/metadecoders/json.go

@@ -1,31 +0,0 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package metadecoders
-
-import "encoding/json"
-
-// HandleJSONData unmarshals JSON-encoded datum and returns a Go interface
-// representing the encoded data structure.
-func HandleJSONData(datum []byte) (interface{}, error) {
-	if datum == nil {
-		// Package json returns on error on nil input.
-		// Return an empty map to be consistent with our other supported
-		// formats.
-		return make(map[string]interface{}), nil
-	}
-
-	var f interface{}
-	err := json.Unmarshal(datum, &f)
-	return f, err
-}

parser/metadecoders/yaml.go

@@ -1,84 +0,0 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// The metadecoders package contains functions to decode metadata (e.g. page front matter)
-// from different formats: TOML, YAML, JSON.
-package metadecoders
-
-import (
-	"fmt"
-
-	"github.com/spf13/cast"
-	yaml "gopkg.in/yaml.v2"
-)
-
-// HandleYAMLData unmarshals YAML-encoded datum and returns a Go interface
-// representing the encoded data structure.
-func HandleYAMLData(datum []byte) (interface{}, error) {
-	var m interface{}
-	err := yaml.Unmarshal(datum, &m)
-	if err != nil {
-		return nil, err
-	}
-
-	// To support boolean keys, the `yaml` package unmarshals maps to
-	// map[interface{}]interface{}. Here we recurse through the result
-	// and change all maps to map[string]interface{} like we would've
-	// gotten from `json`.
-	if mm, changed := stringifyMapKeys(m); changed {
-		return mm, nil
-	}
-
-	return m, nil
-}
-
-// stringifyMapKeys recurses into in and changes all instances of
-// map[interface{}]interface{} to map[string]interface{}. This is useful to
-// work around the impedence mismatch between JSON and YAML unmarshaling that's
-// described here: https://github.com/go-yaml/yaml/issues/139
-//
-// Inspired by https://github.com/stripe/stripe-mock, MIT licensed
-func stringifyMapKeys(in interface{}) (interface{}, bool) {
-	switch in := in.(type) {
-	case []interface{}:
-		for i, v := range in {
-			if vv, replaced := stringifyMapKeys(v); replaced {
-				in[i] = vv
-			}
-		}
-	case map[interface{}]interface{}:
-		res := make(map[string]interface{})
-		var (
-			ok  bool
-			err error
-		)
-		for k, v := range in {
-			var ks string
-
-			if ks, ok = k.(string); !ok {
-				ks, err = cast.ToStringE(k)
-				if err != nil {
-					ks = fmt.Sprintf("%v", k)
-				}
-			}
-			if vv, replaced := stringifyMapKeys(v); replaced {
-				res[ks] = vv
-			} else {
-				res[ks] = v
-			}
-		}
-		return res, true
-	}
-
-	return nil, false
-}

parser/page.go

@@ -101,15 +101,8 @@ func (p *page) IsRenderable() bool {
 
 // Metadata returns the unmarshalled frontmatter data.
 func (p *page) Metadata() (meta map[string]interface{}, err error) {
-	frontmatter := p.FrontMatter()
 
-	if len(frontmatter) != 0 {
+	return nil, nil
-		fm := DetectFrontMatter(rune(frontmatter[0]))
-		if fm != nil {
-			meta, err = fm.Parse(frontmatter)
-		}
-	}
-	return
 }
 
 // ReadFrom reads the content from an io.Reader and constructs a page.

parser/page_test.go

@@ -1,130 +1 @@
 package parser
-
-import (
-	"fmt"
-	"strings"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestPage(t *testing.T) {
-	cases := []struct {
-		raw string
-
-		content     string
-		frontmatter string
-		renderable  bool
-		metadata    map[string]interface{}
-	}{
-		{
-			testPageLeader + jsonPageFrontMatter + "\n" + testPageTrailer + jsonPageContent,
-			jsonPageContent,
-			jsonPageFrontMatter,
-			true,
-			map[string]interface{}{
-				"title": "JSON Test 1",
-				"social": []interface{}{
-					[]interface{}{"a", "#"},
-					[]interface{}{"b", "#"},
-				},
-			},
-		},
-		{
-			testPageLeader + tomlPageFrontMatter + testPageTrailer + tomlPageContent,
-			tomlPageContent,
-			tomlPageFrontMatter,
-			true,
-			map[string]interface{}{
-				"title": "TOML Test 1",
-				"social": []interface{}{
-					[]interface{}{"a", "#"},
-					[]interface{}{"b", "#"},
-				},
-			},
-		},
-		{
-			testPageLeader + yamlPageFrontMatter + testPageTrailer + yamlPageContent,
-			yamlPageContent,
-			yamlPageFrontMatter,
-			true,
-			map[string]interface{}{
-				"title": "YAML Test 1",
-				"social": []interface{}{
-					[]interface{}{"a", "#"},
-					[]interface{}{"b", "#"},
-				},
-			},
-		},
-		{
-			testPageLeader + orgPageFrontMatter + orgPageContent,
-			orgPageContent,
-			orgPageFrontMatter,
-			true,
-			map[string]interface{}{
-				"TITLE":      "Org Test 1",
-				"categories": []string{"a", "b"},
-			},
-		},
-	}
-
-	for i, c := range cases {
-		p := pageMust(ReadFrom(strings.NewReader(c.raw)))
-		meta, err := p.Metadata()
-
-		mesg := fmt.Sprintf("[%d]", i)
-
-		require.Nil(t, err, mesg)
-		assert.Equal(t, c.content, string(p.Content()), mesg+" content")
-		assert.Equal(t, c.frontmatter, string(p.FrontMatter()), mesg+" frontmatter")
-		assert.Equal(t, c.renderable, p.IsRenderable(), mesg+" renderable")
-		assert.Equal(t, c.metadata, meta, mesg+" metadata")
-	}
-}
-
-var (
-	testWhitespace  = "\t\t\n\n"
-	testPageLeader  = "\ufeff" + testWhitespace + "<!--[metadata]>\n"
-	testPageTrailer = "\n<![end-metadata]-->\n"
-
-	jsonPageContent     = "# JSON Test\n"
-	jsonPageFrontMatter = `{
-	"title": "JSON Test 1",
-	"social": [
-		["a", "#"],
-		["b", "#"]
-	]
-}`
-
-	tomlPageContent     = "# TOML Test\n"
-	tomlPageFrontMatter = `+++
-title = "TOML Test 1"
-social = [
-	["a", "#"],
-	["b", "#"],
-]
-+++
-`
-
-	yamlPageContent     = "# YAML Test\n"
-	yamlPageFrontMatter = `---
-title: YAML Test 1
-social:
-  - - "a"
-    - "#"
-  - - "b"
-    - "#"
----
-`
-
-	orgPageContent     = "* Org Test\n"
-	orgPageFrontMatter = `#+TITLE: Org Test 1
-#+categories: a b
-`
-
-	pageHTMLComment = `<!--
-	This is a sample comment.
--->
-`
-)

parser/pageparser/item.go

@@ -20,7 +20,7 @@ import (
 
 type Item struct {
 	Type ItemType
-	pos pos
+	Pos Pos
 	Val []byte
 }
 

parser/pageparser/pagelexer.go

@@ -25,7 +25,7 @@ import (
 )
 
 // position (in bytes)
-type pos int
+type Pos int
 
 const eof = -1
 
@@ -47,9 +47,9 @@ type pageLexer struct {
 	input      []byte
 	stateStart stateFunc
 	state      stateFunc
-	pos        pos // input position
+	pos        Pos // input position
-	start      pos // item start position
+	start      Pos // item start position
-	width      pos // width of last element
+	width      Pos // width of last element
 
 	// Set when we have parsed any summary divider
 	summaryDividerChecked bool
@@ -73,7 +73,7 @@ func (l *pageLexer) Input() []byte {
 // note: the input position here is normally 0 (start), but
 // can be set if position of first shortcode is known
 // TODO(bep) 2errors byte
-func newPageLexer(input []byte, inputPosition pos, stateStart stateFunc) *pageLexer {
+func newPageLexer(input []byte, inputPosition Pos, stateStart stateFunc) *pageLexer {
 	lexer := &pageLexer{
 		input:      input,
 		pos:        inputPosition,
@@ -131,7 +131,7 @@ func (l *pageLexer) next() rune {
 	}
 
 	runeValue, runeWidth := utf8.DecodeRune(l.input[l.pos:])
-	l.width = pos(runeWidth)
+	l.width = Pos(runeWidth)
 	l.pos += l.width
 	return runeValue
 }
@@ -210,7 +210,7 @@ func lexMainSection(l *pageLexer) stateFunc {
 	l3 = l.index(leftDelimSc)
 	skip := minPositiveIndex(l1, l2, l3)
 	if skip > 0 {
-		l.pos += pos(skip)
+		l.pos += Pos(skip)
 	}
 
 	for {
@@ -234,7 +234,7 @@ func lexMainSection(l *pageLexer) stateFunc {
 					l.emit(tText)
 				}
 				l.summaryDividerChecked = true
-				l.pos += pos(len(summaryDivider))
+				l.pos += Pos(len(summaryDivider))
 				//l.consumeCRLF()
 				l.emit(TypeLeadSummaryDivider)
 			} else if l.hasPrefix(summaryDividerOrg) {
@@ -242,7 +242,7 @@ func lexMainSection(l *pageLexer) stateFunc {
 					l.emit(tText)
 				}
 				l.summaryDividerChecked = true
-				l.pos += pos(len(summaryDividerOrg))
+				l.pos += Pos(len(summaryDividerOrg))
 				//l.consumeCRLF()
 				l.emit(TypeSummaryDividerOrg)
 			}
@@ -291,12 +291,12 @@ LOOP:
 					if right == -1 {
 						return l.errorf("starting HTML comment with no end")
 					}
-					l.pos += pos(right) + pos(len(htmlCOmmentEnd))
+					l.pos += Pos(right) + Pos(len(htmlCOmmentEnd))
 					l.emit(TypeHTMLComment)
 				} else {
 					// Not need to look further. Hugo treats this as plain HTML,
 					// no front matter, no shortcodes, no nothing.
-					l.pos = pos(len(l.input))
+					l.pos = Pos(len(l.input))
 					l.emit(TypeHTMLDocument)
 				}
 			}
@@ -434,7 +434,7 @@ func (l *pageLexer) lexFrontMatterSection(tp ItemType, delimr rune, name string,
 }
 
 func lexShortcodeLeftDelim(l *pageLexer) stateFunc {
-	l.pos += pos(len(l.currentLeftShortcodeDelim()))
+	l.pos += Pos(len(l.currentLeftShortcodeDelim()))
 	if l.hasPrefix(leftComment) {
 		return lexShortcodeComment
 	}
@@ -451,20 +451,20 @@ func lexShortcodeComment(l *pageLexer) stateFunc {
 	}
 	// we emit all as text, except the comment markers
 	l.emit(tText)
-	l.pos += pos(len(leftComment))
+	l.pos += Pos(len(leftComment))
 	l.ignore()
-	l.pos += pos(posRightComment - len(leftComment))
+	l.pos += Pos(posRightComment - len(leftComment))
 	l.emit(tText)
-	l.pos += pos(len(rightComment))
+	l.pos += Pos(len(rightComment))
 	l.ignore()
-	l.pos += pos(len(l.currentRightShortcodeDelim()))
+	l.pos += Pos(len(l.currentRightShortcodeDelim()))
 	l.emit(tText)
 	return lexMainSection
 }
 
 func lexShortcodeRightDelim(l *pageLexer) stateFunc {
 	l.closingState = 0
-	l.pos += pos(len(l.currentRightShortcodeDelim()))
+	l.pos += Pos(len(l.currentRightShortcodeDelim()))
 	l.emit(l.currentRightShortcodeDelimItem())
 	return lexMainSection
 }

parser/pageparser/pageparser.go

@@ -48,7 +48,7 @@ func Parse(r io.Reader) (Result, error) {
 }
 
 func parseMainSection(input []byte, from int) Result {
-	lexer := newPageLexer(input, pos(from), lexMainSection) // TODO(bep) 2errors
+	lexer := newPageLexer(input, Pos(from), lexMainSection) // TODO(bep) 2errors
 	lexer.run()
 	return lexer
 }
@@ -57,7 +57,7 @@ func parseMainSection(input []byte, from int) Result {
 // if needed.
 type Iterator struct {
 	l       *pageLexer
-	lastPos pos // position of the last item returned by nextItem
+	lastPos Pos // position of the last item returned by nextItem
 }
 
 // consumes and returns the next item
@@ -69,7 +69,7 @@ func (t *Iterator) Next() Item {
 var errIndexOutOfBounds = Item{tError, 0, []byte("no more tokens")}
 
 func (t *Iterator) current() Item {
-	if t.lastPos >= pos(len(t.l.items)) {
+	if t.lastPos >= Pos(len(t.l.items)) {
 		return errIndexOutOfBounds
 	}
 	return t.l.items[t.lastPos]
@@ -98,7 +98,7 @@ func (t *Iterator) Peek() Item {
 // PeekWalk will feed the next items in the iterator to walkFn
 // until it returns false.
 func (t *Iterator) PeekWalk(walkFn func(item Item) bool) {
-	for i := t.lastPos + 1; i < pos(len(t.l.items)); i++ {
+	for i := t.lastPos + 1; i < Pos(len(t.l.items)); i++ {
 		item := t.l.items[i]
 		if !walkFn(item) {
 			break
@@ -120,5 +120,5 @@ func (t *Iterator) Consume(cnt int) {
 
 // LineNumber returns the current line number. Used for logging.
 func (t *Iterator) LineNumber() int {
-	return bytes.Count(t.l.input[:t.current().pos], lf) + 1
+	return bytes.Count(t.l.input[:t.current().Pos], lf) + 1
 }

parser/pageparser/pageparser_intro_test.go

@@ -59,9 +59,7 @@ var frontMatterTests = []lexerTest{
 	{"No front matter", "\nSome text.\n", []Item{tstSomeText, tstEOF}},
 	{"YAML front matter", "---\nfoo: \"bar\"\n---\n\nSome text.\n", []Item{tstFrontMatterYAML, tstSomeText, tstEOF}},
 	{"YAML empty front matter", "---\n---\n\nSome text.\n", []Item{nti(TypeFrontMatterYAML, "\n"), tstSomeText, tstEOF}},
-
 	{"YAML commented out front matter", "<!--\n---\nfoo: \"bar\"\n---\n-->\nSome text.\n", []Item{nti(TypeHTMLComment, "<!--\n---\nfoo: \"bar\"\n---\n-->"), tstSomeText, tstEOF}},
-
 	// Note that we keep all bytes as they are, but we need to handle CRLF
 	{"YAML front matter CRLF", "---\r\nfoo: \"bar\"\r\n---\n\nSome text.\n", []Item{tstFrontMatterYAMLCRLF, tstSomeText, tstEOF}},
 	{"TOML front matter", "+++\nfoo = \"bar\"\n+++\n\nSome text.\n", []Item{tstFrontMatterTOML, tstSomeText, tstEOF}},

tpl/transform/remarshal.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/gohugoio/hugo/parser"
+	"github.com/gohugoio/hugo/parser/metadecoders"
 	"github.com/spf13/cast"
 )
 
@@ -38,21 +39,7 @@ func (ns *Namespace) Remarshal(format string, data interface{}) (string, error)
 		return "", err
 	}
 
-	var metaHandler func(d []byte) (map[string]interface{}, error)
+	meta, err := metadecoders.UnmarshalToMap([]byte(from), fromFormat)
-
-	switch fromFormat {
-	case "yaml":
-		metaHandler = parser.HandleYAMLMetaData
-	case "toml":
-		metaHandler = parser.HandleTOMLMetaData
-	case "json":
-		metaHandler = parser.HandleJSONMetaData
-	}
-
-	meta, err := metaHandler([]byte(from))
-	if err != nil {
-		return "", err
-	}
 
 	var result bytes.Buffer
 	if err := parser.InterfaceToConfig(meta, mark, &result); err != nil {
@@ -76,21 +63,21 @@ func toFormatMark(format string) (rune, error) {
 	return 0, errors.New("failed to detect target data serialization format")
 }
 
-func detectFormat(data string) (string, error) {
+func detectFormat(data string) (metadecoders.Format, error) {
 	jsonIdx := strings.Index(data, "{")
 	yamlIdx := strings.Index(data, ":")
 	tomlIdx := strings.Index(data, "=")
 
 	if jsonIdx != -1 && (yamlIdx == -1 || jsonIdx < yamlIdx) && (tomlIdx == -1 || jsonIdx < tomlIdx) {
-		return "json", nil
+		return metadecoders.JSON, nil
 	}
 
 	if yamlIdx != -1 && (tomlIdx == -1 || yamlIdx < tomlIdx) {
-		return "yaml", nil
+		return metadecoders.YAML, nil
 	}
 
 	if tomlIdx != -1 {
-		return "toml", nil
+		return metadecoders.TOML, nil
 	}
 
 	return "", errors.New("failed to detect data serialization format")

tpl/transform/remarshal_test.go

@@ -18,6 +18,7 @@ import (
 	"testing"
 
 	"github.com/gohugoio/hugo/helpers"
+	"github.com/gohugoio/hugo/parser/metadecoders"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/require"
 )
@@ -179,12 +180,12 @@ func TestRemarshalDetectFormat(t *testing.T) {
 		data   string
 		expect interface{}
 	}{
-		{`foo = "bar"`, "toml"},
+		{`foo = "bar"`, metadecoders.TOML},
-		{`   foo = "bar"`, "toml"},
+		{`   foo = "bar"`, metadecoders.TOML},
-		{`foo="bar"`, "toml"},
+		{`foo="bar"`, metadecoders.TOML},
-		{`foo: "bar"`, "yaml"},
+		{`foo: "bar"`, metadecoders.YAML},
-		{`foo:"bar"`, "yaml"},
+		{`foo:"bar"`, metadecoders.YAML},
-		{`{ "foo": "bar"`, "json"},
+		{`{ "foo": "bar"`, metadecoders.JSON},
 		{`asdfasdf`, false},
 		{``, false},
 	} {
@@ -198,6 +199,6 @@ func TestRemarshalDetectFormat(t *testing.T) {
 		}
 
 		assert.NoError(err, errMsg)
-		assert.Equal(test.expect, result, errMsg)
+		assert.Equal(test.expect, result)
 	}
 }