mirror of
https://github.com/gohugoio/hugo.git
synced 2025-04-27 06:00:25 +03:00
✨ Implement Page bundling and image handling
This commit is not the smallest in Hugo's history. Some hightlights include: * Page bundles (for complete articles, keeping images and content together etc.). * Bundled images can be processed in as many versions/sizes as you need with the three methods `Resize`, `Fill` and `Fit`. * Processed images are cached inside `resources/_gen/images` (default) in your project. * Symbolic links (both files and dirs) are now allowed anywhere inside /content * A new table based build summary * The "Total in nn ms" now reports the total including the handling of the files inside /static. So if it now reports more than you're used to, it is just **more real** and probably faster than before (see below). A site building benchmark run compared to `v0.31.1` shows that this should be slightly faster and use less memory: ```bash ▶ ./benchSite.sh "TOML,num_langs=.*,num_root_sections=5,num_pages=(500|1000),tags_per_page=5,shortcodes,render" benchmark old ns/op new ns/op delta BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 101785785 78067944 -23.30% BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 185481057 149159919 -19.58% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 103149918 85679409 -16.94% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 203515478 169208775 -16.86% benchmark old allocs new allocs delta BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 532464 391539 -26.47% BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 1056549 772702 -26.87% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 555974 406630 -26.86% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 1086545 789922 -27.30% benchmark old bytes new bytes delta BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 53243246 43598155 -18.12% BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 105811617 86087116 -18.64% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 54558852 44545097 -18.35% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 106903858 86978413 -18.64% ``` Fixes #3651 Closes #3158 Fixes #1014 Closes #2021 Fixes #1240 Updates #3757
This commit is contained in:
Bjørn Erik Pedersen
parent
02f2735f68
commit
3cdf19e9b7
85 changed files with 5791 additions and 3287 deletions
|
|
@ -14,6 +14,7 @@
|
|||
package source
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/gohugoio/hugo/hugofs"
|
||||
|
|
@ -41,21 +42,21 @@ func TestIgnoreDotFilesAndDirectories(t *testing.T) {
|
|||
{"foobar/bar~foo.md", false, nil},
|
||||
{"foobar/foo.md", true, []string{"\\.md$", "\\.boo$"}},
|
||||
{"foobar/foo.html", false, []string{"\\.md$", "\\.boo$"}},
|
||||
{"foobar/foo.md", true, []string{"^foo"}},
|
||||
{"foobar/foo.md", false, []string{"*", "\\.md$", "\\.boo$"}},
|
||||
{"foobar/foo.md", true, []string{"foo.md$"}},
|
||||
{"foobar/foo.md", true, []string{"*", "\\.md$", "\\.boo$"}},
|
||||
{"foobar/.#content.md", true, []string{"/\\.#"}},
|
||||
{".#foobar.md", true, []string{"^\\.#"}},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
for i, test := range tests {
|
||||
|
||||
v := viper.New()
|
||||
v.Set("ignoreFiles", test.ignoreFilesRegexpes)
|
||||
|
||||
s := NewSourceSpec(v, hugofs.NewMem(v))
|
||||
|
||||
if ignored := s.isNonProcessablePath(test.path); test.ignore != ignored {
|
||||
t.Errorf("File not ignored. Expected: %t, got: %t", test.ignore, ignored)
|
||||
if ignored := s.IgnoreFile(filepath.FromSlash(test.path)); test.ignore != ignored {
|
||||
t.Errorf("[%d] File not ignored", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ type Dirs struct {
|
|||
staticDirs []string
|
||||
AbsStaticDirs []string
|
||||
|
||||
publishDir string
|
||||
Language *helpers.Language
|
||||
}
|
||||
|
||||
// NewDirs creates a new dirs with the given configuration and filesystem.
|
||||
|
|
@ -48,7 +48,12 @@ func NewDirs(fs *hugofs.Fs, cfg config.Provider, logger *jww.Notepad) (*Dirs, er
|
|||
return nil, err
|
||||
}
|
||||
|
||||
d := &Dirs{pathSpec: ps, logger: logger}
|
||||
var l *helpers.Language
|
||||
if language, ok := cfg.(*helpers.Language); ok {
|
||||
l = language
|
||||
}
|
||||
|
||||
d := &Dirs{Language: l, pathSpec: ps, logger: logger}
|
||||
|
||||
return d, d.init(cfg)
|
||||
|
||||
|
|
@ -96,8 +101,6 @@ func (d *Dirs) init(cfg config.Provider) error {
|
|||
d.AbsStaticDirs[i] = d.pathSpec.AbsPathify(di) + helpers.FilePathSeparator
|
||||
}
|
||||
|
||||
d.publishDir = d.pathSpec.AbsPathify(cfg.GetString("publishDir")) + helpers.FilePathSeparator
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
172
source/file.go
172
source/file.go
|
|
@ -1,172 +0,0 @@
|
|||
// Copyright 2015 The Hugo Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package source
|
||||
|
||||
import (
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/gohugoio/hugo/hugofs"
|
||||
|
||||
"github.com/gohugoio/hugo/config"
|
||||
"github.com/gohugoio/hugo/helpers"
|
||||
)
|
||||
|
||||
// SourceSpec abstracts language-specific file creation.
|
||||
type SourceSpec struct {
|
||||
Cfg config.Provider
|
||||
Fs *hugofs.Fs
|
||||
|
||||
languages map[string]interface{}
|
||||
defaultContentLanguage string
|
||||
}
|
||||
|
||||
// NewSourceSpec initializes SourceSpec using languages from a given configuration.
|
||||
func NewSourceSpec(cfg config.Provider, fs *hugofs.Fs) SourceSpec {
|
||||
defaultLang := cfg.GetString("defaultContentLanguage")
|
||||
languages := cfg.GetStringMap("languages")
|
||||
return SourceSpec{Cfg: cfg, Fs: fs, languages: languages, defaultContentLanguage: defaultLang}
|
||||
}
|
||||
|
||||
// File represents a source content file.
|
||||
// All paths are relative from the source directory base
|
||||
type File struct {
|
||||
relpath string // Original relative path, e.g. section/foo.txt
|
||||
logicalName string // foo.txt
|
||||
baseName string // `post` for `post.md`, also `post.en` for `post.en.md`
|
||||
Contents io.Reader
|
||||
section string // The first directory
|
||||
dir string // The relative directory Path (minus file name)
|
||||
ext string // Just the ext (eg txt)
|
||||
uniqueID string // MD5 of the file's path
|
||||
|
||||
translationBaseName string // `post` for `post.es.md` (if `Multilingual` is enabled.)
|
||||
lang string // The language code if `Multilingual` is enabled
|
||||
}
|
||||
|
||||
// UniqueID is the MD5 hash of the file's path and is for most practical applications,
|
||||
// Hugo content files being one of them, considered to be unique.
|
||||
func (f *File) UniqueID() string {
|
||||
return f.uniqueID
|
||||
}
|
||||
|
||||
// String returns the file's content as a string.
|
||||
func (f *File) String() string {
|
||||
return helpers.ReaderToString(f.Contents)
|
||||
}
|
||||
|
||||
// Bytes returns the file's content as a byte slice.
|
||||
func (f *File) Bytes() []byte {
|
||||
return helpers.ReaderToBytes(f.Contents)
|
||||
}
|
||||
|
||||
// BaseFileName is a filename without extension.
|
||||
func (f *File) BaseFileName() string {
|
||||
return f.baseName
|
||||
}
|
||||
|
||||
// TranslationBaseName is a filename with no extension,
|
||||
// not even the optional language extension part.
|
||||
func (f *File) TranslationBaseName() string {
|
||||
return f.translationBaseName
|
||||
}
|
||||
|
||||
// Lang for this page, if `Multilingual` is enabled on your site.
|
||||
func (f *File) Lang() string {
|
||||
return f.lang
|
||||
}
|
||||
|
||||
// Section is first directory below the content root.
|
||||
func (f *File) Section() string {
|
||||
return f.section
|
||||
}
|
||||
|
||||
// LogicalName is filename and extension of the file.
|
||||
func (f *File) LogicalName() string {
|
||||
return f.logicalName
|
||||
}
|
||||
|
||||
// SetDir sets the relative directory where this file lives.
|
||||
// TODO(bep) Get rid of this.
|
||||
func (f *File) SetDir(dir string) {
|
||||
f.dir = dir
|
||||
}
|
||||
|
||||
// Dir gets the name of the directory that contains this file.
|
||||
// The directory is relative to the content root.
|
||||
func (f *File) Dir() string {
|
||||
return f.dir
|
||||
}
|
||||
|
||||
// Extension gets the file extension, i.e "myblogpost.md" will return "md".
|
||||
func (f *File) Extension() string {
|
||||
return f.ext
|
||||
}
|
||||
|
||||
// Ext is an alias for Extension.
|
||||
func (f *File) Ext() string {
|
||||
return f.Extension()
|
||||
}
|
||||
|
||||
// Path gets the relative path including file name and extension.
|
||||
// The directory is relative to the content root.
|
||||
func (f *File) Path() string {
|
||||
return f.relpath
|
||||
}
|
||||
|
||||
// NewFileWithContents creates a new File pointer with the given relative path and
|
||||
// content. The language defaults to "en".
|
||||
func (sp SourceSpec) NewFileWithContents(relpath string, content io.Reader) *File {
|
||||
file := sp.NewFile(relpath)
|
||||
file.Contents = content
|
||||
file.lang = "en"
|
||||
return file
|
||||
}
|
||||
|
||||
// NewFile creates a new File pointer with the given relative path.
|
||||
func (sp SourceSpec) NewFile(relpath string) *File {
|
||||
f := &File{
|
||||
relpath: relpath,
|
||||
}
|
||||
|
||||
f.dir, f.logicalName = filepath.Split(f.relpath)
|
||||
f.ext = strings.TrimPrefix(filepath.Ext(f.LogicalName()), ".")
|
||||
f.baseName = helpers.Filename(f.LogicalName())
|
||||
|
||||
lang := strings.TrimPrefix(filepath.Ext(f.baseName), ".")
|
||||
if _, ok := sp.languages[lang]; lang == "" || !ok {
|
||||
f.lang = sp.defaultContentLanguage
|
||||
f.translationBaseName = f.baseName
|
||||
} else {
|
||||
f.lang = lang
|
||||
f.translationBaseName = helpers.Filename(f.baseName)
|
||||
}
|
||||
|
||||
f.section = helpers.GuessSection(f.Dir())
|
||||
f.uniqueID = helpers.Md5String(filepath.ToSlash(f.relpath))
|
||||
|
||||
return f
|
||||
}
|
||||
|
||||
// NewFileFromAbs creates a new File pointer with the given full file path path and
|
||||
// content.
|
||||
func (sp SourceSpec) NewFileFromAbs(base, fullpath string, content io.Reader) (f *File, err error) {
|
||||
var name string
|
||||
if name, err = helpers.GetRelativePath(fullpath, base); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return sp.NewFileWithContents(name, content), nil
|
||||
}
|
||||
213
source/fileInfo.go
Normal file
213
source/fileInfo.go
Normal file
|
|
@ -0,0 +1,213 @@
|
|||
// Copyright 2017-present The Hugo Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package source
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/gohugoio/hugo/helpers"
|
||||
)
|
||||
|
||||
// fileInfo implements the File interface.
|
||||
var (
|
||||
_ File = (*FileInfo)(nil)
|
||||
_ ReadableFile = (*FileInfo)(nil)
|
||||
)
|
||||
|
||||
type File interface {
|
||||
|
||||
// Filename gets the full path and filename to the file.
|
||||
Filename() string
|
||||
|
||||
// Path gets the relative path including file name and extension.
|
||||
// The directory is relative to the content root.
|
||||
Path() string
|
||||
|
||||
// Dir gets the name of the directory that contains this file.
|
||||
// The directory is relative to the content root.
|
||||
Dir() string
|
||||
|
||||
// Extension gets the file extension, i.e "myblogpost.md" will return "md".
|
||||
Extension() string
|
||||
// Ext is an alias for Extension.
|
||||
Ext() string // Hmm... Deprecate Extension
|
||||
|
||||
// Lang for this page, if `Multilingual` is enabled on your site.
|
||||
Lang() string
|
||||
|
||||
// LogicalName is filename and extension of the file.
|
||||
LogicalName() string
|
||||
|
||||
// Section is first directory below the content root.
|
||||
Section() string
|
||||
|
||||
// BaseFileName is a filename without extension.
|
||||
BaseFileName() string
|
||||
|
||||
// TranslationBaseName is a filename with no extension,
|
||||
// not even the optional language extension part.
|
||||
TranslationBaseName() string
|
||||
|
||||
// UniqueID is the MD5 hash of the file's path and is for most practical applications,
|
||||
// Hugo content files being one of them, considered to be unique.
|
||||
UniqueID() string
|
||||
|
||||
FileInfo() os.FileInfo
|
||||
|
||||
String() string
|
||||
|
||||
// Deprecated
|
||||
Bytes() []byte
|
||||
}
|
||||
|
||||
// A ReadableFile is a File that is readable.
|
||||
type ReadableFile interface {
|
||||
File
|
||||
Open() (io.ReadCloser, error)
|
||||
}
|
||||
|
||||
type FileInfo struct {
|
||||
|
||||
// Absolute filename to the file on disk.
|
||||
filename string
|
||||
fi os.FileInfo
|
||||
|
||||
// Derived from filename
|
||||
ext string // Extension without any "."
|
||||
lang string
|
||||
|
||||
name string
|
||||
|
||||
dir string
|
||||
relDir string
|
||||
relPath string
|
||||
baseName string
|
||||
translationBaseName string
|
||||
section string
|
||||
|
||||
uniqueID string
|
||||
|
||||
sp *SourceSpec
|
||||
|
||||
lazyInit sync.Once
|
||||
}
|
||||
|
||||
func (fi *FileInfo) Filename() string { return fi.filename }
|
||||
func (fi *FileInfo) Path() string { return fi.relPath }
|
||||
func (fi *FileInfo) Dir() string { return fi.relDir }
|
||||
func (fi *FileInfo) Extension() string { return fi.Ext() }
|
||||
func (fi *FileInfo) Ext() string { return fi.ext }
|
||||
func (fi *FileInfo) Lang() string { return fi.lang }
|
||||
func (fi *FileInfo) LogicalName() string { return fi.name }
|
||||
func (fi *FileInfo) BaseFileName() string { return fi.baseName }
|
||||
func (fi *FileInfo) TranslationBaseName() string { return fi.translationBaseName }
|
||||
|
||||
func (fi *FileInfo) Section() string {
|
||||
fi.init()
|
||||
return fi.section
|
||||
}
|
||||
|
||||
func (fi *FileInfo) UniqueID() string {
|
||||
fi.init()
|
||||
return fi.uniqueID
|
||||
}
|
||||
func (fi *FileInfo) FileInfo() os.FileInfo {
|
||||
return fi.fi
|
||||
}
|
||||
|
||||
func (fi *FileInfo) Bytes() []byte {
|
||||
// Remove in Hugo 0.34
|
||||
helpers.Deprecated("File", "Bytes", "", false)
|
||||
return []byte("")
|
||||
}
|
||||
|
||||
func (fi *FileInfo) String() string { return fi.BaseFileName() }
|
||||
|
||||
// We create a lot of these FileInfo objects, but there are parts of it used only
|
||||
// in some cases that is slightly expensive to construct.
|
||||
func (fi *FileInfo) init() {
|
||||
fi.lazyInit.Do(func() {
|
||||
parts := strings.Split(fi.relDir, helpers.FilePathSeparator)
|
||||
var section string
|
||||
if len(parts) == 1 {
|
||||
section = parts[0]
|
||||
} else if len(parts) > 1 {
|
||||
if parts[0] == "" {
|
||||
section = parts[1]
|
||||
} else {
|
||||
section = parts[0]
|
||||
}
|
||||
}
|
||||
|
||||
fi.section = section
|
||||
|
||||
fi.uniqueID = helpers.MD5String(filepath.ToSlash(fi.relPath))
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
func (sp *SourceSpec) NewFileInfo(baseDir, filename string, fi os.FileInfo) *FileInfo {
|
||||
dir, name := filepath.Split(filename)
|
||||
|
||||
dir = strings.TrimSuffix(dir, helpers.FilePathSeparator)
|
||||
baseDir = strings.TrimSuffix(baseDir, helpers.FilePathSeparator)
|
||||
|
||||
relDir := ""
|
||||
if dir != baseDir {
|
||||
relDir = strings.TrimPrefix(dir, baseDir)
|
||||
}
|
||||
|
||||
relDir = strings.TrimPrefix(relDir, helpers.FilePathSeparator)
|
||||
|
||||
relPath := filepath.Join(relDir, name)
|
||||
|
||||
ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(name), "."))
|
||||
baseName := helpers.Filename(name)
|
||||
|
||||
lang := strings.TrimPrefix(filepath.Ext(baseName), ".")
|
||||
var translationBaseName string
|
||||
|
||||
if _, ok := sp.Languages[lang]; lang == "" || !ok {
|
||||
lang = sp.DefaultContentLanguage
|
||||
translationBaseName = baseName
|
||||
} else {
|
||||
translationBaseName = helpers.Filename(baseName)
|
||||
}
|
||||
|
||||
f := &FileInfo{
|
||||
sp: sp,
|
||||
filename: filename,
|
||||
fi: fi,
|
||||
lang: lang,
|
||||
ext: ext,
|
||||
dir: dir,
|
||||
relDir: relDir,
|
||||
relPath: relPath,
|
||||
name: name,
|
||||
baseName: baseName,
|
||||
translationBaseName: translationBaseName,
|
||||
}
|
||||
|
||||
return f
|
||||
|
||||
}
|
||||
|
||||
// Open implements ReadableFile.
|
||||
func (fi *FileInfo) Open() (io.ReadCloser, error) {
|
||||
return fi.sp.Fs.Source.Open(fi.Filename())
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
// Copyright 2015 The Hugo Authors. All rights reserved.
|
||||
// Copyright 2017-present The Hugo Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
|
@ -13,13 +13,10 @@
|
|||
|
||||
package source
|
||||
|
||||
// ByteSource represents a source's name and content.
|
||||
// It's currently only used for testing purposes.
|
||||
type ByteSource struct {
|
||||
Name string
|
||||
Content []byte
|
||||
}
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFileInfo(t *testing.T) {
|
||||
|
||||
func (b *ByteSource) String() string {
|
||||
return b.Name + " " + string(b.Content)
|
||||
}
|
||||
|
|
@ -1,62 +0,0 @@
|
|||
// Copyright 2015 The Hugo Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package source
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/gohugoio/hugo/hugofs"
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestFileUniqueID(t *testing.T) {
|
||||
ss := newTestSourceSpec()
|
||||
|
||||
f1 := File{uniqueID: "123"}
|
||||
f2 := ss.NewFile("a")
|
||||
|
||||
assert.Equal(t, "123", f1.UniqueID())
|
||||
assert.Equal(t, "0cc175b9c0f1b6a831c399e269772661", f2.UniqueID())
|
||||
|
||||
f3 := ss.NewFile(filepath.FromSlash("test1/index.md"))
|
||||
f4 := ss.NewFile(filepath.FromSlash("test2/index.md"))
|
||||
|
||||
assert.NotEqual(t, f3.UniqueID(), f4.UniqueID())
|
||||
|
||||
f5l := ss.NewFile("test3/index.md")
|
||||
f5w := ss.NewFile(filepath.FromSlash("test3/index.md"))
|
||||
|
||||
assert.Equal(t, f5l.UniqueID(), f5w.UniqueID())
|
||||
}
|
||||
|
||||
func TestFileString(t *testing.T) {
|
||||
ss := newTestSourceSpec()
|
||||
assert.Equal(t, "abc", ss.NewFileWithContents("a", strings.NewReader("abc")).String())
|
||||
assert.Equal(t, "", ss.NewFile("a").String())
|
||||
}
|
||||
|
||||
func TestFileBytes(t *testing.T) {
|
||||
ss := newTestSourceSpec()
|
||||
assert.Equal(t, []byte("abc"), ss.NewFileWithContents("a", strings.NewReader("abc")).Bytes())
|
||||
assert.Equal(t, []byte(""), ss.NewFile("a").Bytes())
|
||||
}
|
||||
|
||||
func newTestSourceSpec() SourceSpec {
|
||||
v := viper.New()
|
||||
return SourceSpec{Fs: hugofs.NewMem(v), Cfg: v}
|
||||
}
|
||||
|
|
@ -14,73 +14,52 @@
|
|||
package source
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/gohugoio/hugo/helpers"
|
||||
"github.com/spf13/cast"
|
||||
jww "github.com/spf13/jwalterweatherman"
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
type Input interface {
|
||||
Files() []*File
|
||||
}
|
||||
|
||||
type Filesystem struct {
|
||||
files []*File
|
||||
Base string
|
||||
AvoidPaths []string
|
||||
files []ReadableFile
|
||||
filesInit sync.Once
|
||||
|
||||
Base string
|
||||
|
||||
SourceSpec
|
||||
}
|
||||
|
||||
func (sp SourceSpec) NewFilesystem(base string, avoidPaths ...string) *Filesystem {
|
||||
return &Filesystem{SourceSpec: sp, Base: base, AvoidPaths: avoidPaths}
|
||||
type Input interface {
|
||||
Files() []ReadableFile
|
||||
}
|
||||
|
||||
func (f *Filesystem) FilesByExts(exts ...string) []*File {
|
||||
var newFiles []*File
|
||||
|
||||
if len(exts) == 0 {
|
||||
return f.Files()
|
||||
}
|
||||
|
||||
for _, x := range f.Files() {
|
||||
for _, e := range exts {
|
||||
if x.Ext() == strings.TrimPrefix(e, ".") {
|
||||
newFiles = append(newFiles, x)
|
||||
}
|
||||
}
|
||||
}
|
||||
return newFiles
|
||||
func (sp SourceSpec) NewFilesystem(base string) *Filesystem {
|
||||
return &Filesystem{SourceSpec: sp, Base: base}
|
||||
}
|
||||
|
||||
func (f *Filesystem) Files() []*File {
|
||||
if len(f.files) < 1 {
|
||||
func (f *Filesystem) Files() []ReadableFile {
|
||||
f.filesInit.Do(func() {
|
||||
f.captureFiles()
|
||||
}
|
||||
})
|
||||
return f.files
|
||||
}
|
||||
|
||||
// add populates a file in the Filesystem.files
|
||||
func (f *Filesystem) add(name string, reader io.Reader) (err error) {
|
||||
var file *File
|
||||
func (f *Filesystem) add(name string, fi os.FileInfo) (err error) {
|
||||
var file ReadableFile
|
||||
|
||||
if runtime.GOOS == "darwin" {
|
||||
// When a file system is HFS+, its filepath is in NFD form.
|
||||
name = norm.NFC.String(name)
|
||||
}
|
||||
|
||||
file, err = f.SourceSpec.NewFileFromAbs(f.Base, name, reader)
|
||||
file = f.SourceSpec.NewFileInfo(f.Base, name, fi)
|
||||
f.files = append(f.files, file)
|
||||
|
||||
if err == nil {
|
||||
f.files = append(f.files, file)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
@ -90,16 +69,12 @@ func (f *Filesystem) captureFiles() {
|
|||
return nil
|
||||
}
|
||||
|
||||
b, err := f.ShouldRead(filePath, fi)
|
||||
b, err := f.shouldRead(filePath, fi)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if b {
|
||||
rd, err := NewLazyFileReader(f.Fs.Source, filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.add(filePath, rd)
|
||||
f.add(filePath, fi)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
|
@ -118,11 +93,11 @@ func (f *Filesystem) captureFiles() {
|
|||
|
||||
}
|
||||
|
||||
func (f *Filesystem) ShouldRead(filePath string, fi os.FileInfo) (bool, error) {
|
||||
func (f *Filesystem) shouldRead(filename string, fi os.FileInfo) (bool, error) {
|
||||
if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
|
||||
link, err := filepath.EvalSymlinks(filePath)
|
||||
link, err := filepath.EvalSymlinks(filename)
|
||||
if err != nil {
|
||||
jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filePath, err)
|
||||
jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filename, err)
|
||||
return false, nil
|
||||
}
|
||||
linkfi, err := f.Fs.Source.Stat(link)
|
||||
|
|
@ -130,52 +105,25 @@ func (f *Filesystem) ShouldRead(filePath string, fi os.FileInfo) (bool, error) {
|
|||
jww.ERROR.Printf("Cannot stat '%s', error was: %s", link, err)
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if !linkfi.Mode().IsRegular() {
|
||||
jww.ERROR.Printf("Symbolic links for directories not supported, skipping '%s'", filePath)
|
||||
jww.ERROR.Printf("Symbolic links for directories not supported, skipping '%s'", filename)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
ignore := f.SourceSpec.IgnoreFile(filename)
|
||||
|
||||
if fi.IsDir() {
|
||||
if f.avoid(filePath) || f.isNonProcessablePath(filePath) {
|
||||
if ignore {
|
||||
return false, filepath.SkipDir
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if f.isNonProcessablePath(filePath) {
|
||||
if ignore {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (f *Filesystem) avoid(filePath string) bool {
|
||||
for _, avoid := range f.AvoidPaths {
|
||||
if avoid == filePath {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (sp SourceSpec) isNonProcessablePath(filePath string) bool {
|
||||
base := filepath.Base(filePath)
|
||||
if strings.HasPrefix(base, ".") ||
|
||||
strings.HasPrefix(base, "#") ||
|
||||
strings.HasSuffix(base, "~") {
|
||||
return true
|
||||
}
|
||||
ignoreFiles := cast.ToStringSlice(sp.Cfg.Get("ignoreFiles"))
|
||||
if len(ignoreFiles) > 0 {
|
||||
for _, ignorePattern := range ignoreFiles {
|
||||
match, err := regexp.MatchString(ignorePattern, filePath)
|
||||
if err != nil {
|
||||
helpers.DistinctErrorLog.Printf("Invalid regexp '%s' in ignoreFiles: %s", ignorePattern, err)
|
||||
return false
|
||||
} else if match {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,11 +14,13 @@
|
|||
package source
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"path/filepath"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/gohugoio/hugo/hugofs"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
func TestEmptySourceFilesystem(t *testing.T) {
|
||||
|
|
@ -37,54 +39,6 @@ type TestPath struct {
|
|||
dir string
|
||||
}
|
||||
|
||||
func TestAddFile(t *testing.T) {
|
||||
ss := newTestSourceSpec()
|
||||
tests := platformPaths
|
||||
for _, test := range tests {
|
||||
base := platformBase
|
||||
srcDefault := ss.NewFilesystem("")
|
||||
srcWithBase := ss.NewFilesystem(base)
|
||||
|
||||
for _, src := range []*Filesystem{srcDefault, srcWithBase} {
|
||||
|
||||
p := test.filename
|
||||
if !filepath.IsAbs(test.filename) {
|
||||
p = filepath.Join(src.Base, test.filename)
|
||||
}
|
||||
|
||||
if err := src.add(p, bytes.NewReader([]byte(test.content))); err != nil {
|
||||
if err.Error() == "source: missing base directory" {
|
||||
continue
|
||||
}
|
||||
t.Fatalf("%s add returned an error: %s", p, err)
|
||||
}
|
||||
|
||||
if len(src.Files()) != 1 {
|
||||
t.Fatalf("%s Files() should return 1 file", p)
|
||||
}
|
||||
|
||||
f := src.Files()[0]
|
||||
if f.LogicalName() != test.logical {
|
||||
t.Errorf("Filename (Base: %q) expected: %q, got: %q", src.Base, test.logical, f.LogicalName())
|
||||
}
|
||||
|
||||
b := new(bytes.Buffer)
|
||||
b.ReadFrom(f.Contents)
|
||||
if b.String() != test.content {
|
||||
t.Errorf("File (Base: %q) contents should be %q, got: %q", src.Base, test.content, b.String())
|
||||
}
|
||||
|
||||
if f.Section() != test.section {
|
||||
t.Errorf("File section (Base: %q) expected: %q, got: %q", src.Base, test.section, f.Section())
|
||||
}
|
||||
|
||||
if f.Dir() != test.dir {
|
||||
t.Errorf("Dir path (Base: %q) expected: %q, got: %q", src.Base, test.dir, f.Dir())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnicodeNorm(t *testing.T) {
|
||||
if runtime.GOOS != "darwin" {
|
||||
// Normalization code is only for Mac OS, since it is not necessary for other OSes.
|
||||
|
|
@ -100,10 +54,11 @@ func TestUnicodeNorm(t *testing.T) {
|
|||
}
|
||||
|
||||
ss := newTestSourceSpec()
|
||||
var fi os.FileInfo
|
||||
|
||||
for _, path := range paths {
|
||||
src := ss.NewFilesystem("")
|
||||
_ = src.add(path.NFD, strings.NewReader(""))
|
||||
src := ss.NewFilesystem("base")
|
||||
_ = src.add(path.NFD, fi)
|
||||
f := src.Files()[0]
|
||||
if f.BaseFileName() != path.NFC {
|
||||
t.Fatalf("file name in NFD form should be normalized (%s)", path.NFC)
|
||||
|
|
@ -111,3 +66,8 @@ func TestUnicodeNorm(t *testing.T) {
|
|||
}
|
||||
|
||||
}
|
||||
|
||||
func newTestSourceSpec() SourceSpec {
|
||||
v := viper.New()
|
||||
return SourceSpec{Fs: hugofs.NewMem(v), Cfg: v}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,170 +0,0 @@
|
|||
// Copyright 2015 The Hugo Authors. All rights reserved.
|
||||
// Portions Copyright 2009 The Go Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package source
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
// LazyFileReader is an io.Reader implementation to postpone reading the file
|
||||
// contents until it is really needed. It keeps filename and file contents once
|
||||
// it is read.
|
||||
type LazyFileReader struct {
|
||||
fs afero.Fs
|
||||
filename string
|
||||
contents *bytes.Reader
|
||||
pos int64
|
||||
}
|
||||
|
||||
// NewLazyFileReader creates and initializes a new LazyFileReader of filename.
|
||||
// It checks whether the file can be opened. If it fails, it returns nil and an
|
||||
// error.
|
||||
func NewLazyFileReader(fs afero.Fs, filename string) (*LazyFileReader, error) {
|
||||
f, err := fs.Open(filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
return &LazyFileReader{fs: fs, filename: filename, contents: nil, pos: 0}, nil
|
||||
}
|
||||
|
||||
// Filename returns a file name which LazyFileReader keeps
|
||||
func (l *LazyFileReader) Filename() string {
|
||||
return l.filename
|
||||
}
|
||||
|
||||
// Read reads up to len(p) bytes from the LazyFileReader's file and copies them
|
||||
// into p. It returns the number of bytes read and any error encountered. If
|
||||
// the file is once read, it returns its contents from cache, doesn't re-read
|
||||
// the file.
|
||||
func (l *LazyFileReader) Read(p []byte) (n int, err error) {
|
||||
if l.contents == nil {
|
||||
b, err := afero.ReadFile(l.fs, l.filename)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to read content from %s: %s", l.filename, err.Error())
|
||||
}
|
||||
l.contents = bytes.NewReader(b)
|
||||
}
|
||||
if _, err = l.contents.Seek(l.pos, 0); err != nil {
|
||||
return 0, errors.New("failed to set read position: " + err.Error())
|
||||
}
|
||||
n, err = l.contents.Read(p)
|
||||
l.pos += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Seek implements the io.Seeker interface. Once reader contents is consumed by
|
||||
// Read, WriteTo etc, to read it again, it must be rewinded by this function
|
||||
func (l *LazyFileReader) Seek(offset int64, whence int) (pos int64, err error) {
|
||||
if l.contents == nil {
|
||||
switch whence {
|
||||
case 0:
|
||||
pos = offset
|
||||
case 1:
|
||||
pos = l.pos + offset
|
||||
case 2:
|
||||
fi, err := l.fs.Stat(l.filename)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to get %q info: %s", l.filename, err.Error())
|
||||
}
|
||||
pos = fi.Size() + offset
|
||||
default:
|
||||
return 0, errors.New("invalid whence")
|
||||
}
|
||||
if pos < 0 {
|
||||
return 0, errors.New("negative position")
|
||||
}
|
||||
} else {
|
||||
pos, err = l.contents.Seek(offset, whence)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
l.pos = pos
|
||||
return pos, nil
|
||||
}
|
||||
|
||||
// WriteTo writes data to w until all the LazyFileReader's file contents is
|
||||
// drained or an error occurs. If the file is once read, it just writes its
|
||||
// read cache to w, doesn't re-read the file but this method itself doesn't try
|
||||
// to keep the contents in cache.
|
||||
func (l *LazyFileReader) WriteTo(w io.Writer) (n int64, err error) {
|
||||
if l.contents != nil {
|
||||
l.contents.Seek(l.pos, 0)
|
||||
if err != nil {
|
||||
return 0, errors.New("failed to set read position: " + err.Error())
|
||||
}
|
||||
n, err = l.contents.WriteTo(w)
|
||||
l.pos += n
|
||||
return n, err
|
||||
}
|
||||
f, err := l.fs.Open(l.filename)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to open %s to read content: %s", l.filename, err.Error())
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
fi, err := f.Stat()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to get %q info: %s", l.filename, err.Error())
|
||||
}
|
||||
|
||||
if l.pos >= fi.Size() {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
return l.copyBuffer(w, f, nil)
|
||||
}
|
||||
|
||||
// copyBuffer is the actual implementation of Copy and CopyBuffer.
|
||||
// If buf is nil, one is allocated.
|
||||
//
|
||||
// Most of this function is copied from the Go stdlib 'io/io.go'.
|
||||
func (l *LazyFileReader) copyBuffer(dst io.Writer, src io.Reader, buf []byte) (written int64, err error) {
|
||||
if buf == nil {
|
||||
buf = make([]byte, 32*1024)
|
||||
}
|
||||
for {
|
||||
nr, er := src.Read(buf)
|
||||
if nr > 0 {
|
||||
nw, ew := dst.Write(buf[0:nr])
|
||||
if nw > 0 {
|
||||
l.pos += int64(nw)
|
||||
written += int64(nw)
|
||||
}
|
||||
if ew != nil {
|
||||
err = ew
|
||||
break
|
||||
}
|
||||
if nr != nw {
|
||||
err = io.ErrShortWrite
|
||||
break
|
||||
}
|
||||
}
|
||||
if er == io.EOF {
|
||||
break
|
||||
}
|
||||
if er != nil {
|
||||
err = er
|
||||
break
|
||||
}
|
||||
}
|
||||
return written, err
|
||||
}
|
||||
|
|
@ -1,236 +0,0 @@
|
|||
// Copyright 2015 The Hugo Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package source
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
func TestNewLazyFileReader(t *testing.T) {
|
||||
fs := afero.NewOsFs()
|
||||
filename := "itdoesnotexistfile"
|
||||
_, err := NewLazyFileReader(fs, filename)
|
||||
if err == nil {
|
||||
t.Errorf("NewLazyFileReader %s: error expected but no error is returned", filename)
|
||||
}
|
||||
|
||||
filename = "lazy_file_reader_test.go"
|
||||
_, err = NewLazyFileReader(fs, filename)
|
||||
if err != nil {
|
||||
t.Errorf("NewLazyFileReader %s: %v", filename, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilename(t *testing.T) {
|
||||
fs := afero.NewOsFs()
|
||||
filename := "lazy_file_reader_test.go"
|
||||
rd, err := NewLazyFileReader(fs, filename)
|
||||
if err != nil {
|
||||
t.Fatalf("NewLazyFileReader %s: %v", filename, err)
|
||||
}
|
||||
if rd.Filename() != filename {
|
||||
t.Errorf("Filename: expected filename %q, got %q", filename, rd.Filename())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRead(t *testing.T) {
|
||||
fs := afero.NewOsFs()
|
||||
filename := "lazy_file_reader_test.go"
|
||||
fi, err := fs.Stat(filename)
|
||||
if err != nil {
|
||||
t.Fatalf("os.Stat: %v", err)
|
||||
}
|
||||
|
||||
b, err := afero.ReadFile(fs, filename)
|
||||
if err != nil {
|
||||
t.Fatalf("afero.ReadFile: %v", err)
|
||||
}
|
||||
|
||||
rd, err := NewLazyFileReader(fs, filename)
|
||||
if err != nil {
|
||||
t.Fatalf("NewLazyFileReader %s: %v", filename, err)
|
||||
}
|
||||
|
||||
tst := func(testcase string) {
|
||||
p := make([]byte, fi.Size())
|
||||
n, err := rd.Read(p)
|
||||
if err != nil {
|
||||
t.Fatalf("Read %s case: %v", testcase, err)
|
||||
}
|
||||
if int64(n) != fi.Size() {
|
||||
t.Errorf("Read %s case: read bytes length expected %d, got %d", testcase, fi.Size(), n)
|
||||
}
|
||||
if !bytes.Equal(b, p) {
|
||||
t.Errorf("Read %s case: read bytes are different from expected", testcase)
|
||||
}
|
||||
}
|
||||
tst("No cache")
|
||||
_, err = rd.Seek(0, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("Seek: %v", err)
|
||||
}
|
||||
tst("Cache")
|
||||
}
|
||||
|
||||
func TestSeek(t *testing.T) {
|
||||
type testcase struct {
|
||||
seek int
|
||||
offset int64
|
||||
length int
|
||||
moveto int64
|
||||
expected []byte
|
||||
}
|
||||
fs := afero.NewOsFs()
|
||||
filename := "lazy_file_reader_test.go"
|
||||
b, err := afero.ReadFile(fs, filename)
|
||||
if err != nil {
|
||||
t.Fatalf("afero.ReadFile: %v", err)
|
||||
}
|
||||
|
||||
// no cache case
|
||||
for i, this := range []testcase{
|
||||
{seek: os.SEEK_SET, offset: 0, length: 10, moveto: 0, expected: b[:10]},
|
||||
{seek: os.SEEK_SET, offset: 5, length: 10, moveto: 5, expected: b[5:15]},
|
||||
{seek: os.SEEK_CUR, offset: 5, length: 10, moveto: 5, expected: b[5:15]}, // current pos = 0
|
||||
{seek: os.SEEK_END, offset: -1, length: 1, moveto: int64(len(b) - 1), expected: b[len(b)-1:]},
|
||||
{seek: 3, expected: nil},
|
||||
{seek: os.SEEK_SET, offset: -1, expected: nil},
|
||||
} {
|
||||
rd, err := NewLazyFileReader(fs, filename)
|
||||
if err != nil {
|
||||
t.Errorf("[%d] NewLazyFileReader %s: %v", i, filename, err)
|
||||
continue
|
||||
}
|
||||
|
||||
pos, err := rd.Seek(this.offset, this.seek)
|
||||
if this.expected == nil {
|
||||
if err == nil {
|
||||
t.Errorf("[%d] Seek didn't return an expected error", i)
|
||||
}
|
||||
} else {
|
||||
if err != nil {
|
||||
t.Errorf("[%d] Seek failed unexpectedly: %v", i, err)
|
||||
continue
|
||||
}
|
||||
if pos != this.moveto {
|
||||
t.Errorf("[%d] Seek failed to move the pointer: got %d, expected: %d", i, pos, this.moveto)
|
||||
}
|
||||
|
||||
buf := make([]byte, this.length)
|
||||
n, err := rd.Read(buf)
|
||||
if err != nil {
|
||||
t.Errorf("[%d] Read failed unexpectedly: %v", i, err)
|
||||
}
|
||||
if !bytes.Equal(this.expected, buf[:n]) {
|
||||
t.Errorf("[%d] Seek and Read got %q but expected %q", i, buf[:n], this.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cache case
|
||||
rd, err := NewLazyFileReader(fs, filename)
|
||||
if err != nil {
|
||||
t.Fatalf("NewLazyFileReader %s: %v", filename, err)
|
||||
}
|
||||
dummy := make([]byte, len(b))
|
||||
_, err = rd.Read(dummy)
|
||||
if err != nil {
|
||||
t.Fatalf("Read failed unexpectedly: %v", err)
|
||||
}
|
||||
|
||||
for i, this := range []testcase{
|
||||
{seek: os.SEEK_SET, offset: 0, length: 10, moveto: 0, expected: b[:10]},
|
||||
{seek: os.SEEK_SET, offset: 5, length: 10, moveto: 5, expected: b[5:15]},
|
||||
{seek: os.SEEK_CUR, offset: 1, length: 10, moveto: 16, expected: b[16:26]}, // current pos = 15
|
||||
{seek: os.SEEK_END, offset: -1, length: 1, moveto: int64(len(b) - 1), expected: b[len(b)-1:]},
|
||||
{seek: 3, expected: nil},
|
||||
{seek: os.SEEK_SET, offset: -1, expected: nil},
|
||||
} {
|
||||
pos, err := rd.Seek(this.offset, this.seek)
|
||||
if this.expected == nil {
|
||||
if err == nil {
|
||||
t.Errorf("[%d] Seek didn't return an expected error", i)
|
||||
}
|
||||
} else {
|
||||
if err != nil {
|
||||
t.Errorf("[%d] Seek failed unexpectedly: %v", i, err)
|
||||
continue
|
||||
}
|
||||
if pos != this.moveto {
|
||||
t.Errorf("[%d] Seek failed to move the pointer: got %d, expected: %d", i, pos, this.moveto)
|
||||
}
|
||||
|
||||
buf := make([]byte, this.length)
|
||||
n, err := rd.Read(buf)
|
||||
if err != nil {
|
||||
t.Errorf("[%d] Read failed unexpectedly: %v", i, err)
|
||||
}
|
||||
if !bytes.Equal(this.expected, buf[:n]) {
|
||||
t.Errorf("[%d] Seek and Read got %q but expected %q", i, buf[:n], this.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteTo(t *testing.T) {
|
||||
fs := afero.NewOsFs()
|
||||
filename := "lazy_file_reader_test.go"
|
||||
fi, err := fs.Stat(filename)
|
||||
if err != nil {
|
||||
t.Fatalf("os.Stat: %v", err)
|
||||
}
|
||||
|
||||
b, err := afero.ReadFile(fs, filename)
|
||||
if err != nil {
|
||||
t.Fatalf("afero.ReadFile: %v", err)
|
||||
}
|
||||
|
||||
rd, err := NewLazyFileReader(fs, filename)
|
||||
if err != nil {
|
||||
t.Fatalf("NewLazyFileReader %s: %v", filename, err)
|
||||
}
|
||||
|
||||
tst := func(testcase string, expectedSize int64, checkEqual bool) {
|
||||
buf := bytes.NewBuffer(make([]byte, 0, bytes.MinRead))
|
||||
n, err := rd.WriteTo(buf)
|
||||
if err != nil {
|
||||
t.Fatalf("WriteTo %s case: %v", testcase, err)
|
||||
}
|
||||
if n != expectedSize {
|
||||
t.Errorf("WriteTo %s case: written bytes length expected %d, got %d", testcase, expectedSize, n)
|
||||
}
|
||||
if checkEqual && !bytes.Equal(b, buf.Bytes()) {
|
||||
t.Errorf("WriteTo %s case: written bytes are different from expected", testcase)
|
||||
}
|
||||
}
|
||||
tst("No cache", fi.Size(), true)
|
||||
tst("No cache 2nd", 0, false)
|
||||
|
||||
p := make([]byte, fi.Size())
|
||||
_, err = rd.Read(p)
|
||||
if err != nil && err != io.EOF {
|
||||
t.Fatalf("Read: %v", err)
|
||||
}
|
||||
_, err = rd.Seek(0, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("Seek: %v", err)
|
||||
}
|
||||
|
||||
tst("Cache", fi.Size(), true)
|
||||
}
|
||||
117
source/sourceSpec.go
Normal file
117
source/sourceSpec.go
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
// Copyright 2017-present The Hugo Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package source
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
|
||||
"github.com/gohugoio/hugo/config"
|
||||
"github.com/gohugoio/hugo/helpers"
|
||||
"github.com/gohugoio/hugo/hugofs"
|
||||
"github.com/spf13/cast"
|
||||
)
|
||||
|
||||
// SourceSpec abstracts language-specific file creation.
|
||||
// TODO(bep) rename to Spec
|
||||
type SourceSpec struct {
|
||||
Cfg config.Provider
|
||||
Fs *hugofs.Fs
|
||||
|
||||
// This is set if the ignoreFiles config is set.
|
||||
ignoreFilesRe []*regexp.Regexp
|
||||
|
||||
Languages map[string]interface{}
|
||||
DefaultContentLanguage string
|
||||
}
|
||||
|
||||
// NewSourceSpec initializes SourceSpec using languages from a given configuration.
|
||||
func NewSourceSpec(cfg config.Provider, fs *hugofs.Fs) *SourceSpec {
|
||||
defaultLang := cfg.GetString("defaultContentLanguage")
|
||||
languages := cfg.GetStringMap("languages")
|
||||
|
||||
if len(languages) == 0 {
|
||||
l := helpers.NewDefaultLanguage(cfg)
|
||||
languages[l.Lang] = l
|
||||
defaultLang = l.Lang
|
||||
}
|
||||
|
||||
ignoreFiles := cast.ToStringSlice(cfg.Get("ignoreFiles"))
|
||||
var regexps []*regexp.Regexp
|
||||
if len(ignoreFiles) > 0 {
|
||||
for _, ignorePattern := range ignoreFiles {
|
||||
re, err := regexp.Compile(ignorePattern)
|
||||
if err != nil {
|
||||
helpers.DistinctErrorLog.Printf("Invalid regexp %q in ignoreFiles: %s", ignorePattern, err)
|
||||
} else {
|
||||
regexps = append(regexps, re)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
return &SourceSpec{ignoreFilesRe: regexps, Cfg: cfg, Fs: fs, Languages: languages, DefaultContentLanguage: defaultLang}
|
||||
}
|
||||
|
||||
func (s *SourceSpec) IgnoreFile(filename string) bool {
|
||||
base := filepath.Base(filename)
|
||||
|
||||
if len(base) > 0 {
|
||||
first := base[0]
|
||||
last := base[len(base)-1]
|
||||
if first == '.' ||
|
||||
first == '#' ||
|
||||
last == '~' {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
if len(s.ignoreFilesRe) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, re := range s.ignoreFilesRe {
|
||||
if re.MatchString(filename) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *SourceSpec) IsRegularSourceFile(filename string) (bool, error) {
|
||||
fi, err := helpers.LstatIfOs(s.Fs.Source, filename)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if fi.IsDir() {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
|
||||
link, err := filepath.EvalSymlinks(filename)
|
||||
fi, err = helpers.LstatIfOs(s.Fs.Source, link)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if fi.IsDir() {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue