diff options
Diffstat (limited to 'hugolib/filesystems')
| -rw-r--r-- | hugolib/filesystems/basefs.go | 745 | ||||
| -rw-r--r-- | hugolib/filesystems/basefs_test.go | 460 |
2 files changed, 1205 insertions, 0 deletions
diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go new file mode 100644 index 000000000..57a95a037 --- /dev/null +++ b/hugolib/filesystems/basefs.go @@ -0,0 +1,745 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package filesystems provides the fine grained file systems used by Hugo. These +// are typically virtual filesystems that are composites of project and theme content. +package filesystems + +import ( + "io" + "os" + "path" + "path/filepath" + "strings" + "sync" + + "github.com/gohugoio/hugo/common/loggers" + + "github.com/gohugoio/hugo/hugofs/files" + + "github.com/pkg/errors" + + "github.com/gohugoio/hugo/modules" + + "github.com/gohugoio/hugo/hugofs" + + "fmt" + + "github.com/gohugoio/hugo/hugolib/paths" + "github.com/spf13/afero" +) + +var filePathSeparator = string(filepath.Separator) + +// BaseFs contains the core base filesystems used by Hugo. The name "base" is used +// to underline that even if they can be composites, they all have a base path set to a specific +// resource folder, e.g "/my-project/content". So, no absolute filenames needed. +type BaseFs struct { + + // SourceFilesystems contains the different source file systems. + *SourceFilesystems + + // The filesystem used to publish the rendered site. + // This usually maps to /my-project/public. + PublishFs afero.Fs + + theBigFs *filesystemsCollector +} + +// TODO(bep) we can get regular files in here and that is fine, but +// we need to clean up the naming. +func (fs *BaseFs) WatchDirs() []hugofs.FileMetaInfo { + var dirs []hugofs.FileMetaInfo + for _, dir := range fs.AllDirs() { + if dir.Meta().Watch() { + dirs = append(dirs, dir) + } + } + return dirs +} + +func (fs *BaseFs) AllDirs() []hugofs.FileMetaInfo { + var dirs []hugofs.FileMetaInfo + for _, dirSet := range [][]hugofs.FileMetaInfo{ + fs.Archetypes.Dirs, + fs.I18n.Dirs, + fs.Data.Dirs, + fs.Content.Dirs, + fs.Assets.Dirs, + fs.Layouts.Dirs, + //fs.Resources.Dirs, + fs.StaticDirs, + } { + dirs = append(dirs, dirSet...) + } + + return dirs +} + +// RelContentDir tries to create a path relative to the content root from +// the given filename. The return value is the path and language code. +func (b *BaseFs) RelContentDir(filename string) string { + for _, dir := range b.SourceFilesystems.Content.Dirs { + dirname := dir.Meta().Filename() + if strings.HasPrefix(filename, dirname) { + rel := path.Join(dir.Meta().Path(), strings.TrimPrefix(filename, dirname)) + return strings.TrimPrefix(rel, filePathSeparator) + } + } + // Either not a content dir or already relative. + return filename +} + +// SourceFilesystems contains the different source file systems. These can be +// composite file systems (theme and project etc.), and they have all root +// set to the source type the provides: data, i18n, static, layouts. +type SourceFilesystems struct { + Content *SourceFilesystem + Data *SourceFilesystem + I18n *SourceFilesystem + Layouts *SourceFilesystem + Archetypes *SourceFilesystem + Assets *SourceFilesystem + + // Writable filesystem on top the project's resources directory, + // with any sub module's resource fs layered below. + ResourcesCache afero.Fs + + // The project folder. + Work afero.Fs + + // When in multihost we have one static filesystem per language. The sync + // static files is currently done outside of the Hugo build (where there is + // a concept of a site per language). + // When in non-multihost mode there will be one entry in this map with a blank key. + Static map[string]*SourceFilesystem + + // All the /static dirs (including themes/modules). + StaticDirs []hugofs.FileMetaInfo +} + +// FileSystems returns the FileSystems relevant for the change detection +// in server mode. +// Note: This does currently not return any static fs. +func (s *SourceFilesystems) FileSystems() []*SourceFilesystem { + return []*SourceFilesystem{ + s.Content, + s.Data, + s.I18n, + s.Layouts, + s.Archetypes, + // TODO(bep) static + } + +} + +// A SourceFilesystem holds the filesystem for a given source type in Hugo (data, +// i18n, layouts, static) and additional metadata to be able to use that filesystem +// in server mode. +type SourceFilesystem struct { + // Name matches one in files.ComponentFolders + Name string + + // This is a virtual composite filesystem. It expects path relative to a context. + Fs afero.Fs + + // This filesystem as separate root directories, starting from project and down + // to the themes/modules. + Dirs []hugofs.FileMetaInfo + + // When syncing a source folder to the target (e.g. /public), this may + // be set to publish into a subfolder. This is used for static syncing + // in multihost mode. + PublishFolder string +} + +// ContentStaticAssetFs will create a new composite filesystem from the content, +// static, and asset filesystems. The site language is needed to pick the correct static filesystem. +// The order is content, static and then assets. +// TODO(bep) check usage +func (s SourceFilesystems) ContentStaticAssetFs(lang string) afero.Fs { + staticFs := s.StaticFs(lang) + + base := afero.NewCopyOnWriteFs(s.Assets.Fs, staticFs) + return afero.NewCopyOnWriteFs(base, s.Content.Fs) + +} + +// StaticFs returns the static filesystem for the given language. +// This can be a composite filesystem. +func (s SourceFilesystems) StaticFs(lang string) afero.Fs { + var staticFs afero.Fs = hugofs.NoOpFs + + if fs, ok := s.Static[lang]; ok { + staticFs = fs.Fs + } else if fs, ok := s.Static[""]; ok { + staticFs = fs.Fs + } + + return staticFs +} + +// StatResource looks for a resource in these filesystems in order: static, assets and finally content. +// If found in any of them, it returns FileInfo and the relevant filesystem. +// Any non os.IsNotExist error will be returned. +// An os.IsNotExist error wil be returned only if all filesystems return such an error. +// Note that if we only wanted to find the file, we could create a composite Afero fs, +// but we also need to know which filesystem root it lives in. +func (s SourceFilesystems) StatResource(lang, filename string) (fi os.FileInfo, fs afero.Fs, err error) { + for _, fsToCheck := range []afero.Fs{s.StaticFs(lang), s.Assets.Fs, s.Content.Fs} { + fs = fsToCheck + fi, err = fs.Stat(filename) + if err == nil || !os.IsNotExist(err) { + return + } + } + // Not found. + return +} + +// IsStatic returns true if the given filename is a member of one of the static +// filesystems. +func (s SourceFilesystems) IsStatic(filename string) bool { + for _, staticFs := range s.Static { + if staticFs.Contains(filename) { + return true + } + } + return false +} + +// IsContent returns true if the given filename is a member of the content filesystem. +func (s SourceFilesystems) IsContent(filename string) bool { + return s.Content.Contains(filename) +} + +// IsLayout returns true if the given filename is a member of the layouts filesystem. +func (s SourceFilesystems) IsLayout(filename string) bool { + return s.Layouts.Contains(filename) +} + +// IsData returns true if the given filename is a member of the data filesystem. +func (s SourceFilesystems) IsData(filename string) bool { + return s.Data.Contains(filename) +} + +// IsAsset returns true if the given filename is a member of the asset filesystem. +func (s SourceFilesystems) IsAsset(filename string) bool { + return s.Assets.Contains(filename) +} + +// IsI18n returns true if the given filename is a member of the i18n filesystem. +func (s SourceFilesystems) IsI18n(filename string) bool { + return s.I18n.Contains(filename) +} + +// MakeStaticPathRelative makes an absolute static filename into a relative one. +// It will return an empty string if the filename is not a member of a static filesystem. +func (s SourceFilesystems) MakeStaticPathRelative(filename string) string { + for _, staticFs := range s.Static { + rel := staticFs.MakePathRelative(filename) + if rel != "" { + return rel + } + } + return "" +} + +// MakePathRelative creates a relative path from the given filename. +// It will return an empty string if the filename is not a member of this filesystem. +func (d *SourceFilesystem) MakePathRelative(filename string) string { + + for _, dir := range d.Dirs { + meta := dir.(hugofs.FileMetaInfo).Meta() + currentPath := meta.Filename() + + if strings.HasPrefix(filename, currentPath) { + rel := strings.TrimPrefix(filename, currentPath) + if mp := meta.Path(); mp != "" { + rel = filepath.Join(mp, rel) + } + return strings.TrimPrefix(rel, filePathSeparator) + } + } + return "" +} + +func (d *SourceFilesystem) RealFilename(rel string) string { + fi, err := d.Fs.Stat(rel) + if err != nil { + return rel + } + if realfi, ok := fi.(hugofs.FileMetaInfo); ok { + return realfi.Meta().Filename() + } + + return rel +} + +// Contains returns whether the given filename is a member of the current filesystem. +func (d *SourceFilesystem) Contains(filename string) bool { + for _, dir := range d.Dirs { + if strings.HasPrefix(filename, dir.Meta().Filename()) { + return true + } + } + return false +} + +// Path returns the mount relative path to the given filename if it is a member of +// of the current filesystem, an empty string if not. +func (d *SourceFilesystem) Path(filename string) string { + for _, dir := range d.Dirs { + meta := dir.Meta() + if strings.HasPrefix(filename, meta.Filename()) { + p := strings.TrimPrefix(strings.TrimPrefix(filename, meta.Filename()), filePathSeparator) + if mountRoot := meta.MountRoot(); mountRoot != "" { + return filepath.Join(mountRoot, p) + } + return p + } + } + return "" +} + +// RealDirs gets a list of absolute paths to directories starting from the given +// path. +func (d *SourceFilesystem) RealDirs(from string) []string { + var dirnames []string + for _, dir := range d.Dirs { + meta := dir.Meta() + dirname := filepath.Join(meta.Filename(), from) + _, err := meta.Fs().Stat(from) + + if err == nil { + dirnames = append(dirnames, dirname) + } + } + return dirnames +} + +// WithBaseFs allows reuse of some potentially expensive to create parts that remain +// the same across sites/languages. +func WithBaseFs(b *BaseFs) func(*BaseFs) error { + return func(bb *BaseFs) error { + bb.theBigFs = b.theBigFs + bb.SourceFilesystems = b.SourceFilesystems + return nil + } +} + +// NewBase builds the filesystems used by Hugo given the paths and options provided.NewBase +func NewBase(p *paths.Paths, logger *loggers.Logger, options ...func(*BaseFs) error) (*BaseFs, error) { + fs := p.Fs + if logger == nil { + logger = loggers.NewWarningLogger() + } + + publishFs := hugofs.NewBaseFileDecorator(afero.NewBasePathFs(fs.Destination, p.AbsPublishDir)) + + b := &BaseFs{ + PublishFs: publishFs, + } + + for _, opt := range options { + if err := opt(b); err != nil { + return nil, err + } + } + + if b.theBigFs != nil && b.SourceFilesystems != nil { + return b, nil + } + + builder := newSourceFilesystemsBuilder(p, logger, b) + sourceFilesystems, err := builder.Build() + if err != nil { + return nil, errors.Wrap(err, "build filesystems") + } + + b.SourceFilesystems = sourceFilesystems + b.theBigFs = builder.theBigFs + + return b, nil +} + +type sourceFilesystemsBuilder struct { + logger *loggers.Logger + p *paths.Paths + sourceFs afero.Fs + result *SourceFilesystems + theBigFs *filesystemsCollector +} + +func newSourceFilesystemsBuilder(p *paths.Paths, logger *loggers.Logger, b *BaseFs) *sourceFilesystemsBuilder { + sourceFs := hugofs.NewBaseFileDecorator(p.Fs.Source) + return &sourceFilesystemsBuilder{p: p, logger: logger, sourceFs: sourceFs, theBigFs: b.theBigFs, result: &SourceFilesystems{}} +} + +func (b *sourceFilesystemsBuilder) newSourceFilesystem(name string, fs afero.Fs, dirs []hugofs.FileMetaInfo) *SourceFilesystem { + return &SourceFilesystem{ + Name: name, + Fs: fs, + Dirs: dirs, + } +} + +func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { + + if b.theBigFs == nil { + + theBigFs, err := b.createMainOverlayFs(b.p) + if err != nil { + return nil, errors.Wrap(err, "create main fs") + } + + b.theBigFs = theBigFs + } + + createView := func(componentID string) *SourceFilesystem { + if b.theBigFs == nil || b.theBigFs.overlayMounts == nil { + return b.newSourceFilesystem(componentID, hugofs.NoOpFs, nil) + } + + dirs := b.theBigFs.overlayDirs[componentID] + + return b.newSourceFilesystem(componentID, afero.NewBasePathFs(b.theBigFs.overlayMounts, componentID), dirs) + + } + + b.theBigFs.finalizeDirs() + + b.result.Archetypes = createView(files.ComponentFolderArchetypes) + b.result.Layouts = createView(files.ComponentFolderLayouts) + b.result.Assets = createView(files.ComponentFolderAssets) + b.result.ResourcesCache = b.theBigFs.overlayResources + + // Data, i18n and content cannot use the overlay fs + dataDirs := b.theBigFs.overlayDirs[files.ComponentFolderData] + dataFs, err := hugofs.NewSliceFs(dataDirs...) + if err != nil { + return nil, err + } + + b.result.Data = b.newSourceFilesystem(files.ComponentFolderData, dataFs, dataDirs) + + i18nDirs := b.theBigFs.overlayDirs[files.ComponentFolderI18n] + i18nFs, err := hugofs.NewSliceFs(i18nDirs...) + if err != nil { + return nil, err + } + b.result.I18n = b.newSourceFilesystem(files.ComponentFolderI18n, i18nFs, i18nDirs) + + contentDirs := b.theBigFs.overlayDirs[files.ComponentFolderContent] + contentBfs := afero.NewBasePathFs(b.theBigFs.overlayMountsContent, files.ComponentFolderContent) + + contentFs, err := hugofs.NewLanguageFs(b.p.LanguagesDefaultFirst.AsOrdinalSet(), contentBfs) + if err != nil { + return nil, errors.Wrap(err, "create content filesystem") + } + + b.result.Content = b.newSourceFilesystem(files.ComponentFolderContent, contentFs, contentDirs) + + b.result.Work = afero.NewReadOnlyFs(b.theBigFs.overlayFull) + + // Create static filesystem(s) + ms := make(map[string]*SourceFilesystem) + b.result.Static = ms + b.result.StaticDirs = b.theBigFs.overlayDirs[files.ComponentFolderStatic] + + if b.theBigFs.staticPerLanguage != nil { + // Multihost mode + for k, v := range b.theBigFs.staticPerLanguage { + sfs := b.newSourceFilesystem(files.ComponentFolderStatic, v, b.result.StaticDirs) + sfs.PublishFolder = k + ms[k] = sfs + } + } else { + bfs := afero.NewBasePathFs(b.theBigFs.overlayMountsStatic, files.ComponentFolderStatic) + ms[""] = b.newSourceFilesystem(files.ComponentFolderStatic, bfs, b.result.StaticDirs) + } + + return b.result, nil + +} + +func (b *sourceFilesystemsBuilder) createMainOverlayFs(p *paths.Paths) (*filesystemsCollector, error) { + + var staticFsMap map[string]afero.Fs + if b.p.Cfg.GetBool("multihost") { + staticFsMap = make(map[string]afero.Fs) + } + + collector := &filesystemsCollector{ + sourceProject: b.sourceFs, + sourceModules: hugofs.NewNoSymlinkFs(b.sourceFs, b.logger, false), + overlayDirs: make(map[string][]hugofs.FileMetaInfo), + staticPerLanguage: staticFsMap, + } + + mods := p.AllModules + + if len(mods) == 0 { + return collector, nil + } + + modsReversed := make([]mountsDescriptor, len(mods)) + + // The theme components are ordered from left to right. + // We need to revert it to get the + // overlay logic below working as expected, with the project on top. + j := 0 + for i := len(mods) - 1; i >= 0; i-- { + mod := mods[i] + dir := mod.Dir() + + isMainProject := mod.Owner() == nil + modsReversed[j] = mountsDescriptor{ + Module: mod, + dir: dir, + isMainProject: isMainProject, + } + j++ + } + + err := b.createOverlayFs(collector, modsReversed) + + return collector, err + +} + +func (b *sourceFilesystemsBuilder) isContentMount(mnt modules.Mount) bool { + return strings.HasPrefix(mnt.Target, files.ComponentFolderContent) +} + +func (b *sourceFilesystemsBuilder) isStaticMount(mnt modules.Mount) bool { + return strings.HasPrefix(mnt.Target, files.ComponentFolderStatic) +} + +func (b *sourceFilesystemsBuilder) createModFs( + collector *filesystemsCollector, + md mountsDescriptor) error { + + var ( + fromTo []hugofs.RootMapping + fromToContent []hugofs.RootMapping + fromToStatic []hugofs.RootMapping + ) + + absPathify := func(path string) (string, string) { + if filepath.IsAbs(path) { + return "", path + } + return md.dir, paths.AbsPathify(md.dir, path) + } + + for _, mount := range md.Mounts() { + + mountWeight := 1 + if md.isMainProject { + mountWeight++ + } + + base, filename := absPathify(mount.Source) + + rm := hugofs.RootMapping{ + From: mount.Target, + To: filename, + ToBasedir: base, + Module: md.Module.Path(), + Meta: hugofs.FileMeta{ + "watch": md.Watch(), + "mountWeight": mountWeight, + }, + } + + isContentMount := b.isContentMount(mount) + + lang := mount.Lang + if lang == "" && isContentMount { + lang = b.p.DefaultContentLanguage + } + + rm.Meta["lang"] = lang + + if isContentMount { + fromToContent = append(fromToContent, rm) + } else if b.isStaticMount(mount) { + fromToStatic = append(fromToStatic, rm) + } else { + fromTo = append(fromTo, rm) + } + } + + modBase := collector.sourceProject + if !md.isMainProject { + modBase = collector.sourceModules + } + sourceStatic := hugofs.NewNoSymlinkFs(modBase, b.logger, true) + + rmfs, err := hugofs.NewRootMappingFs(modBase, fromTo...) + if err != nil { + return err + } + rmfsContent, err := hugofs.NewRootMappingFs(modBase, fromToContent...) + if err != nil { + return err + } + rmfsStatic, err := hugofs.NewRootMappingFs(sourceStatic, fromToStatic...) + if err != nil { + return err + } + + // We need to keep the ordered list of directories for watching and + // some special merge operations (data, i18n). + collector.addDirs(rmfs) + collector.addDirs(rmfsContent) + collector.addDirs(rmfsStatic) + + if collector.staticPerLanguage != nil { + for _, l := range b.p.Languages { + lang := l.Lang + + lfs := rmfsStatic.Filter(func(rm hugofs.RootMapping) bool { + rlang := rm.Meta.Lang() + return rlang == "" || rlang == lang + }) + + bfs := afero.NewBasePathFs(lfs, files.ComponentFolderStatic) + + sfs, found := collector.staticPerLanguage[lang] + if found { + collector.staticPerLanguage[lang] = afero.NewCopyOnWriteFs(sfs, bfs) + + } else { + collector.staticPerLanguage[lang] = bfs + } + } + } + + getResourcesDir := func() string { + if md.isMainProject { + return b.p.AbsResourcesDir + } + _, filename := absPathify(files.FolderResources) + return filename + } + + if collector.overlayMounts == nil { + collector.overlayMounts = rmfs + collector.overlayMountsContent = rmfsContent + collector.overlayMountsStatic = rmfsStatic + collector.overlayFull = afero.NewBasePathFs(modBase, md.dir) + collector.overlayResources = afero.NewBasePathFs(modBase, getResourcesDir()) + } else { + + collector.overlayMounts = afero.NewCopyOnWriteFs(collector.overlayMounts, rmfs) + collector.overlayMountsContent = hugofs.NewLanguageCompositeFs(collector.overlayMountsContent, rmfsContent) + collector.overlayMountsStatic = hugofs.NewLanguageCompositeFs(collector.overlayMountsStatic, rmfsStatic) + collector.overlayFull = afero.NewCopyOnWriteFs(collector.overlayFull, afero.NewBasePathFs(modBase, md.dir)) + collector.overlayResources = afero.NewCopyOnWriteFs(collector.overlayResources, afero.NewBasePathFs(modBase, getResourcesDir())) + } + + return nil + +} + +func printFs(fs afero.Fs, path string, w io.Writer) { + if fs == nil { + return + } + afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + var filename string + if fim, ok := info.(hugofs.FileMetaInfo); ok { + filename = fim.Meta().Filename() + } + fmt.Fprintf(w, " %q %q\n", path, filename) + return nil + }) +} + +type filesystemsCollector struct { + sourceProject afero.Fs // Source for project folders + sourceModules afero.Fs // Source for modules/themes + + overlayMounts afero.Fs + overlayMountsContent afero.Fs + overlayMountsStatic afero.Fs + overlayFull afero.Fs + overlayResources afero.Fs + + // Maps component type (layouts, static, content etc.) an ordered list of + // directories representing the overlay filesystems above. + overlayDirs map[string][]hugofs.FileMetaInfo + + // Set if in multihost mode + staticPerLanguage map[string]afero.Fs + + finalizerInit sync.Once +} + +func (c *filesystemsCollector) addDirs(rfs *hugofs.RootMappingFs) { + for _, componentFolder := range files.ComponentFolders { + dirs, err := rfs.Dirs(componentFolder) + + if err == nil { + c.overlayDirs[componentFolder] = append(c.overlayDirs[componentFolder], dirs...) + } + } +} + +func (c *filesystemsCollector) finalizeDirs() { + c.finalizerInit.Do(func() { + // Order the directories from top to bottom (project, theme a, theme ...). + for _, dirs := range c.overlayDirs { + c.reverseFis(dirs) + } + }) + +} + +func (c *filesystemsCollector) reverseFis(fis []hugofs.FileMetaInfo) { + for i := len(fis)/2 - 1; i >= 0; i-- { + opp := len(fis) - 1 - i + fis[i], fis[opp] = fis[opp], fis[i] + } +} + +type mountsDescriptor struct { + modules.Module + dir string + isMainProject bool +} + +func (b *sourceFilesystemsBuilder) createOverlayFs(collector *filesystemsCollector, mounts []mountsDescriptor) error { + if len(mounts) == 0 { + return nil + } + + err := b.createModFs(collector, mounts[0]) + if err != nil { + return err + } + + if len(mounts) == 1 { + return nil + } + + return b.createOverlayFs(collector, mounts[1:]) +} diff --git a/hugolib/filesystems/basefs_test.go b/hugolib/filesystems/basefs_test.go new file mode 100644 index 000000000..e3222af48 --- /dev/null +++ b/hugolib/filesystems/basefs_test.go @@ -0,0 +1,460 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filesystems + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/gohugoio/hugo/config" + + "github.com/gohugoio/hugo/langs" + + "github.com/spf13/afero" + + qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/hugolib/paths" + "github.com/gohugoio/hugo/modules" + "github.com/spf13/viper" +) + +func initConfig(fs afero.Fs, cfg config.Provider) error { + if _, err := langs.LoadLanguageSettings(cfg, nil); err != nil { + return err + } + + modConfig, err := modules.DecodeConfig(cfg) + if err != nil { + return err + } + + workingDir := cfg.GetString("workingDir") + themesDir := cfg.GetString("themesDir") + if !filepath.IsAbs(themesDir) { + themesDir = filepath.Join(workingDir, themesDir) + } + modulesClient := modules.NewClient(modules.ClientConfig{ + Fs: fs, + WorkingDir: workingDir, + ThemesDir: themesDir, + ModuleConfig: modConfig, + IgnoreVendor: true, + }) + + moduleConfig, err := modulesClient.Collect() + if err != nil { + return err + } + + if err := modules.ApplyProjectConfigDefaults(cfg, moduleConfig.ActiveModules[0]); err != nil { + return err + } + + cfg.Set("allModules", moduleConfig.ActiveModules) + + return nil +} + +func TestNewBaseFs(t *testing.T) { + c := qt.New(t) + v := viper.New() + + fs := hugofs.NewMem(v) + + themes := []string{"btheme", "atheme"} + + workingDir := filepath.FromSlash("/my/work") + v.Set("workingDir", workingDir) + v.Set("contentDir", "content") + v.Set("themesDir", "themes") + v.Set("defaultContentLanguage", "en") + v.Set("theme", themes[:1]) + + // Write some data to the themes + for _, theme := range themes { + for _, dir := range []string{"i18n", "data", "archetypes", "layouts"} { + base := filepath.Join(workingDir, "themes", theme, dir) + filenameTheme := filepath.Join(base, fmt.Sprintf("theme-file-%s.txt", theme)) + filenameOverlap := filepath.Join(base, "f3.txt") + fs.Source.Mkdir(base, 0755) + content := []byte(fmt.Sprintf("content:%s:%s", theme, dir)) + afero.WriteFile(fs.Source, filenameTheme, content, 0755) + afero.WriteFile(fs.Source, filenameOverlap, content, 0755) + } + // Write some files to the root of the theme + base := filepath.Join(workingDir, "themes", theme) + afero.WriteFile(fs.Source, filepath.Join(base, fmt.Sprintf("theme-root-%s.txt", theme)), []byte(fmt.Sprintf("content:%s", theme)), 0755) + afero.WriteFile(fs.Source, filepath.Join(base, "file-theme-root.txt"), []byte(fmt.Sprintf("content:%s", theme)), 0755) + } + + afero.WriteFile(fs.Source, filepath.Join(workingDir, "file-root.txt"), []byte("content-project"), 0755) + + afero.WriteFile(fs.Source, filepath.Join(workingDir, "themes", "btheme", "config.toml"), []byte(` +theme = ["atheme"] +`), 0755) + + setConfigAndWriteSomeFilesTo(fs.Source, v, "contentDir", "mycontent", 3) + setConfigAndWriteSomeFilesTo(fs.Source, v, "i18nDir", "myi18n", 4) + setConfigAndWriteSomeFilesTo(fs.Source, v, "layoutDir", "mylayouts", 5) + setConfigAndWriteSomeFilesTo(fs.Source, v, "staticDir", "mystatic", 6) + setConfigAndWriteSomeFilesTo(fs.Source, v, "dataDir", "mydata", 7) + setConfigAndWriteSomeFilesTo(fs.Source, v, "archetypeDir", "myarchetypes", 8) + setConfigAndWriteSomeFilesTo(fs.Source, v, "assetDir", "myassets", 9) + setConfigAndWriteSomeFilesTo(fs.Source, v, "resourceDir", "myrsesource", 10) + + v.Set("publishDir", "public") + c.Assert(initConfig(fs.Source, v), qt.IsNil) + + p, err := paths.New(fs, v) + c.Assert(err, qt.IsNil) + + bfs, err := NewBase(p, nil) + c.Assert(err, qt.IsNil) + c.Assert(bfs, qt.Not(qt.IsNil)) + + root, err := bfs.I18n.Fs.Open("") + c.Assert(err, qt.IsNil) + dirnames, err := root.Readdirnames(-1) + c.Assert(err, qt.IsNil) + c.Assert(dirnames, qt.DeepEquals, []string{"f1.txt", "f2.txt", "f3.txt", "f4.txt", "f3.txt", "theme-file-btheme.txt", "f3.txt", "theme-file-atheme.txt"}) + + root, err = bfs.Data.Fs.Open("") + c.Assert(err, qt.IsNil) + dirnames, err = root.Readdirnames(-1) + c.Assert(err, qt.IsNil) + c.Assert(dirnames, qt.DeepEquals, []string{"f1.txt", "f2.txt", "f3.txt", "f4.txt", "f5.txt", "f6.txt", "f7.txt", "f3.txt", "theme-file-btheme.txt", "f3.txt", "theme-file-atheme.txt"}) + + checkFileCount(bfs.Layouts.Fs, "", c, 7) + + checkFileCount(bfs.Content.Fs, "", c, 3) + checkFileCount(bfs.I18n.Fs, "", c, 8) // 4 + 4 themes + + checkFileCount(bfs.Static[""].Fs, "", c, 6) + checkFileCount(bfs.Data.Fs, "", c, 11) // 7 + 4 themes + checkFileCount(bfs.Archetypes.Fs, "", c, 10) // 8 + 2 themes + checkFileCount(bfs.Assets.Fs, "", c, 9) + checkFileCount(bfs.Work, "", c, 82) + + c.Assert(bfs.IsData(filepath.Join(workingDir, "mydata", "file1.txt")), qt.Equals, true) + c.Assert(bfs.IsI18n(filepath.Join(workingDir, "myi18n", "file1.txt")), qt.Equals, true) + c.Assert(bfs.IsLayout(filepath.Join(workingDir, "mylayouts", "file1.txt")), qt.Equals, true) + c.Assert(bfs.IsStatic(filepath.Join(workingDir, "mystatic", "file1.txt")), qt.Equals, true) + c.Assert(bfs.IsAsset(filepath.Join(workingDir, "myassets", "file1.txt")), qt.Equals, true) + + contentFilename := filepath.Join(workingDir, "mycontent", "file1.txt") + c.Assert(bfs.IsContent(contentFilename), qt.Equals, true) + rel := bfs.RelContentDir(contentFilename) + c.Assert(rel, qt.Equals, "file1.txt") + + // Check Work fs vs theme + checkFileContent(bfs.Work, "file-root.txt", c, "content-project") + checkFileContent(bfs.Work, "theme-root-atheme.txt", c, "content:atheme") + + // https://github.com/gohugoio/hugo/issues/5318 + // Check both project and theme. + for _, fs := range []afero.Fs{bfs.Archetypes.Fs, bfs.Layouts.Fs} { + for _, filename := range []string{"/f1.txt", "/theme-file-atheme.txt"} { + filename = filepath.FromSlash(filename) + f, err := fs.Open(filename) + c.Assert(err, qt.IsNil) + f.Close() + } + } +} + +func createConfig() *viper.Viper { + v := viper.New() + v.Set("contentDir", "mycontent") + v.Set("i18nDir", "myi18n") + v.Set("staticDir", "mystatic") + v.Set("dataDir", "mydata") + v.Set("layoutDir", "mylayouts") + v.Set("archetypeDir", "myarchetypes") + v.Set("assetDir", "myassets") + v.Set("resourceDir", "resources") + v.Set("publishDir", "public") + v.Set("defaultContentLanguage", "en") + + return v +} + +func TestNewBaseFsEmpty(t *testing.T) { + c := qt.New(t) + v := createConfig() + fs := hugofs.NewMem(v) + c.Assert(initConfig(fs.Source, v), qt.IsNil) + + p, err := paths.New(fs, v) + c.Assert(err, qt.IsNil) + bfs, err := NewBase(p, nil) + c.Assert(err, qt.IsNil) + c.Assert(bfs, qt.Not(qt.IsNil)) + c.Assert(bfs.Archetypes.Fs, qt.Not(qt.IsNil)) + c.Assert(bfs.Layouts.Fs, qt.Not(qt.IsNil)) + c.Assert(bfs.Data.Fs, qt.Not(qt.IsNil)) + c.Assert(bfs.I18n.Fs, qt.Not(qt.IsNil)) + c.Assert(bfs.Work, qt.Not(qt.IsNil)) + c.Assert(bfs.Content.Fs, qt.Not(qt.IsNil)) + c.Assert(bfs.Static, qt.Not(qt.IsNil)) +} + +func TestRealDirs(t *testing.T) { + c := qt.New(t) + v := createConfig() + fs := hugofs.NewDefault(v) + sfs := fs.Source + + root, err := afero.TempDir(sfs, "", "realdir") + c.Assert(err, qt.IsNil) + themesDir, err := afero.TempDir(sfs, "", "themesDir") + c.Assert(err, qt.IsNil) + defer func() { + os.RemoveAll(root) + os.RemoveAll(themesDir) + }() + + v.Set("workingDir", root) + v.Set("themesDir", themesDir) + v.Set("theme", "mytheme") + + c.Assert(sfs.MkdirAll(filepath.Join(root, "myassets", "scss", "sf1"), 0755), qt.IsNil) + c.Assert(sfs.MkdirAll(filepath.Join(root, "myassets", "scss", "sf2"), 0755), qt.IsNil) + c.Assert(sfs.MkdirAll(filepath.Join(themesDir, "mytheme", "assets", "scss", "sf2"), 0755), qt.IsNil) + c.Assert(sfs.MkdirAll(filepath.Join(themesDir, "mytheme", "assets", "scss", "sf3"), 0755), qt.IsNil) + c.Assert(sfs.MkdirAll(filepath.Join(root, "resources"), 0755), qt.IsNil) + c.Assert(sfs.MkdirAll(filepath.Join(themesDir, "mytheme", "resources"), 0755), qt.IsNil) + + c.Assert(sfs.MkdirAll(filepath.Join(root, "myassets", "js", "f2"), 0755), qt.IsNil) + + afero.WriteFile(sfs, filepath.Join(filepath.Join(root, "myassets", "scss", "sf1", "a1.scss")), []byte("content"), 0755) + afero.WriteFile(sfs, filepath.Join(filepath.Join(root, "myassets", "scss", "sf2", "a3.scss")), []byte("content"), 0755) + afero.WriteFile(sfs, filepath.Join(filepath.Join(root, "myassets", "scss", "a2.scss")), []byte("content"), 0755) + afero.WriteFile(sfs, filepath.Join(filepath.Join(themesDir, "mytheme", "assets", "scss", "sf2", "a3.scss")), []byte("content"), 0755) + afero.WriteFile(sfs, filepath.Join(filepath.Join(themesDir, "mytheme", "assets", "scss", "sf3", "a4.scss")), []byte("content"), 0755) + + afero.WriteFile(sfs, filepath.Join(filepath.Join(themesDir, "mytheme", "resources", "t1.txt")), []byte("content"), 0755) + afero.WriteFile(sfs, filepath.Join(filepath.Join(root, "resources", "p1.txt")), []byte("content"), 0755) + afero.WriteFile(sfs, filepath.Join(filepath.Join(root, "resources", "p2.txt")), []byte("content"), 0755) + + afero.WriteFile(sfs, filepath.Join(filepath.Join(root, "myassets", "js", "f2", "a1.js")), []byte("content"), 0755) + afero.WriteFile(sfs, filepath.Join(filepath.Join(root, "myassets", "js", "a2.js")), []byte("content"), 0755) + + c.Assert(initConfig(fs.Source, v), qt.IsNil) + + p, err := paths.New(fs, v) + c.Assert(err, qt.IsNil) + bfs, err := NewBase(p, nil) + c.Assert(err, qt.IsNil) + c.Assert(bfs, qt.Not(qt.IsNil)) + + checkFileCount(bfs.Assets.Fs, "", c, 6) + + realDirs := bfs.Assets.RealDirs("scss") + c.Assert(len(realDirs), qt.Equals, 2) + c.Assert(realDirs[0], qt.Equals, filepath.Join(root, "myassets/scss")) + c.Assert(realDirs[len(realDirs)-1], qt.Equals, filepath.Join(themesDir, "mytheme/assets/scss")) + + c.Assert(bfs.theBigFs, qt.Not(qt.IsNil)) + +} + +func TestStaticFs(t *testing.T) { + c := qt.New(t) + v := createConfig() + workDir := "mywork" + v.Set("workingDir", workDir) + v.Set("themesDir", "themes") + v.Set("theme", []string{"t1", "t2"}) + + fs := hugofs.NewMem(v) + + themeStaticDir := filepath.Join(workDir, "themes", "t1", "static") + themeStaticDir2 := filepath.Join(workDir, "themes", "t2", "static") + + afero.WriteFile(fs.Source, filepath.Join(workDir, "mystatic", "f1.txt"), []byte("Hugo Rocks!"), 0755) + afero.WriteFile(fs.Source, filepath.Join(themeStaticDir, "f1.txt"), []byte("Hugo Themes Rocks!"), 0755) + afero.WriteFile(fs.Source, filepath.Join(themeStaticDir, "f2.txt"), []byte("Hugo Themes Still Rocks!"), 0755) + afero.WriteFile(fs.Source, filepath.Join(themeStaticDir2, "f2.txt"), []byte("Hugo Themes Rocks in t2!"), 0755) + + c.Assert(initConfig(fs.Source, v), qt.IsNil) + + p, err := paths.New(fs, v) + c.Assert(err, qt.IsNil) + bfs, err := NewBase(p, nil) + c.Assert(err, qt.IsNil) + + sfs := bfs.StaticFs("en") + checkFileContent(sfs, "f1.txt", c, "Hugo Rocks!") + checkFileContent(sfs, "f2.txt", c, "Hugo Themes Still Rocks!") + +} + +func TestStaticFsMultiHost(t *testing.T) { + c := qt.New(t) + v := createConfig() + workDir := "mywork" + v.Set("workingDir", workDir) + v.Set("themesDir", "themes") + v.Set("theme", "t1") + v.Set("defaultContentLanguage", "en") + + langConfig := map[string]interface{}{ + "no": map[string]interface{}{ + "staticDir": "static_no", + "baseURL": "https://example.org/no/", + }, + "en": map[string]interface{}{ + "baseURL": "https://example.org/en/", + }, + } + + v.Set("languages", langConfig) + + fs := hugofs.NewMem(v) + + themeStaticDir := filepath.Join(workDir, "themes", "t1", "static") + + afero.WriteFile(fs.Source, filepath.Join(workDir, "mystatic", "f1.txt"), []byte("Hugo Rocks!"), 0755) + afero.WriteFile(fs.Source, filepath.Join(workDir, "static_no", "f1.txt"), []byte("Hugo Rocks in Norway!"), 0755) + + afero.WriteFile(fs.Source, filepath.Join(themeStaticDir, "f1.txt"), []byte("Hugo Themes Rocks!"), 0755) + afero.WriteFile(fs.Source, filepath.Join(themeStaticDir, "f2.txt"), []byte("Hugo Themes Still Rocks!"), 0755) + + c.Assert(initConfig(fs.Source, v), qt.IsNil) + + p, err := paths.New(fs, v) + c.Assert(err, qt.IsNil) + bfs, err := NewBase(p, nil) + c.Assert(err, qt.IsNil) + enFs := bfs.StaticFs("en") + checkFileContent(enFs, "f1.txt", c, "Hugo Rocks!") + checkFileContent(enFs, "f2.txt", c, "Hugo Themes Still Rocks!") + + noFs := bfs.StaticFs("no") + checkFileContent(noFs, "f1.txt", c, "Hugo Rocks in Norway!") + checkFileContent(noFs, "f2.txt", c, "Hugo Themes Still Rocks!") +} + +func TestMakePathRelative(t *testing.T) { + c := qt.New(t) + v := createConfig() + fs := hugofs.NewMem(v) + workDir := "mywork" + v.Set("workingDir", workDir) + + c.Assert(fs.Source.MkdirAll(filepath.Join(workDir, "dist", "d1"), 0777), qt.IsNil) + c.Assert(fs.Source.MkdirAll(filepath.Join(workDir, "static", "d2"), 0777), qt.IsNil) + c.Assert(fs.Source.MkdirAll(filepath.Join(workDir, "dust", "d2"), 0777), qt.IsNil) + + moduleCfg := map[string]interface{}{ + "mounts": []interface{}{ + map[string]interface{}{ + "source": "dist", + "target": "static/mydist", + }, + map[string]interface{}{ + "source": "dust", + "target": "static/foo/bar", + }, + map[string]interface{}{ + "source": "static", + "target": "static", + }, + }, + } + + v.Set("module", moduleCfg) + + c.Assert(initConfig(fs.Source, v), qt.IsNil) + + p, err := paths.New(fs, v) + c.Assert(err, qt.IsNil) + bfs, err := NewBase(p, nil) + c.Assert(err, qt.IsNil) + + sfs := bfs.Static[""] + c.Assert(sfs, qt.Not(qt.IsNil)) + + c.Assert(sfs.MakePathRelative(filepath.Join(workDir, "dist", "d1", "foo.txt")), qt.Equals, filepath.FromSlash("mydist/d1/foo.txt")) + c.Assert(sfs.MakePathRelative(filepath.Join(workDir, "static", "d2", "foo.txt")), qt.Equals, filepath.FromSlash("d2/foo.txt")) + c.Assert(sfs.MakePathRelative(filepath.Join(workDir, "dust", "d3", "foo.txt")), qt.Equals, filepath.FromSlash("foo/bar/d3/foo.txt")) + +} + +func checkFileCount(fs afero.Fs, dirname string, c *qt.C, expected int) { + count, _, err := countFilesAndGetFilenames(fs, dirname) + c.Assert(err, qt.IsNil) + c.Assert(count, qt.Equals, expected) +} + +func checkFileContent(fs afero.Fs, filename string, c *qt.C, expected ...string) { + + b, err := afero.ReadFile(fs, filename) + c.Assert(err, qt.IsNil) + + content := string(b) + + for _, e := range expected { + c.Assert(content, qt.Contains, e) + } +} + +func countFilesAndGetFilenames(fs afero.Fs, dirname string) (int, []string, error) { + if fs == nil { + return 0, nil, errors.New("no fs") + } + + counter := 0 + var filenames []string + + wf := func(path string, info hugofs.FileMetaInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + counter++ + } + + if info.Name() != "." { + name := info.Name() + name = strings.Replace(name, filepath.FromSlash("/my/work"), "WORK_DIR", 1) + filenames = append(filenames, name) + } + + return nil + } + + w := hugofs.NewWalkway(hugofs.WalkwayConfig{Fs: fs, Root: dirname, WalkFn: wf}) + + if err := w.Walk(); err != nil { + return -1, nil, err + } + + return counter, filenames, nil +} + +func setConfigAndWriteSomeFilesTo(fs afero.Fs, v *viper.Viper, key, val string, num int) { + workingDir := v.GetString("workingDir") + v.Set(key, val) + fs.Mkdir(val, 0755) + for i := 0; i < num; i++ { + filename := filepath.Join(workingDir, val, fmt.Sprintf("f%d.txt", i+1)) + afero.WriteFile(fs, filename, []byte(fmt.Sprintf("content:%s:%d", key, i+1)), 0755) + } +} |