diff options
Diffstat (limited to 'hugolib/filesystems/basefs.go')
| -rw-r--r-- | hugolib/filesystems/basefs.go | 745 |
1 files changed, 745 insertions, 0 deletions
diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go new file mode 100644 index 000000000..57a95a037 --- /dev/null +++ b/hugolib/filesystems/basefs.go @@ -0,0 +1,745 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package filesystems provides the fine grained file systems used by Hugo. These +// are typically virtual filesystems that are composites of project and theme content. +package filesystems + +import ( + "io" + "os" + "path" + "path/filepath" + "strings" + "sync" + + "github.com/gohugoio/hugo/common/loggers" + + "github.com/gohugoio/hugo/hugofs/files" + + "github.com/pkg/errors" + + "github.com/gohugoio/hugo/modules" + + "github.com/gohugoio/hugo/hugofs" + + "fmt" + + "github.com/gohugoio/hugo/hugolib/paths" + "github.com/spf13/afero" +) + +var filePathSeparator = string(filepath.Separator) + +// BaseFs contains the core base filesystems used by Hugo. The name "base" is used +// to underline that even if they can be composites, they all have a base path set to a specific +// resource folder, e.g "/my-project/content". So, no absolute filenames needed. +type BaseFs struct { + + // SourceFilesystems contains the different source file systems. + *SourceFilesystems + + // The filesystem used to publish the rendered site. + // This usually maps to /my-project/public. + PublishFs afero.Fs + + theBigFs *filesystemsCollector +} + +// TODO(bep) we can get regular files in here and that is fine, but +// we need to clean up the naming. +func (fs *BaseFs) WatchDirs() []hugofs.FileMetaInfo { + var dirs []hugofs.FileMetaInfo + for _, dir := range fs.AllDirs() { + if dir.Meta().Watch() { + dirs = append(dirs, dir) + } + } + return dirs +} + +func (fs *BaseFs) AllDirs() []hugofs.FileMetaInfo { + var dirs []hugofs.FileMetaInfo + for _, dirSet := range [][]hugofs.FileMetaInfo{ + fs.Archetypes.Dirs, + fs.I18n.Dirs, + fs.Data.Dirs, + fs.Content.Dirs, + fs.Assets.Dirs, + fs.Layouts.Dirs, + //fs.Resources.Dirs, + fs.StaticDirs, + } { + dirs = append(dirs, dirSet...) + } + + return dirs +} + +// RelContentDir tries to create a path relative to the content root from +// the given filename. The return value is the path and language code. +func (b *BaseFs) RelContentDir(filename string) string { + for _, dir := range b.SourceFilesystems.Content.Dirs { + dirname := dir.Meta().Filename() + if strings.HasPrefix(filename, dirname) { + rel := path.Join(dir.Meta().Path(), strings.TrimPrefix(filename, dirname)) + return strings.TrimPrefix(rel, filePathSeparator) + } + } + // Either not a content dir or already relative. + return filename +} + +// SourceFilesystems contains the different source file systems. These can be +// composite file systems (theme and project etc.), and they have all root +// set to the source type the provides: data, i18n, static, layouts. +type SourceFilesystems struct { + Content *SourceFilesystem + Data *SourceFilesystem + I18n *SourceFilesystem + Layouts *SourceFilesystem + Archetypes *SourceFilesystem + Assets *SourceFilesystem + + // Writable filesystem on top the project's resources directory, + // with any sub module's resource fs layered below. + ResourcesCache afero.Fs + + // The project folder. + Work afero.Fs + + // When in multihost we have one static filesystem per language. The sync + // static files is currently done outside of the Hugo build (where there is + // a concept of a site per language). + // When in non-multihost mode there will be one entry in this map with a blank key. + Static map[string]*SourceFilesystem + + // All the /static dirs (including themes/modules). + StaticDirs []hugofs.FileMetaInfo +} + +// FileSystems returns the FileSystems relevant for the change detection +// in server mode. +// Note: This does currently not return any static fs. +func (s *SourceFilesystems) FileSystems() []*SourceFilesystem { + return []*SourceFilesystem{ + s.Content, + s.Data, + s.I18n, + s.Layouts, + s.Archetypes, + // TODO(bep) static + } + +} + +// A SourceFilesystem holds the filesystem for a given source type in Hugo (data, +// i18n, layouts, static) and additional metadata to be able to use that filesystem +// in server mode. +type SourceFilesystem struct { + // Name matches one in files.ComponentFolders + Name string + + // This is a virtual composite filesystem. It expects path relative to a context. + Fs afero.Fs + + // This filesystem as separate root directories, starting from project and down + // to the themes/modules. + Dirs []hugofs.FileMetaInfo + + // When syncing a source folder to the target (e.g. /public), this may + // be set to publish into a subfolder. This is used for static syncing + // in multihost mode. + PublishFolder string +} + +// ContentStaticAssetFs will create a new composite filesystem from the content, +// static, and asset filesystems. The site language is needed to pick the correct static filesystem. +// The order is content, static and then assets. +// TODO(bep) check usage +func (s SourceFilesystems) ContentStaticAssetFs(lang string) afero.Fs { + staticFs := s.StaticFs(lang) + + base := afero.NewCopyOnWriteFs(s.Assets.Fs, staticFs) + return afero.NewCopyOnWriteFs(base, s.Content.Fs) + +} + +// StaticFs returns the static filesystem for the given language. +// This can be a composite filesystem. +func (s SourceFilesystems) StaticFs(lang string) afero.Fs { + var staticFs afero.Fs = hugofs.NoOpFs + + if fs, ok := s.Static[lang]; ok { + staticFs = fs.Fs + } else if fs, ok := s.Static[""]; ok { + staticFs = fs.Fs + } + + return staticFs +} + +// StatResource looks for a resource in these filesystems in order: static, assets and finally content. +// If found in any of them, it returns FileInfo and the relevant filesystem. +// Any non os.IsNotExist error will be returned. +// An os.IsNotExist error wil be returned only if all filesystems return such an error. +// Note that if we only wanted to find the file, we could create a composite Afero fs, +// but we also need to know which filesystem root it lives in. +func (s SourceFilesystems) StatResource(lang, filename string) (fi os.FileInfo, fs afero.Fs, err error) { + for _, fsToCheck := range []afero.Fs{s.StaticFs(lang), s.Assets.Fs, s.Content.Fs} { + fs = fsToCheck + fi, err = fs.Stat(filename) + if err == nil || !os.IsNotExist(err) { + return + } + } + // Not found. + return +} + +// IsStatic returns true if the given filename is a member of one of the static +// filesystems. +func (s SourceFilesystems) IsStatic(filename string) bool { + for _, staticFs := range s.Static { + if staticFs.Contains(filename) { + return true + } + } + return false +} + +// IsContent returns true if the given filename is a member of the content filesystem. +func (s SourceFilesystems) IsContent(filename string) bool { + return s.Content.Contains(filename) +} + +// IsLayout returns true if the given filename is a member of the layouts filesystem. +func (s SourceFilesystems) IsLayout(filename string) bool { + return s.Layouts.Contains(filename) +} + +// IsData returns true if the given filename is a member of the data filesystem. +func (s SourceFilesystems) IsData(filename string) bool { + return s.Data.Contains(filename) +} + +// IsAsset returns true if the given filename is a member of the asset filesystem. +func (s SourceFilesystems) IsAsset(filename string) bool { + return s.Assets.Contains(filename) +} + +// IsI18n returns true if the given filename is a member of the i18n filesystem. +func (s SourceFilesystems) IsI18n(filename string) bool { + return s.I18n.Contains(filename) +} + +// MakeStaticPathRelative makes an absolute static filename into a relative one. +// It will return an empty string if the filename is not a member of a static filesystem. +func (s SourceFilesystems) MakeStaticPathRelative(filename string) string { + for _, staticFs := range s.Static { + rel := staticFs.MakePathRelative(filename) + if rel != "" { + return rel + } + } + return "" +} + +// MakePathRelative creates a relative path from the given filename. +// It will return an empty string if the filename is not a member of this filesystem. +func (d *SourceFilesystem) MakePathRelative(filename string) string { + + for _, dir := range d.Dirs { + meta := dir.(hugofs.FileMetaInfo).Meta() + currentPath := meta.Filename() + + if strings.HasPrefix(filename, currentPath) { + rel := strings.TrimPrefix(filename, currentPath) + if mp := meta.Path(); mp != "" { + rel = filepath.Join(mp, rel) + } + return strings.TrimPrefix(rel, filePathSeparator) + } + } + return "" +} + +func (d *SourceFilesystem) RealFilename(rel string) string { + fi, err := d.Fs.Stat(rel) + if err != nil { + return rel + } + if realfi, ok := fi.(hugofs.FileMetaInfo); ok { + return realfi.Meta().Filename() + } + + return rel +} + +// Contains returns whether the given filename is a member of the current filesystem. +func (d *SourceFilesystem) Contains(filename string) bool { + for _, dir := range d.Dirs { + if strings.HasPrefix(filename, dir.Meta().Filename()) { + return true + } + } + return false +} + +// Path returns the mount relative path to the given filename if it is a member of +// of the current filesystem, an empty string if not. +func (d *SourceFilesystem) Path(filename string) string { + for _, dir := range d.Dirs { + meta := dir.Meta() + if strings.HasPrefix(filename, meta.Filename()) { + p := strings.TrimPrefix(strings.TrimPrefix(filename, meta.Filename()), filePathSeparator) + if mountRoot := meta.MountRoot(); mountRoot != "" { + return filepath.Join(mountRoot, p) + } + return p + } + } + return "" +} + +// RealDirs gets a list of absolute paths to directories starting from the given +// path. +func (d *SourceFilesystem) RealDirs(from string) []string { + var dirnames []string + for _, dir := range d.Dirs { + meta := dir.Meta() + dirname := filepath.Join(meta.Filename(), from) + _, err := meta.Fs().Stat(from) + + if err == nil { + dirnames = append(dirnames, dirname) + } + } + return dirnames +} + +// WithBaseFs allows reuse of some potentially expensive to create parts that remain +// the same across sites/languages. +func WithBaseFs(b *BaseFs) func(*BaseFs) error { + return func(bb *BaseFs) error { + bb.theBigFs = b.theBigFs + bb.SourceFilesystems = b.SourceFilesystems + return nil + } +} + +// NewBase builds the filesystems used by Hugo given the paths and options provided.NewBase +func NewBase(p *paths.Paths, logger *loggers.Logger, options ...func(*BaseFs) error) (*BaseFs, error) { + fs := p.Fs + if logger == nil { + logger = loggers.NewWarningLogger() + } + + publishFs := hugofs.NewBaseFileDecorator(afero.NewBasePathFs(fs.Destination, p.AbsPublishDir)) + + b := &BaseFs{ + PublishFs: publishFs, + } + + for _, opt := range options { + if err := opt(b); err != nil { + return nil, err + } + } + + if b.theBigFs != nil && b.SourceFilesystems != nil { + return b, nil + } + + builder := newSourceFilesystemsBuilder(p, logger, b) + sourceFilesystems, err := builder.Build() + if err != nil { + return nil, errors.Wrap(err, "build filesystems") + } + + b.SourceFilesystems = sourceFilesystems + b.theBigFs = builder.theBigFs + + return b, nil +} + +type sourceFilesystemsBuilder struct { + logger *loggers.Logger + p *paths.Paths + sourceFs afero.Fs + result *SourceFilesystems + theBigFs *filesystemsCollector +} + +func newSourceFilesystemsBuilder(p *paths.Paths, logger *loggers.Logger, b *BaseFs) *sourceFilesystemsBuilder { + sourceFs := hugofs.NewBaseFileDecorator(p.Fs.Source) + return &sourceFilesystemsBuilder{p: p, logger: logger, sourceFs: sourceFs, theBigFs: b.theBigFs, result: &SourceFilesystems{}} +} + +func (b *sourceFilesystemsBuilder) newSourceFilesystem(name string, fs afero.Fs, dirs []hugofs.FileMetaInfo) *SourceFilesystem { + return &SourceFilesystem{ + Name: name, + Fs: fs, + Dirs: dirs, + } +} + +func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { + + if b.theBigFs == nil { + + theBigFs, err := b.createMainOverlayFs(b.p) + if err != nil { + return nil, errors.Wrap(err, "create main fs") + } + + b.theBigFs = theBigFs + } + + createView := func(componentID string) *SourceFilesystem { + if b.theBigFs == nil || b.theBigFs.overlayMounts == nil { + return b.newSourceFilesystem(componentID, hugofs.NoOpFs, nil) + } + + dirs := b.theBigFs.overlayDirs[componentID] + + return b.newSourceFilesystem(componentID, afero.NewBasePathFs(b.theBigFs.overlayMounts, componentID), dirs) + + } + + b.theBigFs.finalizeDirs() + + b.result.Archetypes = createView(files.ComponentFolderArchetypes) + b.result.Layouts = createView(files.ComponentFolderLayouts) + b.result.Assets = createView(files.ComponentFolderAssets) + b.result.ResourcesCache = b.theBigFs.overlayResources + + // Data, i18n and content cannot use the overlay fs + dataDirs := b.theBigFs.overlayDirs[files.ComponentFolderData] + dataFs, err := hugofs.NewSliceFs(dataDirs...) + if err != nil { + return nil, err + } + + b.result.Data = b.newSourceFilesystem(files.ComponentFolderData, dataFs, dataDirs) + + i18nDirs := b.theBigFs.overlayDirs[files.ComponentFolderI18n] + i18nFs, err := hugofs.NewSliceFs(i18nDirs...) + if err != nil { + return nil, err + } + b.result.I18n = b.newSourceFilesystem(files.ComponentFolderI18n, i18nFs, i18nDirs) + + contentDirs := b.theBigFs.overlayDirs[files.ComponentFolderContent] + contentBfs := afero.NewBasePathFs(b.theBigFs.overlayMountsContent, files.ComponentFolderContent) + + contentFs, err := hugofs.NewLanguageFs(b.p.LanguagesDefaultFirst.AsOrdinalSet(), contentBfs) + if err != nil { + return nil, errors.Wrap(err, "create content filesystem") + } + + b.result.Content = b.newSourceFilesystem(files.ComponentFolderContent, contentFs, contentDirs) + + b.result.Work = afero.NewReadOnlyFs(b.theBigFs.overlayFull) + + // Create static filesystem(s) + ms := make(map[string]*SourceFilesystem) + b.result.Static = ms + b.result.StaticDirs = b.theBigFs.overlayDirs[files.ComponentFolderStatic] + + if b.theBigFs.staticPerLanguage != nil { + // Multihost mode + for k, v := range b.theBigFs.staticPerLanguage { + sfs := b.newSourceFilesystem(files.ComponentFolderStatic, v, b.result.StaticDirs) + sfs.PublishFolder = k + ms[k] = sfs + } + } else { + bfs := afero.NewBasePathFs(b.theBigFs.overlayMountsStatic, files.ComponentFolderStatic) + ms[""] = b.newSourceFilesystem(files.ComponentFolderStatic, bfs, b.result.StaticDirs) + } + + return b.result, nil + +} + +func (b *sourceFilesystemsBuilder) createMainOverlayFs(p *paths.Paths) (*filesystemsCollector, error) { + + var staticFsMap map[string]afero.Fs + if b.p.Cfg.GetBool("multihost") { + staticFsMap = make(map[string]afero.Fs) + } + + collector := &filesystemsCollector{ + sourceProject: b.sourceFs, + sourceModules: hugofs.NewNoSymlinkFs(b.sourceFs, b.logger, false), + overlayDirs: make(map[string][]hugofs.FileMetaInfo), + staticPerLanguage: staticFsMap, + } + + mods := p.AllModules + + if len(mods) == 0 { + return collector, nil + } + + modsReversed := make([]mountsDescriptor, len(mods)) + + // The theme components are ordered from left to right. + // We need to revert it to get the + // overlay logic below working as expected, with the project on top. + j := 0 + for i := len(mods) - 1; i >= 0; i-- { + mod := mods[i] + dir := mod.Dir() + + isMainProject := mod.Owner() == nil + modsReversed[j] = mountsDescriptor{ + Module: mod, + dir: dir, + isMainProject: isMainProject, + } + j++ + } + + err := b.createOverlayFs(collector, modsReversed) + + return collector, err + +} + +func (b *sourceFilesystemsBuilder) isContentMount(mnt modules.Mount) bool { + return strings.HasPrefix(mnt.Target, files.ComponentFolderContent) +} + +func (b *sourceFilesystemsBuilder) isStaticMount(mnt modules.Mount) bool { + return strings.HasPrefix(mnt.Target, files.ComponentFolderStatic) +} + +func (b *sourceFilesystemsBuilder) createModFs( + collector *filesystemsCollector, + md mountsDescriptor) error { + + var ( + fromTo []hugofs.RootMapping + fromToContent []hugofs.RootMapping + fromToStatic []hugofs.RootMapping + ) + + absPathify := func(path string) (string, string) { + if filepath.IsAbs(path) { + return "", path + } + return md.dir, paths.AbsPathify(md.dir, path) + } + + for _, mount := range md.Mounts() { + + mountWeight := 1 + if md.isMainProject { + mountWeight++ + } + + base, filename := absPathify(mount.Source) + + rm := hugofs.RootMapping{ + From: mount.Target, + To: filename, + ToBasedir: base, + Module: md.Module.Path(), + Meta: hugofs.FileMeta{ + "watch": md.Watch(), + "mountWeight": mountWeight, + }, + } + + isContentMount := b.isContentMount(mount) + + lang := mount.Lang + if lang == "" && isContentMount { + lang = b.p.DefaultContentLanguage + } + + rm.Meta["lang"] = lang + + if isContentMount { + fromToContent = append(fromToContent, rm) + } else if b.isStaticMount(mount) { + fromToStatic = append(fromToStatic, rm) + } else { + fromTo = append(fromTo, rm) + } + } + + modBase := collector.sourceProject + if !md.isMainProject { + modBase = collector.sourceModules + } + sourceStatic := hugofs.NewNoSymlinkFs(modBase, b.logger, true) + + rmfs, err := hugofs.NewRootMappingFs(modBase, fromTo...) + if err != nil { + return err + } + rmfsContent, err := hugofs.NewRootMappingFs(modBase, fromToContent...) + if err != nil { + return err + } + rmfsStatic, err := hugofs.NewRootMappingFs(sourceStatic, fromToStatic...) + if err != nil { + return err + } + + // We need to keep the ordered list of directories for watching and + // some special merge operations (data, i18n). + collector.addDirs(rmfs) + collector.addDirs(rmfsContent) + collector.addDirs(rmfsStatic) + + if collector.staticPerLanguage != nil { + for _, l := range b.p.Languages { + lang := l.Lang + + lfs := rmfsStatic.Filter(func(rm hugofs.RootMapping) bool { + rlang := rm.Meta.Lang() + return rlang == "" || rlang == lang + }) + + bfs := afero.NewBasePathFs(lfs, files.ComponentFolderStatic) + + sfs, found := collector.staticPerLanguage[lang] + if found { + collector.staticPerLanguage[lang] = afero.NewCopyOnWriteFs(sfs, bfs) + + } else { + collector.staticPerLanguage[lang] = bfs + } + } + } + + getResourcesDir := func() string { + if md.isMainProject { + return b.p.AbsResourcesDir + } + _, filename := absPathify(files.FolderResources) + return filename + } + + if collector.overlayMounts == nil { + collector.overlayMounts = rmfs + collector.overlayMountsContent = rmfsContent + collector.overlayMountsStatic = rmfsStatic + collector.overlayFull = afero.NewBasePathFs(modBase, md.dir) + collector.overlayResources = afero.NewBasePathFs(modBase, getResourcesDir()) + } else { + + collector.overlayMounts = afero.NewCopyOnWriteFs(collector.overlayMounts, rmfs) + collector.overlayMountsContent = hugofs.NewLanguageCompositeFs(collector.overlayMountsContent, rmfsContent) + collector.overlayMountsStatic = hugofs.NewLanguageCompositeFs(collector.overlayMountsStatic, rmfsStatic) + collector.overlayFull = afero.NewCopyOnWriteFs(collector.overlayFull, afero.NewBasePathFs(modBase, md.dir)) + collector.overlayResources = afero.NewCopyOnWriteFs(collector.overlayResources, afero.NewBasePathFs(modBase, getResourcesDir())) + } + + return nil + +} + +func printFs(fs afero.Fs, path string, w io.Writer) { + if fs == nil { + return + } + afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + var filename string + if fim, ok := info.(hugofs.FileMetaInfo); ok { + filename = fim.Meta().Filename() + } + fmt.Fprintf(w, " %q %q\n", path, filename) + return nil + }) +} + +type filesystemsCollector struct { + sourceProject afero.Fs // Source for project folders + sourceModules afero.Fs // Source for modules/themes + + overlayMounts afero.Fs + overlayMountsContent afero.Fs + overlayMountsStatic afero.Fs + overlayFull afero.Fs + overlayResources afero.Fs + + // Maps component type (layouts, static, content etc.) an ordered list of + // directories representing the overlay filesystems above. + overlayDirs map[string][]hugofs.FileMetaInfo + + // Set if in multihost mode + staticPerLanguage map[string]afero.Fs + + finalizerInit sync.Once +} + +func (c *filesystemsCollector) addDirs(rfs *hugofs.RootMappingFs) { + for _, componentFolder := range files.ComponentFolders { + dirs, err := rfs.Dirs(componentFolder) + + if err == nil { + c.overlayDirs[componentFolder] = append(c.overlayDirs[componentFolder], dirs...) + } + } +} + +func (c *filesystemsCollector) finalizeDirs() { + c.finalizerInit.Do(func() { + // Order the directories from top to bottom (project, theme a, theme ...). + for _, dirs := range c.overlayDirs { + c.reverseFis(dirs) + } + }) + +} + +func (c *filesystemsCollector) reverseFis(fis []hugofs.FileMetaInfo) { + for i := len(fis)/2 - 1; i >= 0; i-- { + opp := len(fis) - 1 - i + fis[i], fis[opp] = fis[opp], fis[i] + } +} + +type mountsDescriptor struct { + modules.Module + dir string + isMainProject bool +} + +func (b *sourceFilesystemsBuilder) createOverlayFs(collector *filesystemsCollector, mounts []mountsDescriptor) error { + if len(mounts) == 0 { + return nil + } + + err := b.createModFs(collector, mounts[0]) + if err != nil { + return err + } + + if len(mounts) == 1 { + return nil + } + + return b.createOverlayFs(collector, mounts[1:]) +} |