summaryrefslogtreecommitdiffstats
path: root/hugolib/filesystems
diff options
context:
space:
mode:
authorBjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>2023-12-24 19:11:05 +0100
committerBjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>2024-01-27 16:28:14 +0100
commit7285e74090852b5d52f25e577850fa75f4aa8573 (patch)
tree54d07cb4a7de2db5c89f2590266595f0aca6cbd6 /hugolib/filesystems
parent5fd1e7490305570872d3899f5edda950903c5213 (diff)
downloadhugo-develop2024.tar.gz
hugo-develop2024.zip
all: Rework page store, add a dynacache, improve partial rebuilds, and some general spring cleaningdevelop2024
There are some breaking changes in this commit, see #11455. Closes #11455 Closes #11549 This fixes a set of bugs (see issue list) and it is also paying some technical debt accumulated over the years. We now build with Staticcheck enabled in the CI build. The performance should be about the same as before for regular sized Hugo sites, but it should perform and scale much better to larger data sets, as objects that uses lots of memory (e.g. rendered Markdown, big JSON files read into maps with transform.Unmarshal etc.) will now get automatically garbage collected if needed. Performance on partial rebuilds when running the server in fast render mode should be the same, but the change detection should be much more accurate. A list of the notable new features: * A new dependency tracker that covers (almost) all of Hugo's API and is used to do fine grained partial rebuilds when running the server. * A new and simpler tree document store which allows fast lookups and prefix-walking in all dimensions (e.g. language) concurrently. * You can now configure an upper memory limit allowing for much larger data sets and/or running on lower specced PCs. We have lifted the "no resources in sub folders" restriction for branch bundles (e.g. sections). Memory Limit * Hugos will, by default, set aside a quarter of the total system memory, but you can set this via the OS environment variable HUGO_MEMORYLIMIT (in gigabytes). This is backed by a partitioned LRU cache used throughout Hugo. A cache that gets dynamically resized in low memory situations, allowing Go's Garbage Collector to free the memory. New Dependency Tracker: Hugo has had a rule based coarse grained approach to server rebuilds that has worked mostly pretty well, but there have been some surprises (e.g. stale content). This is now revamped with a new dependency tracker that can quickly calculate the delta given a changed resource (e.g. a content file, template, JS file etc.). This handles transitive relations, e.g. $page -> js.Build -> JS import, or $page1.Content -> render hook -> site.GetPage -> $page2.Title, or $page1.Content -> shortcode -> partial -> site.RegularPages -> $page2.Content -> shortcode ..., and should also handle changes to aggregated values (e.g. site.Lastmod) effectively. This covers all of Hugo's API with 2 known exceptions (a list that may not be fully exhaustive): Changes to files loaded with template func os.ReadFile may not be handled correctly. We recommend loading resources with resources.Get Changes to Hugo objects (e.g. Page) passed in the template context to lang.Translate may not be detected correctly. We recommend having simple i18n templates without too much data context passed in other than simple types such as strings and numbers. Note that the cachebuster configuration (when A changes then rebuild B) works well with the above, but we recommend that you revise that configuration, as it in most situations should not be needed. One example where it is still needed is with TailwindCSS and using changes to hugo_stats.json to trigger new CSS rebuilds. Document Store: Previously, a little simplified, we split the document store (where we store pages and resources) in a tree per language. This worked pretty well, but the structure made some operations harder than they needed to be. We have now restructured it into one Radix tree for all languages. Internally the language is considered to be a dimension of that tree, and the tree can be viewed in all dimensions concurrently. This makes some operations re. language simpler (e.g. finding translations is just a slice range), but the idea is that it should also be relatively inexpensive to add more dimensions if needed (e.g. role). Fixes #10169 Fixes #10364 Fixes #10482 Fixes #10630 Fixes #10656 Fixes #10694 Fixes #10918 Fixes #11262 Fixes #11439 Fixes #11453 Fixes #11457 Fixes #11466 Fixes #11540 Fixes #11551 Fixes #11556 Fixes #11654 Fixes #11661 Fixes #11663 Fixes #11664 Fixes #11669 Fixes #11671 Fixes #11807 Fixes #11808 Fixes #11809 Fixes #11815 Fixes #11840 Fixes #11853 Fixes #11860 Fixes #11883 Fixes #11904 Fixes #7388 Fixes #7425 Fixes #7436 Fixes #7544 Fixes #7882 Fixes #7960 Fixes #8255 Fixes #8307 Fixes #8863 Fixes #8927 Fixes #9192 Fixes #9324
Diffstat (limited to 'hugolib/filesystems')
-rw-r--r--hugolib/filesystems/basefs.go441
-rw-r--r--hugolib/filesystems/basefs_test.go319
2 files changed, 431 insertions, 329 deletions
diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go
index a7c2a6271..b37fb8cb4 100644
--- a/hugolib/filesystems/basefs.go
+++ b/hugolib/filesystems/basefs.go
@@ -19,12 +19,12 @@ import (
"fmt"
"io"
"os"
- "path"
"path/filepath"
"strings"
"sync"
"github.com/bep/overlayfs"
+ "github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/htesting"
"github.com/gohugoio/hugo/hugofs/glob"
@@ -60,9 +60,12 @@ type BaseFs struct {
// SourceFilesystems contains the different source file systems.
*SourceFilesystems
- // The project source.
+ // The source filesystem (needs absolute filenames).
SourceFs afero.Fs
+ // The project source.
+ ProjectSourceFs afero.Fs
+
// The filesystem used to publish the rendered site.
// This usually maps to /my-project/public.
PublishFs afero.Fs
@@ -95,67 +98,84 @@ func (f *fakeLockfileMutex) Lock() (func(), error) {
}
// Tries to acquire a build lock.
-func (fs *BaseFs) LockBuild() (unlock func(), err error) {
- return fs.buildMu.Lock()
+func (b *BaseFs) LockBuild() (unlock func(), err error) {
+ return b.buildMu.Lock()
}
-// TODO(bep) we can get regular files in here and that is fine, but
-// we need to clean up the naming.
-func (fs *BaseFs) WatchDirs() []hugofs.FileMetaInfo {
- var dirs []hugofs.FileMetaInfo
- for _, dir := range fs.AllDirs() {
- if dir.Meta().Watch {
- dirs = append(dirs, dir)
- }
- }
- return dirs
-}
+func (b *BaseFs) WatchFilenames() []string {
+ var filenames []string
+ sourceFs := b.SourceFs
+
+ for _, rfs := range b.RootFss {
+ for _, component := range files.ComponentFolders {
+ fis, err := rfs.Mounts(component)
+ if err != nil {
+ continue
+ }
+
+ for _, fim := range fis {
+ meta := fim.Meta()
+ if !meta.Watch {
+ continue
+ }
+
+ if !fim.IsDir() {
+ filenames = append(filenames, meta.Filename)
+ continue
+ }
+
+ w := hugofs.NewWalkway(hugofs.WalkwayConfig{
+ Fs: sourceFs,
+ Root: meta.Filename,
+ WalkFn: func(path string, fi hugofs.FileMetaInfo) error {
+ if !fi.IsDir() {
+ return nil
+ }
+ if fi.Name() == ".git" ||
+ fi.Name() == "node_modules" || fi.Name() == "bower_components" {
+ return filepath.SkipDir
+ }
+ filenames = append(filenames, fi.Meta().Filename)
+ return nil
+ },
+ })
+
+ w.Walk()
+ }
-func (fs *BaseFs) AllDirs() []hugofs.FileMetaInfo {
- var dirs []hugofs.FileMetaInfo
- for _, dirSet := range [][]hugofs.FileMetaInfo{
- fs.Archetypes.Dirs,
- fs.I18n.Dirs,
- fs.Data.Dirs,
- fs.Content.Dirs,
- fs.Assets.Dirs,
- fs.Layouts.Dirs,
- // fs.Resources.Dirs,
- fs.StaticDirs,
- } {
- dirs = append(dirs, dirSet...)
+ }
}
- return dirs
+ return filenames
}
-// RelContentDir tries to create a path relative to the content root from
-// the given filename. The return value is the path and language code.
-func (b *BaseFs) RelContentDir(filename string) string {
- for _, dir := range b.SourceFilesystems.Content.Dirs {
- dirname := dir.Meta().Filename
- if strings.HasPrefix(filename, dirname) {
- rel := path.Join(dir.Meta().Path, strings.TrimPrefix(filename, dirname))
- return strings.TrimPrefix(rel, filePathSeparator)
+func (b *BaseFs) mountsForComponent(component string) []hugofs.FileMetaInfo {
+ var result []hugofs.FileMetaInfo
+ for _, rfs := range b.RootFss {
+ dirs, err := rfs.Mounts(component)
+ if err == nil {
+ result = append(result, dirs...)
}
}
- // Either not a content dir or already relative.
- return filename
+ return result
}
// AbsProjectContentDir tries to construct a filename below the most
// relevant content directory.
func (b *BaseFs) AbsProjectContentDir(filename string) (string, string, error) {
isAbs := filepath.IsAbs(filename)
- for _, dir := range b.SourceFilesystems.Content.Dirs {
- meta := dir.Meta()
+ for _, fi := range b.mountsForComponent(files.ComponentFolderContent) {
+ if !fi.IsDir() {
+ continue
+ }
+ meta := fi.Meta()
if !meta.IsProject {
continue
}
if isAbs {
if strings.HasPrefix(filename, meta.Filename) {
- return strings.TrimPrefix(filename, meta.Filename), filename, nil
+ return strings.TrimPrefix(filename, meta.Filename+filePathSeparator), filename, nil
}
} else {
contentDir := strings.TrimPrefix(strings.TrimPrefix(meta.Filename, meta.BaseDir), filePathSeparator) + filePathSeparator
@@ -173,7 +193,10 @@ func (b *BaseFs) AbsProjectContentDir(filename string) (string, string, error) {
// A filename on the form "posts/mypage.md", put it inside
// the first content folder, usually <workDir>/content.
// Pick the first project dir (which is probably the most important one).
- for _, dir := range b.SourceFilesystems.Content.Dirs {
+ for _, dir := range b.SourceFilesystems.Content.mounts() {
+ if !dir.IsDir() {
+ continue
+ }
meta := dir.Meta()
if meta.IsProject {
return filename, filepath.Join(meta.Filename, filename), nil
@@ -186,14 +209,14 @@ func (b *BaseFs) AbsProjectContentDir(filename string) (string, string, error) {
// ResolveJSConfigFile resolves the JS-related config file to a absolute
// filename. One example of such would be postcss.config.js.
-func (fs *BaseFs) ResolveJSConfigFile(name string) string {
+func (b *BaseFs) ResolveJSConfigFile(name string) string {
// First look in assets/_jsconfig
- fi, err := fs.Assets.Fs.Stat(filepath.Join(files.FolderJSConfig, name))
+ fi, err := b.Assets.Fs.Stat(filepath.Join(files.FolderJSConfig, name))
if err == nil {
return fi.(hugofs.FileMetaInfo).Meta().Filename
}
// Fall back to the work dir.
- fi, err = fs.Work.Stat(name)
+ fi, err = b.Work.Stat(name)
if err == nil {
return fi.(hugofs.FileMetaInfo).Meta().Filename
}
@@ -201,27 +224,6 @@ func (fs *BaseFs) ResolveJSConfigFile(name string) string {
return ""
}
-// MakePathRelative creates a relative path from the given filename.
-// It returns both the component name (e.g. layouts) and the path relative to that.
-func (fs *BaseFs) MakePathRelative(filename string) (string, string) {
- for _, sfs := range fs.FileSystems() {
- if sfs.Contains(filename) {
- if s, found := sfs.MakePathRelative(filename); found {
- return sfs.Name, s
- }
- }
- }
- // May be a static file.
- if s := fs.MakeStaticPathRelative(filename); s != "" {
- return files.ComponentFolderStatic, s
- }
- // Fall back to relative to the working dir.
- if strings.HasPrefix(filename, fs.workingDir) {
- return "", strings.TrimPrefix(filename, fs.workingDir)
- }
- return "", ""
-}
-
// SourceFilesystems contains the different source file systems. These can be
// composite file systems (theme and project etc.), and they have all root
// set to the source type the provides: data, i18n, static, layouts.
@@ -233,6 +235,10 @@ type SourceFilesystems struct {
Archetypes *SourceFilesystem
Assets *SourceFilesystem
+ AssetsWithDuplicatesPreserved *SourceFilesystem
+
+ RootFss []*hugofs.RootMappingFs
+
// Writable filesystem on top the project's resources directory,
// with any sub module's resource fs layered below.
ResourcesCache afero.Fs
@@ -246,23 +252,7 @@ type SourceFilesystems struct {
// When in non-multihost mode there will be one entry in this map with a blank key.
Static map[string]*SourceFilesystem
- // All the /static dirs (including themes/modules).
- StaticDirs []hugofs.FileMetaInfo
-}
-
-// FileSystems returns the FileSystems relevant for the change detection
-// in server mode.
-// Note: This does currently not return any static fs.
-func (s *SourceFilesystems) FileSystems() []*SourceFilesystem {
- return []*SourceFilesystem{
- s.Content,
- s.Assets,
- s.Data,
- s.I18n,
- s.Layouts,
- s.Archetypes,
- // TODO(bep) static
- }
+ conf config.AllProvider
}
// A SourceFilesystem holds the filesystem for a given source type in Hugo (data,
@@ -275,32 +265,12 @@ type SourceFilesystem struct {
// This is a virtual composite filesystem. It expects path relative to a context.
Fs afero.Fs
- // This filesystem as separate root directories, starting from project and down
- // to the themes/modules.
- Dirs []hugofs.FileMetaInfo
-
// When syncing a source folder to the target (e.g. /public), this may
// be set to publish into a subfolder. This is used for static syncing
// in multihost mode.
PublishFolder string
}
-// ContentStaticAssetFs will create a new composite filesystem from the content,
-// static, and asset filesystems. The site language is needed to pick the correct static filesystem.
-// The order is content, static and then assets.
-// TODO(bep) check usage
-func (s SourceFilesystems) ContentStaticAssetFs(lang string) afero.Fs {
- return overlayfs.New(
- overlayfs.Options{
- Fss: []afero.Fs{
- s.Content.Fs,
- s.StaticFs(lang),
- s.Assets.Fs,
- },
- },
- )
-}
-
// StaticFs returns the static filesystem for the given language.
// This can be a composite filesystem.
func (s SourceFilesystems) StaticFs(lang string) afero.Fs {
@@ -349,24 +319,17 @@ func (s SourceFilesystems) IsContent(filename string) bool {
return s.Content.Contains(filename)
}
-// IsLayout returns true if the given filename is a member of the layouts filesystem.
-func (s SourceFilesystems) IsLayout(filename string) bool {
- return s.Layouts.Contains(filename)
-}
-
-// IsData returns true if the given filename is a member of the data filesystem.
-func (s SourceFilesystems) IsData(filename string) bool {
- return s.Data.Contains(filename)
-}
-
-// IsAsset returns true if the given filename is a member of the asset filesystem.
-func (s SourceFilesystems) IsAsset(filename string) bool {
- return s.Assets.Contains(filename)
-}
-
-// IsI18n returns true if the given filename is a member of the i18n filesystem.
-func (s SourceFilesystems) IsI18n(filename string) bool {
- return s.I18n.Contains(filename)
+// ResolvePaths resolves the given filename to a list of paths in the filesystems.
+func (s *SourceFilesystems) ResolvePaths(filename string, checkExists bool) []hugofs.ComponentPath {
+ var cpss []hugofs.ComponentPath
+ for _, rfs := range s.RootFss {
+ cps, err := rfs.ReverseLookup(filename, checkExists)
+ if err != nil {
+ panic(err)
+ }
+ cpss = append(cpss, cps...)
+ }
+ return cpss
}
// MakeStaticPathRelative makes an absolute static filename into a relative one.
@@ -383,19 +346,53 @@ func (s SourceFilesystems) MakeStaticPathRelative(filename string) string {
// MakePathRelative creates a relative path from the given filename.
func (d *SourceFilesystem) MakePathRelative(filename string) (string, bool) {
- for _, dir := range d.Dirs {
- meta := dir.(hugofs.FileMetaInfo).Meta()
- currentPath := meta.Filename
-
- if strings.HasPrefix(filename, currentPath) {
- rel := strings.TrimPrefix(filename, currentPath)
- if mp := meta.Path; mp != "" {
- rel = filepath.Join(mp, rel)
+ cps, err := d.ReverseLookup(filename)
+ if err != nil {
+ panic(err)
+ }
+ if len(cps) == 0 {
+ return "", false
+ }
+
+ return filepath.FromSlash(cps[0].Path), true
+}
+
+// ReverseLookup returns the component paths for the given filename.
+func (d *SourceFilesystem) ReverseLookup(filename string) ([]hugofs.ComponentPath, error) {
+ var cps []hugofs.ComponentPath
+ hugofs.WalkFilesystems(d.Fs, func(fs afero.Fs) bool {
+ if rfs, ok := fs.(hugofs.ReverseLookupProvder); ok {
+ if c, err := rfs.ReverseLookup(filename, true); err == nil {
+ cps = append(cps, c...)
}
- return strings.TrimPrefix(rel, filePathSeparator), true
+ }
+ return false
+ })
+ return cps, nil
+}
+
+func (d *SourceFilesystem) mounts() []hugofs.FileMetaInfo {
+ var m []hugofs.FileMetaInfo
+ hugofs.WalkFilesystems(d.Fs, func(fs afero.Fs) bool {
+ if rfs, ok := fs.(*hugofs.RootMappingFs); ok {
+ mounts, err := rfs.Mounts(d.Name)
+ if err == nil {
+ m = append(m, mounts...)
+ }
+
+ }
+ return false
+ })
+ // Filter out any mounts not belonging to this filesystem.
+ n := 0
+ for _, mm := range m {
+ if mm.Meta().Component == d.Name {
+ m[n] = mm
+ n++
}
}
- return "", false
+ m = m[:n]
+ return m
}
func (d *SourceFilesystem) RealFilename(rel string) string {
@@ -412,7 +409,10 @@ func (d *SourceFilesystem) RealFilename(rel string) string {
// Contains returns whether the given filename is a member of the current filesystem.
func (d *SourceFilesystem) Contains(filename string) bool {
- for _, dir := range d.Dirs {
+ for _, dir := range d.mounts() {
+ if !dir.IsDir() {
+ continue
+ }
if strings.HasPrefix(filename, dir.Meta().Filename) {
return true
}
@@ -420,32 +420,18 @@ func (d *SourceFilesystem) Contains(filename string) bool {
return false
}
-// Path returns the mount relative path to the given filename if it is a member of
-// of the current filesystem, an empty string if not.
-func (d *SourceFilesystem) Path(filename string) string {
- for _, dir := range d.Dirs {
- meta := dir.Meta()
- if strings.HasPrefix(filename, meta.Filename) {
- p := strings.TrimPrefix(strings.TrimPrefix(filename, meta.Filename), filePathSeparator)
- if mountRoot := meta.MountRoot; mountRoot != "" {
- return filepath.Join(mountRoot, p)
- }
- return p
- }
- }
- return ""
-}
-
// RealDirs gets a list of absolute paths to directories starting from the given
// path.
func (d *SourceFilesystem) RealDirs(from string) []string {
var dirnames []string
- for _, dir := range d.Dirs {
- meta := dir.Meta()
- dirname := filepath.Join(meta.Filename, from)
- _, err := meta.Fs.Stat(from)
-
+ for _, m := range d.mounts() {
+ if !m.IsDir() {
+ continue
+ }
+ meta := m.Meta()
+ _, err := d.Fs.Stat(from)
if err == nil {
+ dirname := filepath.Join(meta.Filename, from)
dirnames = append(dirnames, dirname)
}
}
@@ -462,8 +448,6 @@ func WithBaseFs(b *BaseFs) func(*BaseFs) error {
}
}
-var counter int
-
// NewBase builds the filesystems used by Hugo given the paths and options provided.NewBase
func NewBase(p *paths.Paths, logger loggers.Logger, options ...func(*BaseFs) error) (*BaseFs, error) {
fs := p.Fs
@@ -472,7 +456,8 @@ func NewBase(p *paths.Paths, logger loggers.Logger, options ...func(*BaseFs) err
}
publishFs := hugofs.NewBaseFileDecorator(fs.PublishDir)
- sourceFs := hugofs.NewBaseFileDecorator(afero.NewBasePathFs(fs.Source, p.Cfg.BaseConfig().WorkingDir))
+ projectSourceFs := hugofs.NewBaseFileDecorator(hugofs.NewBasePathFs(fs.Source, p.Cfg.BaseConfig().WorkingDir))
+ sourceFs := hugofs.NewBaseFileDecorator(fs.Source)
publishFsStatic := fs.PublishDirStatic
var buildMu Lockable
@@ -484,6 +469,7 @@ func NewBase(p *paths.Paths, logger loggers.Logger, options ...func(*BaseFs) err
b := &BaseFs{
SourceFs: sourceFs,
+ ProjectSourceFs: projectSourceFs,
WorkDir: fs.WorkingDirReadOnly,
PublishFs: publishFs,
PublishFsStatic: publishFsStatic,
@@ -523,14 +509,18 @@ type sourceFilesystemsBuilder struct {
func newSourceFilesystemsBuilder(p *paths.Paths, logger loggers.Logger, b *BaseFs) *sourceFilesystemsBuilder {
sourceFs := hugofs.NewBaseFileDecorator(p.Fs.Source)
- return &sourceFilesystemsBuilder{p: p, logger: logger, sourceFs: sourceFs, theBigFs: b.theBigFs, result: &SourceFilesystems{}}
+ return &sourceFilesystemsBuilder{
+ p: p, logger: logger, sourceFs: sourceFs, theBigFs: b.theBigFs,
+ result: &SourceFilesystems{
+ conf: p.Cfg,
+ },
+ }
}
-func (b *sourceFilesystemsBuilder) newSourceFilesystem(name string, fs afero.Fs, dirs []hugofs.FileMetaInfo) *SourceFilesystem {
+func (b *sourceFilesystemsBuilder) newSourceFilesystem(name string, fs afero.Fs) *SourceFilesystem {
return &SourceFilesystem{
Name: name,
Fs: fs,
- Dirs: dirs,
}
}
@@ -544,64 +534,61 @@ func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) {
b.theBigFs = theBigFs
}
- createView := func(componentID string) *SourceFilesystem {
+ createView := func(componentID string, overlayFs *overlayfs.OverlayFs) *SourceFilesystem {
if b.theBigFs == nil || b.theBigFs.overlayMounts == nil {
- return b.newSourceFilesystem(componentID, hugofs.NoOpFs, nil)
+ return b.newSourceFilesystem(componentID, hugofs.NoOpFs)
}
- dirs := b.theBigFs.overlayDirs[componentID]
+ fs := hugofs.NewComponentFs(
+ hugofs.ComponentFsOptions{
+ Fs: overlayFs,
+ Component: componentID,
+ DefaultContentLanguage: b.p.Cfg.DefaultContentLanguage(),
+ PathParser: b.p.Cfg.PathParser(),
+ },
+ )
- return b.newSourceFilesystem(componentID, afero.NewBasePathFs(b.theBigFs.overlayMounts, componentID), dirs)
+ return b.newSourceFilesystem(componentID, fs)
}
- b.result.Archetypes = createView(files.ComponentFolderArchetypes)
- b.result.Layouts = createView(files.ComponentFolderLayouts)
- b.result.Assets = createView(files.ComponentFolderAssets)
+ b.result.Archetypes = createView(files.ComponentFolderArchetypes, b.theBigFs.overlayMounts)
+ b.result.Layouts = createView(files.ComponentFolderLayouts, b.theBigFs.overlayMounts)
+ b.result.Assets = createView(files.ComponentFolderAssets, b.theBigFs.overlayMounts)
b.result.ResourcesCache = b.theBigFs.overlayResources
+ b.result.RootFss = b.theBigFs.rootFss
+
+ // data and i18n needs a different merge strategy.
+ overlayMountsPreserveDupes := b.theBigFs.overlayMounts.WithDirsMerger(hugofs.AppendDirsMerger)
+ b.result.Data = createView(files.ComponentFolderData, overlayMountsPreserveDupes)
+ b.result.I18n = createView(files.ComponentFolderI18n, overlayMountsPreserveDupes)
+ b.result.AssetsWithDuplicatesPreserved = createView(files.ComponentFolderAssets, overlayMountsPreserveDupes)
+
+ contentFs := hugofs.NewComponentFs(
+ hugofs.ComponentFsOptions{
+ Fs: b.theBigFs.overlayMountsContent,
+ Component: files.ComponentFolderContent,
+ DefaultContentLanguage: b.p.Cfg.DefaultContentLanguage(),
+ PathParser: b.p.Cfg.PathParser(),
+ },
+ )
- // Data, i18n and content cannot use the overlay fs
- dataDirs := b.theBigFs.overlayDirs[files.ComponentFolderData]
- dataFs, err := hugofs.NewSliceFs(dataDirs...)
- if err != nil {
- return nil, err
- }
-
- b.result.Data = b.newSourceFilesystem(files.ComponentFolderData, dataFs, dataDirs)
-
- i18nDirs := b.theBigFs.overlayDirs[files.ComponentFolderI18n]
- i18nFs, err := hugofs.NewSliceFs(i18nDirs...)
- if err != nil {
- return nil, err
- }
- b.result.I18n = b.newSourceFilesystem(files.ComponentFolderI18n, i18nFs, i18nDirs)
-
- contentDirs := b.theBigFs.overlayDirs[files.ComponentFolderContent]
- contentBfs := afero.NewBasePathFs(b.theBigFs.overlayMountsContent, files.ComponentFolderContent)
-
- contentFs, err := hugofs.NewLanguageFs(b.p.Cfg.LanguagesDefaultFirst().AsOrdinalSet(), contentBfs)
- if err != nil {
- return nil, fmt.Errorf("create content filesystem: %w", err)
- }
-
- b.result.Content = b.newSourceFilesystem(files.ComponentFolderContent, contentFs, contentDirs)
-
- b.result.Work = afero.NewReadOnlyFs(b.theBigFs.overlayFull)
+ b.result.Content = b.newSourceFilesystem(files.ComponentFolderContent, contentFs)
+ b.result.Work = hugofs.NewReadOnlyFs(b.theBigFs.overlayFull)
// Create static filesystem(s)
ms := make(map[string]*SourceFilesystem)
b.result.Static = ms
- b.result.StaticDirs = b.theBigFs.overlayDirs[files.ComponentFolderStatic]
if b.theBigFs.staticPerLanguage != nil {
// Multihost mode
for k, v := range b.theBigFs.staticPerLanguage {
- sfs := b.newSourceFilesystem(files.ComponentFolderStatic, v, b.result.StaticDirs)
+ sfs := b.newSourceFilesystem(files.ComponentFolderStatic, v)
sfs.PublishFolder = k
ms[k] = sfs
}
} else {
- bfs := afero.NewBasePathFs(b.theBigFs.overlayMountsStatic, files.ComponentFolderStatic)
- ms[""] = b.newSourceFilesystem(files.ComponentFolderStatic, bfs, b.result.StaticDirs)
+ bfs := hugofs.NewBasePathFs(b.theBigFs.overlayMountsStatic, files.ComponentFolderStatic)
+ ms[""] = b.newSourceFilesystem(files.ComponentFolderStatic, bfs)
}
return b.result, nil
@@ -619,8 +606,7 @@ func (b *sourceFilesystemsBuilder) createMainOverlayFs(p *paths.Paths) (*filesys
collector := &filesystemsCollector{
sourceProject: b.sourceFs,
- sourceModules: hugofs.NewNoSymlinkFs(b.sourceFs, b.logger, false),
- overlayDirs: make(map[string][]hugofs.FileMetaInfo),
+ sourceModules: b.sourceFs,
staticPerLanguage: staticFsMap,
overlayMounts: overlayfs.New(overlayfs.Options{}),
@@ -675,6 +661,7 @@ func (b *sourceFilesystemsBuilder) createOverlayFs(
collector.overlayMounts = appendNopIfEmpty(collector.overlayMounts)
collector.overlayMountsContent = appendNopIfEmpty(collector.overlayMountsContent)
collector.overlayMountsStatic = appendNopIfEmpty(collector.overlayMountsStatic)
+ collector.overlayMountsFull = appendNopIfEmpty(collector.overlayMountsFull)
collector.overlayFull = appendNopIfEmpty(collector.overlayFull)
collector.overlayResources = appendNopIfEmpty(collector.overlayResources)
@@ -712,15 +699,15 @@ func (b *sourceFilesystemsBuilder) createOverlayFs(
base, filename := absPathify(mount.Source)
rm := hugofs.RootMapping{
- From: mount.Target,
- To: filename,
- ToBasedir: base,
- Module: md.Module.Path(),
- IsProject: md.isMainProject,
+ From: mount.Target,
+ To: filename,
+ ToBase: base,
+ Module: md.Module.Path(),
+ ModuleOrdinal: md.ordinal,
+ IsProject: md.isMainProject,
Meta: &hugofs.FileMeta{
Watch: md.Watch(),
Weight: mountWeight,
- Classifier: files.ContentClassContent,
InclusionFilter: inclusionFilter,
},
}
@@ -747,7 +734,8 @@ func (b *sourceFilesystemsBuilder) createOverlayFs(
if !md.isMainProject {
modBase = collector.sourceModules
}
- sourceStatic := hugofs.NewNoSymlinkFs(modBase, b.logger, true)
+
+ sourceStatic := modBase
rmfs, err := hugofs.NewRootMappingFs(modBase, fromTo...)
if err != nil {
@@ -762,11 +750,10 @@ func (b *sourceFilesystemsBuilder) createOverlayFs(
return err
}
- // We need to keep the ordered list of directories for watching and
- // some special merge operations (data, i18n).
- collector.addDirs(rmfs)
- collector.addDirs(rmfsContent)
- collector.addDirs(rmfsStatic)
+ // We need to keep the list of directories for watching.
+ collector.addRootFs(rmfs)
+ collector.addRootFs(rmfsContent)
+ collector.addRootFs(rmfsStatic)
if collector.staticPerLanguage != nil {
for _, l := range b.p.Cfg.Languages() {
@@ -776,7 +763,7 @@ func (b *sourceFilesystemsBuilder) createOverlayFs(
rlang := rm.Meta.Lang
return rlang == "" || rlang == lang
})
- bfs := afero.NewBasePathFs(lfs, files.ComponentFolderStatic)
+ bfs := hugofs.NewBasePathFs(lfs, files.ComponentFolderStatic)
collector.staticPerLanguage[lang] = collector.staticPerLanguage[lang].Append(bfs)
}
}
@@ -792,14 +779,15 @@ func (b *sourceFilesystemsBuilder) createOverlayFs(
collector.overlayMounts = collector.overlayMounts.Append(rmfs)
collector.overlayMountsContent = collector.overlayMountsContent.Append(rmfsContent)
collector.overlayMountsStatic = collector.overlayMountsStatic.Append(rmfsStatic)
- collector.overlayFull = collector.overlayFull.Append(afero.NewBasePathFs(modBase, md.dir))
- collector.overlayResources = collector.overlayResources.Append(afero.NewBasePathFs(modBase, getResourcesDir()))
+ collector.overlayFull = collector.overlayFull.Append(hugofs.NewBasePathFs(modBase, md.dir))
+ collector.overlayResources = collector.overlayResources.Append(hugofs.NewBasePathFs(modBase, getResourcesDir()))
}
return nil
}
+//lint:ignore U1000 useful for debugging
func printFs(fs afero.Fs, path string, w io.Writer) {
if fs == nil {
return
@@ -827,36 +815,23 @@ type filesystemsCollector struct {
overlayMounts *overlayfs.OverlayFs
overlayMountsContent *overlayfs.OverlayFs
overlayMountsStatic *overlayfs.OverlayFs
+ overlayMountsFull *overlayfs.OverlayFs
overlayFull *overlayfs.OverlayFs
overlayResources *overlayfs.OverlayFs
- // Maps component type (layouts, static, content etc.) an ordered list of
- // directories representing the overlay filesystems above.
- overlayDirs map[string][]hugofs.FileMetaInfo
+ rootFss []*hugofs.RootMappingFs
// Set if in multihost mode
staticPerLanguage map[string]*overlayfs.OverlayFs
-
- finalizerInit sync.Once
}
-func (c *filesystemsCollector) addDirs(rfs *hugofs.RootMappingFs) {
- for _, componentFolder := range files.ComponentFolders {
- c.addDir(rfs, componentFolder)
- }
-}
-
-func (c *filesystemsCollector) addDir(rfs *hugofs.RootMappingFs, componentFolder string) {
- dirs, err := rfs.Dirs(componentFolder)
-
- if err == nil {
- c.overlayDirs[componentFolder] = append(c.overlayDirs[componentFolder], dirs...)
- }
+func (c *filesystemsCollector) addRootFs(rfs *hugofs.RootMappingFs) {
+ c.rootFss = append(c.rootFss, rfs)
}
type mountsDescriptor struct {
modules.Module
dir string
isMainProject bool
- ordinal int
+ ordinal int // zero based starting from the project.
}
diff --git a/hugolib/filesystems/basefs_test.go b/hugolib/filesystems/basefs_test.go
index 1724f3838..5398055ed 100644
--- a/hugolib/filesystems/basefs_test.go
+++ b/hugolib/filesystems/basefs_test.go
@@ -1,4 +1,4 @@
-// Copyright 2023 The Hugo Authors. All rights reserved.
+// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -18,11 +18,13 @@ import (
"fmt"
"os"
"path/filepath"
+ "runtime"
"strings"
"testing"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/config/testconfig"
+ "github.com/gohugoio/hugo/hugolib"
"github.com/spf13/afero"
@@ -54,22 +56,22 @@ func TestNewBaseFs(t *testing.T) {
base := filepath.Join(workingDir, "themes", theme, dir)
filenameTheme := filepath.Join(base, fmt.Sprintf("theme-file-%s.txt", theme))
filenameOverlap := filepath.Join(base, "f3.txt")
- afs.Mkdir(base, 0755)
+ afs.Mkdir(base, 0o755)
content := []byte(fmt.Sprintf("content:%s:%s", theme, dir))
- afero.WriteFile(afs, filenameTheme, content, 0755)
- afero.WriteFile(afs, filenameOverlap, content, 0755)
+ afero.WriteFile(afs, filenameTheme, content, 0o755)
+ afero.WriteFile(afs, filenameOverlap, content, 0o755)
}
// Write some files to the root of the theme
base := filepath.Join(workingDir, "themes", theme)
- afero.WriteFile(afs, filepath.Join(base, fmt.Sprintf("theme-root-%s.txt", theme)), []byte(fmt.Sprintf("content:%s", theme)), 0755)
- afero.WriteFile(afs, filepath.Join(base, "file-theme-root.txt"), []byte(fmt.Sprintf("content:%s", theme)), 0755)
+ afero.WriteFile(afs, filepath.Join(base, fmt.Sprintf("theme-root-%s.txt", theme)), []byte(fmt.Sprintf("content:%s", theme)), 0o755)
+ afero.WriteFile(afs, filepath.Join(base, "file-theme-root.txt"), []byte(fmt.Sprintf("content:%s", theme)), 0o755)
}
- afero.WriteFile(afs, filepath.Join(workingDir, "file-root.txt"), []byte("content-project"), 0755)
+ afero.WriteFile(afs, filepath.Join(workingDir, "file-root.txt"), []byte("content-project"), 0o755)
afero.WriteFile(afs, filepath.Join(workingDir, "themes", "btheme", "config.toml"), []byte(`
theme = ["atheme"]
-`), 0755)
+`), 0o755)
setConfigAndWriteSomeFilesTo(afs, v, "contentDir", "mycontent", 3)
setConfigAndWriteSomeFilesTo(afs, v, "i18nDir", "myi18n", 4)
@@ -113,17 +115,10 @@ theme = ["atheme"]
checkFileCount(bfs.Assets.Fs, "", c, 9)
checkFileCount(bfs.Work, "", c, 90)
- c.Assert(bfs.IsData(filepath.Join(workingDir, "mydata", "file1.txt")), qt.Equals, true)
- c.Assert(bfs.IsI18n(filepath.Join(workingDir, "myi18n", "file1.txt")), qt.Equals, true)
- c.Assert(bfs.IsLayout(filepath.Join(workingDir, "mylayouts", "file1.txt")), qt.Equals, true)
c.Assert(bfs.IsStatic(filepath.Join(workingDir, "mystatic", "file1.txt")), qt.Equals, true)
- c.Assert(bfs.IsAsset(filepath.Join(workingDir, "myassets", "file1.txt")), qt.Equals, true)
contentFilename := filepath.Join(workingDir, "mycontent", "file1.txt")
c.Assert(bfs.IsContent(contentFilename), qt.Equals, true)
- rel := bfs.RelContentDir(contentFilename)
- c.Assert(rel, qt.Equals, "file1.txt")
-
// Check Work fs vs theme
checkFileContent(bfs.Work, "file-root.txt", c, "content-project")
checkFileContent(bfs.Work, "theme-root-atheme.txt", c, "content:atheme")
@@ -168,34 +163,29 @@ func TestRealDirs(t *testing.T) {
v.Set("assetDir", "myassets")
v.Set("theme", "mytheme")
- afs := hugofs.Os
-
- defer func() {
- os.RemoveAll(root)
- os.RemoveAll(themesDir)
- }()
+ afs := &hugofs.OpenFilesFs{Fs: hugofs.Os}
- c.Assert(afs.MkdirAll(filepath.Join(root, "myassets", "scss", "sf1"), 0755), qt.IsNil)
- c.Assert(afs.MkdirAll(filepath.Join(root, "myassets", "scss", "sf2"), 0755), qt.IsNil)
- c.Assert(afs.MkdirAll(filepath.Join(themesDir, "mytheme", "assets", "scss", "sf2"), 0755), qt.IsNil)
- c.Assert(afs.MkdirAll(filepath.Join(themesDir, "mytheme", "assets", "scss", "sf3"), 0755), qt.IsNil)
- c.Assert(afs.MkdirAll(filepath.Join(root, "resources"), 0755), qt.IsNil)
- c.Assert(afs.MkdirAll(filepath.Join(themesDir, "mytheme", "resources"), 0755), qt.IsNil)
+ c.Assert(afs.MkdirAll(filepath.Join(root, "myassets", "scss", "sf1"), 0o755), qt.IsNil)
+ c.Assert(afs.MkdirAll(filepath.Join(root, "myassets", "scss", "sf2"), 0o755), qt.IsNil)
+ c.Assert(afs.MkdirAll(filepath.Join(themesDir, "mytheme", "assets", "scss", "sf2"), 0o755), qt.IsNil)
+ c.Assert(afs.MkdirAll(filepath.Join(themesDir, "mytheme", "assets", "scss", "sf3"), 0o755), qt.IsNil)
+ c.Assert(afs.MkdirAll(filepath.Join(root, "resources"), 0o755), qt.IsNil)
+ c.Assert(afs.MkdirAll(filepath.Join(themesDir, "mytheme", "resources"), 0o755), qt.IsNil)
- c.Assert(afs.MkdirAll(filepath.Join(root, "myassets", "js", "f2"), 0755), qt.IsNil)
+ c.Assert(afs.MkdirAll(filepath.Join(root, "myassets", "js", "f2"), 0o755), qt.IsNil)
- afero.WriteFile(afs, filepath.Join(filepath.Join(root, "myassets", "scss", "sf1", "a1.scss")), []byte("content"), 0755)
- afero.WriteFile(afs, filepath.Join(filepath.Join(root, "myassets", "scss", "sf2", "a3.scss")), []byte("content"), 0755)
- afero.WriteFile(afs, filepath.Join(filepath.Join(root, "myassets", "scss", "a2.scss")), []byte("content"), 0755)
- afero.WriteFile(afs, filepath.Join(filepath.Join(themesDir, "mytheme", "assets", "scss", "sf2", "a3.scss")), []byte("content"), 0755)
- afero.WriteFile(afs, filepath.Join(filepath.Join(themesDir, "mytheme", "assets", "scss", "sf3", "a4.scss")), []byte("content"), 0755)
+ afero.WriteFile(afs, filepath.Join(filepath.Join(root, "myassets", "scss", "sf1", "a1.scss")), []byte("content"), 0o755)
+ afero.WriteFile(afs, filepath.Join(filepath.Join(root, "myassets", "scss", "sf2", "a3.scss")), []byte("content"), 0o755)
+ afero.WriteFile(afs, filepath.Join(filepath.Join(root, "myassets", "scss", "a2.scss")), []byte("content"), 0o755)
+ afero.WriteFile(afs, filepath.Join(filepath.Join(themesDir, "mytheme", "assets", "scss", "sf2", "a3.scss")), []byte("content"), 0o755)
+ afero.WriteFile(afs, filepath.Join(filepath.Join(themesDir, "mytheme", "assets", "scss", "sf3", "a4.scss")), []byte("content"), 0o755)
- afero.WriteFile(afs, filepath.Join(filepath.Join(themesDir, "mytheme", "resources", "t1.txt")), []byte("content"), 0755)
- afero.WriteFile(afs, filepath.Join(filepath.Join(root, "resources", "p1.txt")), []byte("content"), 0755)
- afero.WriteFile(afs, filepath.Join(filepath.Join(root, "resources", "p2.txt")), []byte("content"), 0755)
+ afero.WriteFile(afs, filepath.Join(filepath.Join(themesDir, "mytheme", "resources", "t1.txt")), []byte("content"), 0o755)
+ afero.WriteFile(afs, filepath.Join(filepath.Join(root, "resources", "p1.txt")), []byte("content"), 0o755)
+ afero.WriteFile(afs, filepath.Join(filepath.Join(root, "resources", "p2.txt")), []byte("content"), 0o755)
- afero.WriteFile(afs, filepath.Join(filepath.Join(root, "myassets", "js", "f2", "a1.js")), []byte("content"), 0755)
- afero.WriteFile(afs, filepath.Join(filepath.Join(root, "myassets", "js", "a2.js")), []byte("content"), 0755)
+ afero.WriteFile(afs, filepath.Join(filepath.Join(root, "myassets", "js", "f2", "a1.js")), []byte("content"), 0o755)
+ afero.WriteFile(afs, filepath.Join(filepath.Join(root, "myassets", "js", "a2.js")), []byte("content"), 0o755)
conf := testconfig.GetTestConfig(afs, v)
fs := hugofs.NewFrom(afs, conf.BaseConfig())
@@ -212,6 +202,83 @@ func TestRealDirs(t *testing.T) {
c.Assert(realDirs[0], qt.Equals, filepath.Join(root, "myassets/scss"))
c.Assert(realDirs[len(realDirs)-1], qt.Equals, filepath.Join(themesDir, "mytheme/assets/scss"))
+ realDirs = bfs.Assets.RealDirs("foo")
+ c.Assert(len(realDirs), qt.Equals, 0)
+
+ c.Assert(afs.OpenFiles(), qt.HasLen, 0)
+}
+
+func TestWatchFilenames(t *testing.T) {
+ t.Parallel()
+ files := `
+-- hugo.toml --
+theme = "t1"
+[[module.mounts]]
+source = 'content'
+target = 'content'
+[[module.mounts]]
+source = 'content2'
+target = 'content/c2'
+[[module.mounts]]
+source = "hugo_stats.json"
+target = "assets/watching/hugo_stats.json"
+-- hugo_stats.json --
+Some stats.
+-- content/foo.md --
+foo
+-- content2/bar.md --
+-- themes/t1/layouts/_default/single.html --
+{{ .Content }}
+-- themes/t1/static/f1.txt --
+`
+ b := hugolib.Test(t, files)
+ bfs := b.H.BaseFs
+ watchFilenames := bfs.WatchFilenames()
+ b.Assert(watchFilenames, qt.HasLen, 6)
+}
+
+func TestNoSymlinks(t *testing.T) {
+ if runtime.GOOS == "windows" {
+ t.Skip("skip on Windows")
+ }
+ files := `
+-- hugo.toml --
+theme = "t1"
+-- content/a/foo.md --
+foo
+-- static/a/f1.txt --
+F1 text
+-- themes/t1/layouts/_default/single.html --
+{{ .Content }}
+-- themes/t1/static/a/f1.txt --
+`
+ tmpDir := t.TempDir()
+
+ wd, _ := os.Getwd()
+
+ for _, component := range []string{"content", "static"} {
+ aDir := filepath.Join(tmpDir, component, "a")
+ bDir := filepath.Join(tmpDir, component, "b")
+ os.MkdirAll(aDir, 0o755)
+ os.MkdirAll(bDir, 0o755)
+ os.Chdir(bDir)
+ os.Symlink("../a", "c")
+ }
+
+ os.Chdir(wd)
+
+ b := hugolib.NewIntegrationTestBuilder(
+ hugolib.IntegrationTestConfig{
+ T: t,
+ TxtarString: files,
+ NeedsOsFS: true,
+ WorkingDir: tmpDir,
+ },
+ ).Build()
+
+ bfs := b.H.BaseFs
+ watchFilenames := bfs.WatchFilenames()
+ b.Assert(watchFilenames, qt.HasLen, 10)
}
func TestStaticFs(t *testing.T) {
@@ -228,10 +295,10 @@ func TestStaticFs(t *testing.T) {
themeStaticDir := filepath.Join(workDir, "themes", "t1", "static")
themeStaticDir2 := filepath.Join(workDir, "themes", "t2", "static")
- afero.WriteFile(afs, filepath.Join(workDir, "mystatic", "f1.txt"), []byte("Hugo Rocks!"), 0755)
- afero.WriteFile(afs, filepath.Join(themeStaticDir, "f1.txt"), []byte("Hugo Themes Rocks!"), 0755)
- afero.WriteFile(afs, filepath.Join(themeStaticDir, "f2.txt"), []byte("Hugo Themes Still Rocks!"), 0755)
- afero.WriteFile(afs, filepath.Join(themeStaticDir2, "f2.txt"), []byte("Hugo Themes Rocks in t2!"), 0755)
+ afero.WriteFile(afs, filepath.Join(workDir, "mystatic", "f1.txt"), []byte("Hugo Rocks!"), 0o755)
+ afero.WriteFile(afs, filepath.Join(themeStaticDir, "f1.txt"), []byte("Hugo Themes Rocks!"), 0o755)
+ afero.WriteFile(afs, filepath.Join(themeStaticDir, "f2.txt"), []byte("Hugo Themes Still Rocks!"), 0o755)
+ afero.WriteFile(afs, filepath.Join(themeStaticDir2, "f2.txt"), []byte("Hugo Themes Rocks in t2!"), 0o755)
conf := testconfig.GetTestConfig(afs, v)
fs := hugofs.NewFrom(afs, conf.BaseConfig())
@@ -273,17 +340,15 @@ func TestStaticFsMultiHost(t *testing.T) {
themeStaticDir := filepath.Join(workDir, "themes", "t1", "static")
- afero.WriteFile(afs, filepath.Join(workDir, "mystatic", "f1.txt"), []byte("Hugo Rocks!"), 0755)
- afero.WriteFile(afs, filepath.Join(workDir, "static_no", "f1.txt"), []byte("Hugo Rocks in Norway!"), 0755)
+ afero.WriteFile(afs, filepath.Join(workDir, "mystatic", "f1.txt"), []byte("Hugo Rocks!"), 0o755)
+ afero.WriteFile(afs, filepath.Join(workDir, "static_no", "f1.txt"), []byte("Hugo Rocks in Norway!"), 0o755)
- afero.WriteFile(afs, filepath.Join(themeStaticDir, "f1.txt"), []byte("Hugo Themes Rocks!"), 0755)
- afero.WriteFile(afs, filepath.Join(themeStaticDir, "f2.txt"), []byte("Hugo Themes Still Rocks!"), 0755)
+ afero.WriteFile(afs, filepath.Join(themeStaticDir, "f1.txt"), []byte("Hugo Themes Rocks!"), 0o755)
+ afero.WriteFile(afs, filepath.Join(themeStaticDir, "f2.txt"), []byte("Hugo Themes Still Rocks!"), 0o755)
conf := testconfig.GetTestConfig(afs, v)
fs := hugofs.NewFrom(afs, conf.BaseConfig())
- fmt.Println("IS", conf.IsMultihost())
-
p, err := paths.New(fs, conf)
c.Assert(err, qt.IsNil)
bfs, err := filesystems.NewBase(p, nil)
@@ -298,54 +363,119 @@ func TestStaticFsMultiHost(t *testing.T) {
}
func TestMakePathRelative(t *testing.T) {
- c := qt.New(t)
- v := config.New()
- afs := afero.NewMemMapFs()
- workDir := "mywork"
- v.Set("workingDir", workDir)
+ files := `
+-- hugo.toml --
+[[module.mounts]]
+source = "bar.txt"
+target = "assets/foo/baz.txt"
+[[module.imports]]
+path = "t1"
+[[module.imports.mounts]]
+source = "src"
+target = "assets/foo/bar"
+-- bar.txt --
+Bar.
+-- themes/t1/src/main.js --
+Main.
+`
+ b := hugolib.Test(t, files)
+
+ rel, found := b.H.BaseFs.Assets.MakePathRelative(filepath.FromSlash("/themes/t1/src/main.js"))
+ b.Assert(found, qt.Equals, true)
+ b.Assert(rel, qt.Equals, filepath.FromSlash("foo/bar/main.js"))
+
+ rel, found = b.H.BaseFs.Assets.MakePathRelative(filepath.FromSlash("/bar.txt"))
+ b.Assert(found, qt.Equals, true)
+ b.Assert(rel, qt.Equals, filepath.FromSlash("foo/baz.txt"))
+}
- c.Assert(afs.MkdirAll(filepath.Join(workDir, "dist", "d1"), 0777), qt.IsNil)
- c.Assert(afs.MkdirAll(filepath.Join(workDir, "static", "d2"), 0777), qt.IsNil)
- c.Assert(afs.MkdirAll(filepath.Join(workDir, "dust", "d2"), 0777), qt.IsNil)
-
- moduleCfg := map[string]any{
- "mounts": []any{
- map[string]any{
- "source": "dist",
- "target": "static/mydist",
- },
- map[string]any{
- "source": "dust",
- "target": "static/foo/bar",
- },
- map[string]any{
- "source": "static",
- "target": "static",
- },
+func TestAbsProjectContentDir(t *testing.T) {
+ tempDir := t.TempDir()
+
+ files := `
+-- hugo.toml --
+[[module.mounts]]
+source = "content"
+target = "content"
+-- content/foo.md --
+---
+title: "Foo"
+---
+`
+
+ b := hugolib.NewIntegrationTestBuilder(
+ hugolib.IntegrationTestConfig{
+ T: t,
+ WorkingDir: tempDir,
+ TxtarString: files,
},
- }
-
- v.Set("module", moduleCfg)
-
- conf := testconfig.GetTestConfig(afs, v)
- fs := hugofs.NewFrom(afs, conf.BaseConfig())
+ ).Build()
+
+ abs1 := filepath.Join(tempDir, "content", "foo.md")
+ rel, abs2, err := b.H.BaseFs.AbsProjectContentDir("foo.md")
+ b.Assert(err, qt.IsNil)
+ b.Assert(abs2, qt.Equals, abs1)
+ b.Assert(rel, qt.Equals, filepath.FromSlash("foo.md"))
+ rel2, abs3, err := b.H.BaseFs.AbsProjectContentDir(abs1)
+ b.Assert(err, qt.IsNil)
+ b.Assert(abs3, qt.Equals, abs1)
+ b.Assert(rel2, qt.Equals, rel)
+}
- p, err := paths.New(fs, conf)
- c.Assert(err, qt.IsNil)
- bfs, err := filesystems.NewBase(p, nil)
- c.Assert(err, qt.IsNil)
+func TestContentReverseLookup(t *testing.T) {
+ files := `
+-- README.md --
+---
+title: README
+---
+-- blog/b1.md --
+---
+title: b1
+---
+-- docs/d1.md --
+---
+title: d1
+---
+-- hugo.toml --
+baseURL = "https://example.com/"
+[module]
+[[module.mounts]]
+source = "layouts"
+target = "layouts"
+[[module.mounts]]
+source = "README.md"
+target = "content/_index.md"
+[[module.mounts]]
+source = "blog"
+target = "content/posts"
+[[module.mounts]]
+source = "docs"
+target = "content/mydocs"
+-- layouts/index.html --
+Home.
+
+`
+ b := hugolib.Test(t, files)
+
+ b.AssertFileContent("public/index.html", "Home.")
+
+ stat := func(path string) hugofs.FileMetaInfo {
+ ps, err := b.H.BaseFs.Content.ReverseLookup(filepath.FromSlash(path))
+ b.Assert(err, qt.IsNil)
+ b.Assert(ps, qt.HasLen, 1)
+ first := ps[0]
+ fi, err := b.H.BaseFs.Content.Fs.Stat(filepath.FromSlash(first.Path))
+ b.Assert(err, qt.IsNil)
+ b.Assert(fi, qt.Not(qt.IsNil))
+ return fi.(hugofs.FileMetaInfo)
+ }
- sfs := bfs.Static[""]
- c.Assert(sfs, qt.Not(qt.IsNil))
+ sfs := b.H.Fs.Source
- makeRel := func(s string) string {
- r, _ := sfs.MakePathRelative(s)
- return r
- }
+ _, err := sfs.Stat("blog/b1.md")
+ b.Assert(err, qt.Not(qt.IsNil))
- c.Assert(makeRel(filepath.Join(workDir, "dist", "d1", "foo.txt")), qt.Equals, filepath.FromSlash("mydist/d1/foo.txt"))
- c.Assert(makeRel(filepath.Join(workDir, "static", "d2", "foo.txt")), qt.Equals, filepath.FromSlash("d2/foo.txt"))
- c.Assert(makeRel(filepath.Join(workDir, "dust", "d3", "foo.txt")), qt.Equals, filepath.FromSlash("foo/bar/d3/foo.txt"))
+ _ = stat("blog/b1.md")
}
func checkFileCount(fs afero.Fs, dirname string, c *qt.C, expected int) {
@@ -374,10 +504,7 @@ func countFilesAndGetFilenames(fs afero.Fs, dirname string) (int, []string, erro
counter := 0
var filenames []string
- wf := func(path string, info hugofs.FileMetaInfo, err error) error {
- if err != nil {
- return err
- }
+ wf := func(path string, info hugofs.FileMetaInfo) error {
if !info.IsDir() {
counter++
}
@@ -403,9 +530,9 @@ func countFilesAndGetFilenames(fs afero.Fs, dirname string) (int, []string, erro
func setConfigAndWriteSomeFilesTo(fs afero.Fs, v config.Provider, key, val string, num int) {
workingDir := v.GetString("workingDir")
v.Set(key, val)
- fs.Mkdir(val, 0755)
+ fs.Mkdir(val, 0o755)
for i := 0; i < num; i++ {
filename := filepath.Join(workingDir, val, fmt.Sprintf("f%d.txt", i+1))
- afero.WriteFile(fs, filename, []byte(fmt.Sprintf("content:%s:%d", key, i+1)), 0755)
+ afero.WriteFile(fs, filename, []byte(fmt.Sprintf("content:%s:%d", key, i+1)), 0o755)
}
}